StrIdx 0.1.3 → 0.1.5

data/test.rb

@@ -1,8 +1,13 @@
 #!/usr/bin/env ruby
 
+$:.unshift File.dirname(__FILE__)
+
 require "stridx"
 idx = StrIdx::StringIndex.new
 
+# "/" for unix-style file paths
+idx.setDirSeparator("/") #(comment out if not file paths)
+
 t = Time.new
 fn = File.expand_path("flist.txt")
 lines = IO.read(fn).lines.collect { |x| x.strip }
@@ -14,12 +19,12 @@ end
 
 idx_time = Time.new 
 # Time to start the threadpool to process indexing
-puts "\nIndexing launch time (#{lines.size} files}): #{(idx_time - t).round(4)} seconds"
+puts "\nIndexing launch time (#{lines.size} files): #{(idx_time - t).round(4)} seconds"
 
 idx.waitUntilDone() # Not necessary, will be called by idx.find
 idx_time = Time.new 
 # Time when all threads have completed
-puts "\nIndexing completed time (#{lines.size} files}): #{(idx_time - t).round(4)} seconds"
+puts "\nIndexing completed time (#{lines.size} files): #{(idx_time - t).round(4)} seconds"
 
 query = "rngnomadriv"
 res = idx.find(query)

data/thread_pool.hpp

@@ -21,14 +21,21 @@ public:
 
     // Creating worker threads
     for (size_t i = 0; i < num_threads; ++i) {
-      workerThreads.emplace_back([this] {
+      workDone.push_back(false);
+      workerThreads.emplace_back([this, i] {
         while (true) {
+
+          // std::lock_guard<std::mutex) mu_guard(mu_done);
           std::function<void()> task;
           {
             std::unique_lock<std::mutex> lock(mu_queue);
 
             // Waiting until there is a task to execute or the pool is stopped
-            cv_.wait(lock, [this] { return !taskQueue.empty() || stop_; });
+
+            workDone[i] = true;
+            cv_.wait(lock, [this, i] { return !taskQueue.empty() || stop_; });
+
+            workDone[i] = false;
 
             // Exit the thread in case the pool is stopped and there are no tasks
             if (stop_ && taskQueue.empty()) {
@@ -39,7 +46,6 @@ public:
             task = std::move(taskQueue.front());
             taskQueue.pop();
           }
-
           task();
         }
       });
@@ -68,11 +74,18 @@ public:
     while (true) {
       {
         std::lock_guard<std::mutex> guard(mu_queue);
-        if (taskQueue.empty()) {
+        bool done = true;
+
+        for (auto x : workDone) {
+          if (x == false) {
+            done = false;
+          }
+        }
+        if (done && taskQueue.empty()) {
           return;
         }
       }
-      std::this_thread::sleep_for(std::chrono::milliseconds(50));
+      std::this_thread::sleep_for(std::chrono::milliseconds(10));
     }
   }
 
@@ -89,6 +102,8 @@ private:
   std::vector<std::thread> workerThreads;
   std::queue<std::function<void()>> taskQueue;
   std::mutex mu_queue;
+  std::vector<bool> workDone;
+  std::mutex mu_done;
 
   // Condition variable to signal changes in the state of the tasks queue
   std::condition_variable cv_;

data/unit_tests.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+cmake -S . -B build
+cmake --build build
+cd build && ctest

data/unittest.cpp

@@ -0,0 +1,189 @@
+
+#include <gtest/gtest.h>
+#include "stridx.hpp"
+#include <cmath>
+#include <memory>
+#include <chrono>
+#include <thread>
+
+TEST(SplitString, MatchSize) {
+  std::vector<std::string> svec = StrIdx::splitString("foo/bar/test1.txt", '/');
+  EXPECT_EQ(svec.size(), 3);
+  if (svec.size() == 3) {
+    EXPECT_EQ(svec[0].size(), 3);
+    EXPECT_EQ(svec[2].size(), 9);
+  }
+}
+
+std::vector<std::string> flist{
+                               "./drivers/char/hw_random/nomadik-rng.c",
+                               "./drivers/pinctrl/nomadik",
+                               "./drivers/clk/clk-nomadik.c",
+                               "./drivers/gpio/gpio-nomadik.c",
+                               "./drivers/i2c/busses/i2c-nomadik.c",
+                               "./drivers/clocksource/nomadik-mtu.c",
+                               "./drivers/gpu/drm/pl111/pl111_nomadik.h",
+                               "./drivers/gpu/drm/pl111/pl111_nomadik.c",
+                               "./drivers/pinctrl/nomadik/pinctrl-nomadik.c",
+                               "./drivers/input/keyboard/nomadik-ske-keypad.c",
+                               "./drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c",
+                               "./drivers/pinctrl/nomadik/pinctrl-nomadik-stn8816.c",
+                               "./drivers/char/hw_random/omap-rng.c",
+                               "./drivers/char/hw_random/omap3-rom-rng.c",
+                               "./include/dt-bindings/pinctrl/nomadik.h",
+                               "./Documentation/devicetree/bindings/arm/ste-nomadik.txt"};
+
+std::vector<float> target_scores{0.342944, 0.271396,  0.271126, 0.270893, 0.270431, 0.270355,
+                                 0.270088, 0.270088, 0.26987,  0.269776, 0.269574, 0.269538,
+                                 0.236358, 0.236074, 0.224804, 0.224238};
+
+void createIndex(bool threaded) {
+
+  StrIdx::StringIndex idx('/'); // Separate directories using unix style "/" char
+  std::string query = "rngnomadriv";
+
+  int i = 1;
+  for (const auto &str : flist) {
+    if (threaded && i > 2) {
+      idx.addStrToIndexThreaded(str, i);
+    } else {
+      idx.addStrToIndex(str, i);
+    }
+    i++;
+  }
+
+  idx.waitUntilReady();
+
+  EXPECT_EQ(idx.size(), 16);
+}
+
+void scoreTest(bool threaded, bool runSearch) {
+
+  StrIdx::StringIndex idx('/'); // Separate directories using unix style "/" char
+  std::string query = "rngnomadriv";
+
+  int i = 1;
+  for (const auto &str : flist) {
+    if (threaded) {
+      idx.addStrToIndexThreaded(str, i);
+    } else {
+      idx.addStrToIndex(str, i);
+    }
+    i++;
+  }
+
+  idx.waitUntilReady();
+  std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+  std::cout << "s:" << idx.size() << ":";
+  EXPECT_EQ(idx.size(), 16);
+
+  if (runSearch) {
+    const std::vector<std::pair<float, int>> &results = idx.findSimilar(query);
+
+    std::cout << results[0].first;
+    EXPECT_EQ(results[0].second, 1);
+    if (results.size() == 16) {
+      int i = 0;
+      for (const auto &res : results) {
+        // Check if first five digits of the scores match
+        std::cout<< "{" << res.first << " " << target_scores[i] << "\n";
+        EXPECT_EQ(std::floor(res.first * 1e5), std::floor(1e5 * target_scores[i]));
+        i++;
+      }
+    }
+  }
+  
+  // EXPECT_EQ(0,1);
+}
+
+TEST(Index, Create) { createIndex(false); }
+TEST(Index, CreateThreaded) { createIndex(true); }
+TEST(IndexSearch, MatchingScoresSingleThread) {
+  scoreTest(false, true);
+}
+TEST(IndexSearch, MatchingScoresThreaded) {
+  for (int i = 0; i < 3; i++) {
+    scoreTest(true, true);
+  }
+}
+
+class IndexTest : public testing::Test {
+protected:
+  std::unique_ptr<StrIdx::StringIndex> idx = std::make_unique<StrIdx::StringIndex>('/');
+
+  IndexTest() {}
+
+  void SetUp() override {
+    // Code here will be called immediately after the constructor (right
+    // before each test).
+    idx = std::make_unique<StrIdx::StringIndex>('/');
+  }
+
+  void TearDown() override {
+    // Code here will be called immediately after each test (right
+    // before the destructor).
+  }
+};
+
+TEST_F(IndexTest, BinaryRepresentation1) {
+  int64_t num = idx->getKeyAtIdx("abcdefgh", 0, 8);
+  std::string s = StrIdx::int64ToBinaryString(num);
+  //                a       b       c       d ...
+  EXPECT_TRUE(s == "0110000101100010011000110110010001100101011001100110011101101000");
+}
+
+TEST_F(IndexTest, BinaryRepresentation2) {
+  int64_t num = idx->getKeyAtIdx("abcdefgh", 0, 1);
+  std::string s = StrIdx::int64ToBinaryString(num);
+  EXPECT_TRUE(
+      s == "0000000000000000000000000000000000000000000000000000000001100001"); // 01100001 == "a"
+}
+TEST_F(IndexTest, BinaryRepresentation3) {
+  int64_t num = idx->getKeyAtIdx("abcdefgh", 7, 1);
+  std::string s = StrIdx::int64ToBinaryString(num);
+  EXPECT_TRUE(
+      s == "0000000000000000000000000000000000000000000000000000000001101000"); // 01101000 == "h"
+}
+
+TEST_F(IndexTest, AccessString) {
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 0);
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadiksdf.c", 2);
+  idx->addStrToIndex("./drivers//i2c///busses////aa-i2c-nomadiksdf.c", 3);
+  idx->addStrToIndex("/test/foo/bar.txt", 4);
+  idx->addStrToIndex("bar.txt", 5);
+
+  EXPECT_EQ(idx->size(), 5);
+  EXPECT_STREQ(idx->getString(0).c_str(), "./drivers/i2c/busses/i2c-nomadik.c");
+
+  // TODO: does not work yet
+  //  EXPECT_STREQ(idx->getString(3).c_str(), "./drivers//i2c///busses////aa-i2c-nomadiksdf.c");
+
+  // TODO: does not work yet
+  //  EXPECT_STREQ(idx->getString(4).c_str(), "/test/foo/bar.txt");
+  EXPECT_STREQ(idx->getString(5).c_str(), "bar.txt");
+}
+
+TEST_F(IndexTest, Size1) {
+  // Should not add different files with same id
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 0);
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadiksdf.c", 0);
+
+  // Should not be added because essentially same as 0:
+  idx->addStrToIndex("./drivers//i2c///busses////i2c-nomadik.c", 44);
+
+  // Should not add same file with different id
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 1);
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 2);
+
+  EXPECT_EQ(idx->size(), 1);
+
+  // Test no-overwriting
+  EXPECT_STREQ(idx->getString(0).c_str(), "./drivers/i2c/busses/i2c-nomadik.c");
+}
+
+TEST_F(IndexTest, Size2) {
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 22);
+  idx->addStrToIndex("./Documentation/devicetree/bindings/arm/ste-nomadik.txt", 1);
+  idx->addStrToIndex("./Documentation/devicetree/bindings/arm/ste-nomadik33.txt", 3);
+  EXPECT_EQ(idx->size(), 3);
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: StrIdx
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Sami Sieranoja
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-05-24 00:00:00.000000000 Z
+date: 2025-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,24 +38,123 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 13.1.0
+- !ruby/object:Gem::Dependency
+  name: tty-cursor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+- !ruby/object:Gem::Dependency
+  name: tty-prompt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.23.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.23.1
+- !ruby/object:Gem::Dependency
+  name: tty-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: tty-screen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+- !ruby/object:Gem::Dependency
+  name: pastel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.1
 description: " Fast fuzzy string similarity search and indexing (for filenames)"
 email:
 - sami.sieranoja@gmail.com
-executables: []
+executables:
+- stridx.rb
 extensions:
 - rubyext/extconf.rb
 extra_rdoc_files: []
 files:
+- CMakeLists.txt
+- Gemfile
 - LICENSE
 - Makefile
 - README.md
 - demo.cpp
+- exe/stridx.rb
 - flist.txt
+- gem_install
+- py_example.py
+- py_interf.cpp
 - rubyext/extconf.rb
 - rubyext/ruby_interf.cpp
+- runserver.rb
+- server.rb
+- setup.py
+- stridx-screencast.mp4
+- stridx-tty.rb
+- stridx.gemspec
 - stridx.hpp
 - test.rb
 - thread_pool.hpp
+- unit_tests.sh
+- unittest.cpp
 - unordered_dense.h
 homepage: https://github.com/SamiSieranoja/stridx
 licenses:
@@ -63,7 +162,7 @@ licenses:
 metadata:
   source_code_uri: https://github.com/SamiSieranoja/stridx
   homepage_uri: https://github.com/SamiSieranoja/stridx
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -79,8 +178,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key: 
+rubygems_version: 3.4.20
+signing_key:
 specification_version: 4
 summary: StrIdx
 test_files: []