StrIdx 0.1.2 → 0.1.4

data/thread_pool.hpp

@@ -0,0 +1,98 @@
+
+// Based on example in https://www.geeksforgeeks.org/thread-pool-in-cpp/
+
+#include <condition_variable>
+#include <functional>
+#include <iostream>
+#include <mutex>
+#include <queue>
+#include <thread>
+#include <algorithm>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <string>
+#include <chrono>
+
+class ThreadPool {
+public:
+  // Create a thread pool with given number of threads
+  ThreadPool(size_t num_threads) {
+
+    // Creating worker threads
+    for (size_t i = 0; i < num_threads; ++i) {
+      workerThreads.emplace_back([this] {
+        while (true) {
+          std::function<void()> task;
+          {
+            std::unique_lock<std::mutex> lock(mu_queue);
+
+            // Waiting until there is a task to execute or the pool is stopped
+            cv_.wait(lock, [this] { return !taskQueue.empty() || stop_; });
+
+            // Exit the thread in case the pool is stopped and there are no tasks
+            if (stop_ && taskQueue.empty()) {
+              return;
+            }
+
+            // Get the next task from the queue
+            task = std::move(taskQueue.front());
+            taskQueue.pop();
+          }
+
+          task();
+        }
+      });
+    }
+  }
+
+  // Destructor to stop the thread pool
+  ~ThreadPool() {
+    {
+      std::lock_guard<std::mutex> lock(mu_queue);
+      stop_ = true;
+    }
+
+    // Notify all threads
+    cv_.notify_all();
+
+    // Joining all worker threads to ensure they have
+    // completed their tasks
+    for (auto &thread : workerThreads) {
+      thread.join();
+    }
+  }
+
+  // Wait until all tasks assigned to the threads have been finished
+  void waitUntilDone() {
+    while (true) {
+      {
+        std::lock_guard<std::mutex> guard(mu_queue);
+        if (taskQueue.empty()) {
+          return;
+        }
+      }
+      std::this_thread::sleep_for(std::chrono::milliseconds(50));
+    }
+  }
+
+  // Enqueue task for execution by the thread pool
+  void enqueue(std::function<void()> task) {
+    {
+      std::lock_guard<std::mutex> lock(mu_queue);
+      taskQueue.emplace(move(task));
+    }
+    cv_.notify_one();
+  }
+
+private:
+  std::vector<std::thread> workerThreads;
+  std::queue<std::function<void()>> taskQueue;
+  std::mutex mu_queue;
+
+  // Condition variable to signal changes in the state of the tasks queue
+  std::condition_variable cv_;
+
+  // Flag to indicate whether the thread pool should stop
+  bool stop_ = false;
+};

data/unit_tests.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+cmake -S . -B build
+cmake --build build
+cd build && ctest

data/unittest.cpp

@@ -0,0 +1,147 @@
+
+#include <gtest/gtest.h>
+#include "stridx.hpp"
+#include <cmath>
+#include <memory>
+
+TEST(SplitString, MatchSize) {
+  std::vector<std::string> svec = StrIdx::splitString("foo/bar/test1.txt", '/');
+  EXPECT_EQ(svec.size(), 3);
+  if (svec.size() == 3) {
+    EXPECT_EQ(svec[0].size(), 3);
+    EXPECT_EQ(svec[2].size(), 9);
+  }
+}
+
+std::vector<std::string> flist{"./drivers/char/hw_random/nomadik-rng.c",
+                               "./drivers/pinctrl/nomadik",
+                               "./drivers/clk/clk-nomadik.c",
+                               "./drivers/gpio/gpio-nomadik.c",
+                               "./drivers/i2c/busses/i2c-nomadik.c",
+                               "./drivers/clocksource/nomadik-mtu.c",
+                               "./drivers/gpu/drm/pl111/pl111_nomadik.h",
+                               "./drivers/gpu/drm/pl111/pl111_nomadik.c",
+                               "./drivers/pinctrl/nomadik/pinctrl-nomadik.c",
+                               "./drivers/input/keyboard/nomadik-ske-keypad.c",
+                               "./drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c",
+                               "./drivers/pinctrl/nomadik/pinctrl-nomadik-stn8815.c",
+                               "./drivers/char/hw_random/omap-rng.c",
+                               "./drivers/char/hw_random/omap3-rom-rng.c",
+                               "./include/dt-bindings/pinctrl/nomadik.h",
+                               "./Documentation/devicetree/bindings/arm/ste-nomadik.txt"};
+
+std::vector<float> target_scores{0.342944, 0.271396, 0.271126, 0.270893, 0.270431, 0.270355,
+                                 0.270088, 0.270088, 0.26987,  0.269776, 0.269574, 0.269538,
+                                 0.236358, 0.236074, 0.224804, 0.224238};
+
+void scoreTest(bool threaded) {
+
+  StrIdx::StringIndex idx('/'); // Separate directories using unix style "/" char
+  std::string query = "rngnomadriv";
+
+  int i = 1;
+  for (const auto &str : flist) {
+    if (threaded) {
+      idx.addStrToIndexThreaded(str, i);
+    } else {
+      idx.addStrToIndex(str, i);
+    }
+    i++;
+  }
+  const std::vector<std::pair<float, int>> &results = idx.findSimilar(query);
+
+  std::cout << results[0].first;
+  EXPECT_EQ(results[0].second, 1);
+  if (results.size() == 16) {
+    int i = 0;
+    for (const auto &res : results) {
+      // Check if first five digits of the scores match
+      EXPECT_EQ(std::floor(res.first * 1e5), std::floor(1e5 * target_scores[i]));
+      i++;
+    }
+  }
+}
+
+TEST(IndexSearch, MatchingScoresSingleThread) { scoreTest(false); }
+TEST(IndexSearch, MatchingScoresThreaded) { scoreTest(true); }
+
+class IndexTest : public testing::Test {
+protected:
+  std::unique_ptr<StrIdx::StringIndex> idx = std::make_unique<StrIdx::StringIndex>('/');
+
+  IndexTest() {}
+
+  void SetUp() override {
+    // Code here will be called immediately after the constructor (right
+    // before each test).
+    idx = std::make_unique<StrIdx::StringIndex>('/');
+  }
+
+  void TearDown() override {
+    // Code here will be called immediately after each test (right
+    // before the destructor).
+  }
+};
+
+TEST_F(IndexTest, BinaryRepresentation1) {
+  int64_t num = idx->getKeyAtIdx("abcdefgh", 0, 8);
+  std::string s = StrIdx::int64ToBinaryString(num);
+  //                a       b       c       d ...
+  EXPECT_TRUE(s == "0110000101100010011000110110010001100101011001100110011101101000");
+}
+
+TEST_F(IndexTest, BinaryRepresentation2) {
+  int64_t num = idx->getKeyAtIdx("abcdefgh", 0, 1);
+  std::string s = StrIdx::int64ToBinaryString(num);
+  EXPECT_TRUE(
+      s == "0000000000000000000000000000000000000000000000000000000001100001"); // 01100001 == "a"
+}
+TEST_F(IndexTest, BinaryRepresentation3) {
+  int64_t num = idx->getKeyAtIdx("abcdefgh", 7, 1);
+  std::string s = StrIdx::int64ToBinaryString(num);
+  EXPECT_TRUE(
+      s == "0000000000000000000000000000000000000000000000000000000001101000"); // 01101000 == "h"
+}
+
+TEST_F(IndexTest, AccessString) {
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 0);
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadiksdf.c", 2);
+  idx->addStrToIndex("./drivers//i2c///busses////aa-i2c-nomadiksdf.c", 3);
+  idx->addStrToIndex("/test/foo/bar.txt", 4);
+  idx->addStrToIndex("bar.txt", 5);
+
+  EXPECT_EQ(idx->size(), 5);
+  EXPECT_STREQ(idx->getString(0).c_str(), "./drivers/i2c/busses/i2c-nomadik.c");
+
+  // TODO: does not work yet
+  //  EXPECT_STREQ(idx->getString(3).c_str(), "./drivers//i2c///busses////aa-i2c-nomadiksdf.c");
+
+  // TODO: does not work yet
+  //  EXPECT_STREQ(idx->getString(4).c_str(), "/test/foo/bar.txt");
+  EXPECT_STREQ(idx->getString(5).c_str(), "bar.txt");
+}
+
+TEST_F(IndexTest, Size1) {
+  // Should not add different files with same id
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 0);
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadiksdf.c", 0);
+
+  // Should not be added because essentially same as 0:
+  idx->addStrToIndex("./drivers//i2c///busses////i2c-nomadik.c", 44);
+
+  // Should not add same file with different id
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 1);
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 2);
+
+  EXPECT_EQ(idx->size(), 1);
+
+  // Test no-overwriting
+  EXPECT_STREQ(idx->getString(0).c_str(), "./drivers/i2c/busses/i2c-nomadik.c");
+}
+
+TEST_F(IndexTest, Size2) {
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 22);
+  idx->addStrToIndex("./Documentation/devicetree/bindings/arm/ste-nomadik.txt", 1);
+  idx->addStrToIndex("./Documentation/devicetree/bindings/arm/ste-nomadik33.txt", 3);
+  EXPECT_EQ(idx->size(), 3);
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: StrIdx
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
 platform: ruby
 authors:
 - Sami Sieranoja
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-05-08 00:00:00.000000000 Z
+date: 2024-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,23 +38,123 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 13.1.0
+- !ruby/object:Gem::Dependency
+  name: tty-cursor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+- !ruby/object:Gem::Dependency
+  name: tty-prompt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.23.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.23.1
+- !ruby/object:Gem::Dependency
+  name: tty-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: tty-screen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+- !ruby/object:Gem::Dependency
+  name: pastel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.1
 description: " Fast fuzzy string similarity search and indexing (for filenames)"
 email:
 - sami.sieranoja@gmail.com
-executables: []
+executables:
+- stridx.rb
 extensions:
 - rubyext/extconf.rb
 extra_rdoc_files: []
 files:
+- CMakeLists.txt
+- Gemfile
 - LICENSE
 - Makefile
 - README.md
 - demo.cpp
+- exe/stridx.rb
 - flist.txt
+- gem_install
+- py_example.py
+- py_interf.cpp
 - rubyext/extconf.rb
 - rubyext/ruby_interf.cpp
+- runserver.rb
+- server.rb
+- setup.py
+- stridx-screencast.mp4
+- stridx-tty.rb
+- stridx.gemspec
 - stridx.hpp
 - test.rb
+- thread_pool.hpp
+- unit_tests.sh
+- unittest.cpp
 - unordered_dense.h
 homepage: https://github.com/SamiSieranoja/stridx
 licenses: