native-vector-store 0.1.0 → 0.3.0

package/docs/styles/prettify-jsdoc.css

@@ -0,0 +1,111 @@
+/* JSDoc prettify.js theme */
+
+/* plain text */
+.pln {
+  color: #000000;
+  font-weight: normal;
+  font-style: normal;
+}
+
+/* string content */
+.str {
+  color: #006400;
+  font-weight: normal;
+  font-style: normal;
+}
+
+/* a keyword */
+.kwd {
+  color: #000000;
+  font-weight: bold;
+  font-style: normal;
+}
+
+/* a comment */
+.com {
+  font-weight: normal;
+  font-style: italic;
+}
+
+/* a type name */
+.typ {
+  color: #000000;
+  font-weight: normal;
+  font-style: normal;
+}
+
+/* a literal value */
+.lit {
+  color: #006400;
+  font-weight: normal;
+  font-style: normal;
+}
+
+/* punctuation */
+.pun {
+  color: #000000;
+  font-weight: bold;
+  font-style: normal;
+}
+
+/* lisp open bracket */
+.opn {
+  color: #000000;
+  font-weight: bold;
+  font-style: normal;
+}
+
+/* lisp close bracket */
+.clo {
+  color: #000000;
+  font-weight: bold;
+  font-style: normal;
+}
+
+/* a markup tag name */
+.tag {
+  color: #006400;
+  font-weight: normal;
+  font-style: normal;
+}
+
+/* a markup attribute name */
+.atn {
+  color: #006400;
+  font-weight: normal;
+  font-style: normal;
+}
+
+/* a markup attribute value */
+.atv {
+  color: #006400;
+  font-weight: normal;
+  font-style: normal;
+}
+
+/* a declaration */
+.dec {
+  color: #000000;
+  font-weight: bold;
+  font-style: normal;
+}
+
+/* a variable name */
+.var {
+  color: #000000;
+  font-weight: normal;
+  font-style: normal;
+}
+
+/* a function name */
+.fun {
+  color: #000000;
+  font-weight: bold;
+  font-style: normal;
+}
+
+/* Specify class=linenums on a pre to get line numbering */
+ol.linenums {
+  margin-top: 0;
+  margin-bottom: 0;
+}

package/docs/styles/prettify-tomorrow.css

@@ -0,0 +1,132 @@
+/* Tomorrow Theme */
+/* Original theme - https://github.com/chriskempson/tomorrow-theme */
+/* Pretty printing styles. Used with prettify.js. */
+/* SPAN elements with the classes below are added by prettyprint. */
+/* plain text */
+.pln {
+  color: #4d4d4c; }
+
+@media screen {
+  /* string content */
+  .str {
+    color: #718c00; }
+
+  /* a keyword */
+  .kwd {
+    color: #8959a8; }
+
+  /* a comment */
+  .com {
+    color: #8e908c; }
+
+  /* a type name */
+  .typ {
+    color: #4271ae; }
+
+  /* a literal value */
+  .lit {
+    color: #f5871f; }
+
+  /* punctuation */
+  .pun {
+    color: #4d4d4c; }
+
+  /* lisp open bracket */
+  .opn {
+    color: #4d4d4c; }
+
+  /* lisp close bracket */
+  .clo {
+    color: #4d4d4c; }
+
+  /* a markup tag name */
+  .tag {
+    color: #c82829; }
+
+  /* a markup attribute name */
+  .atn {
+    color: #f5871f; }
+
+  /* a markup attribute value */
+  .atv {
+    color: #3e999f; }
+
+  /* a declaration */
+  .dec {
+    color: #f5871f; }
+
+  /* a variable name */
+  .var {
+    color: #c82829; }
+
+  /* a function name */
+  .fun {
+    color: #4271ae; } }
+/* Use higher contrast and text-weight for printable form. */
+@media print, projection {
+  .str {
+    color: #060; }
+
+  .kwd {
+    color: #006;
+    font-weight: bold; }
+
+  .com {
+    color: #600;
+    font-style: italic; }
+
+  .typ {
+    color: #404;
+    font-weight: bold; }
+
+  .lit {
+    color: #044; }
+
+  .pun, .opn, .clo {
+    color: #440; }
+
+  .tag {
+    color: #006;
+    font-weight: bold; }
+
+  .atn {
+    color: #404; }
+
+  .atv {
+    color: #060; } }
+/* Style */
+/*
+pre.prettyprint {
+  background: white;
+  font-family: Consolas, Monaco, 'Andale Mono', monospace;
+  font-size: 12px;
+  line-height: 1.5;
+  border: 1px solid #ccc;
+  padding: 10px; }
+*/
+
+/* Specify class=linenums on a pre to get line numbering */
+ol.linenums {
+  margin-top: 0;
+  margin-bottom: 0; }
+
+/* IE indents via margin-left */
+li.L0,
+li.L1,
+li.L2,
+li.L3,
+li.L4,
+li.L5,
+li.L6,
+li.L7,
+li.L8,
+li.L9 {
+  /* */ }
+
+/* Alternate shading for lines */
+li.L1,
+li.L3,
+li.L5,
+li.L7,
+li.L9 {
+  /* */ }

package/index.js

@@ -1,3 +1,165 @@
 const { VectorStore } = require('node-gyp-build')(__dirname);
 
+/**
+ * @typedef {Object} Document
+ * @property {string} id - Unique identifier for the document
+ * @property {string} text - The text content of the document
+ * @property {Object} metadata - Document metadata
+ * @property {number[]} metadata.embedding - The embedding vector for the document
+ * @property {*} [metadata.*] - Additional metadata properties
+ */
+
+/**
+ * @typedef {Object} SearchResult
+ * @property {number} score - Similarity score (0-1, higher is more similar)
+ * @property {string} id - Document identifier
+ * @property {string} text - Document text content
+ * @property {string} metadata_json - Serialized metadata as JSON string
+ */
+
+/**
+ * High-performance vector store with SIMD optimization for similarity search.
+ * Designed for immutable, one-time loading scenarios with fast searches over focused corpora.
+ * 
+ * @class VectorStore
+ * @example
+ * // Basic usage
+ * const store = new VectorStore(1536);
+ * store.loadDir('./documents');
+ * const results = store.search(queryEmbedding, 10);
+ * 
+ * @example
+ * // Multiple domain-specific stores
+ * const productStore = new VectorStore(1536);
+ * const supportStore = new VectorStore(1536);
+ * productStore.loadDir('./knowledge/products');
+ * supportStore.loadDir('./knowledge/support');
+ */
+class VectorStoreWrapper {
+  /**
+   * Creates a new VectorStore instance
+   * @param {number} dimensions - The dimensionality of embedding vectors (e.g., 1536 for OpenAI embeddings)
+   * @throws {TypeError} If dimensions is not a positive integer
+   */
+  constructor(dimensions) {
+    return new VectorStore(dimensions);
+  }
+
+  /**
+   * Load all JSON documents from a directory and automatically finalize the store.
+   * Documents should contain embedding vectors in their metadata field.
+   * Supports both single documents and arrays of documents per file.
+   * 
+   * @param {string} path - Absolute or relative path to directory containing JSON files
+   * @returns {void}
+   * @throws {Error} If directory doesn't exist or contains invalid JSON
+   * 
+   * @example
+   * // Load documents from a directory
+   * store.loadDir('./knowledge-base');
+   * // Store is automatically finalized and ready for searches
+   * 
+   * @example
+   * // Document format in JSON files
+   * {
+   *   "id": "doc-123",
+   *   "text": "Document content...",
+   *   "metadata": {
+   *     "embedding": [0.1, 0.2, ...],  // Required: embedding vector
+   *     "category": "product"           // Optional: additional metadata
+   *   }
+   * }
+   */
+  loadDir(path) {}
+
+  /**
+   * Add a single document to the store. Only works before finalization.
+   * 
+   * @param {Document} doc - Document object with embedding in metadata
+   * @returns {void}
+   * @throws {Error} If called after finalization or document format is invalid
+   * 
+   * @example
+   * store.addDocument({
+   *   id: 'doc-1',
+   *   text: 'Sample document',
+   *   metadata: {
+   *     embedding: new Array(1536).fill(0).map(() => Math.random())
+   *   }
+   * });
+   */
+  addDocument(doc) {}
+
+  /**
+   * Search for the k most similar documents to a query embedding.
+   * Uses SIMD-optimized cosine similarity for fast performance.
+   * 
+   * @param {Float32Array} query - Query embedding vector (must match store dimensions)
+   * @param {number} k - Number of results to return (top-k nearest neighbors)
+   * @param {boolean} [normalizeQuery=true] - Whether to L2-normalize the query vector
+   * @returns {SearchResult[]} Array of search results sorted by similarity (highest first)
+   * @throws {Error} If store is not finalized or query dimensions don't match
+   * 
+   * @example
+   * // Basic search
+   * const queryEmbedding = new Float32Array(1536);
+   * const results = store.search(queryEmbedding, 10);
+   * 
+   * @example
+   * // Search with pre-normalized query
+   * const normalized = normalizeVector(queryEmbedding);
+   * const results = store.search(normalized, 5, false);
+   * 
+   * @example
+   * // Filter results by score threshold
+   * const results = store.search(queryEmbedding, 20)
+   *   .filter(r => r.score > 0.7);
+   */
+  search(query, k, normalizeQuery = true) {}
+
+  /**
+   * Finalize the store: normalize all embeddings and switch to serving mode.
+   * After calling this, no more documents can be added but searches become available.
+   * This is automatically called by loadDir().
+   * 
+   * @returns {void}
+   * 
+   * @example
+   * // Manual finalization after adding documents
+   * store.addDocument(doc1);
+   * store.addDocument(doc2);
+   * store.finalize(); // Must call before searching
+   */
+  finalize() {}
+
+  /**
+   * Check if the store has been finalized and is ready for searching.
+   * 
+   * @returns {boolean} True if finalized, false otherwise
+   * 
+   * @example
+   * if (store.isFinalized()) {
+   *   const results = store.search(query, 10);
+   * }
+   */
+  isFinalized() {}
+
+  /**
+   * Get the number of documents in the store.
+   * 
+   * @returns {number} Number of documents loaded
+   * 
+   * @example
+   * console.log(`Loaded ${store.size()} documents`);
+   */
+  size() {}
+
+  /**
+   * @deprecated Use finalize() instead
+   * @returns {void}
+   */
+  normalize() {}
+}
+
+// Re-export the native VectorStore as-is, but the JSDoc above provides documentation
 module.exports = { VectorStore };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "native-vector-store",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "High-performance local vector store with SIMD optimization for MCP servers",
   "main": "index.js",
   "types": "lib/index.d.ts",
@@ -23,7 +23,9 @@
     "prebuildify-cross": "prebuildify-cross -i centos7-devtoolset7 -i alpine -i linux-arm64 -i darwin-arm64 -i darwin-x64+arm64",
     "benchmark": "node test/test.js",
     "example": "node examples/mcp-server.js demo",
-    "embed": "node embed_dir.js"
+    "embed": "node embed_dir.js",
+    "docs": "jsdoc -c jsdoc.json",
+    "prepublishOnly": "npm run docs"
   },
   "keywords": [
     "vector",
@@ -36,17 +38,38 @@
   "dependencies": {
     "dotenv": "^17.2.0",
     "node-addon-api": "^7.0.0",
-    "node-gyp-build": "^4.8.0",
+    "node-gyp-build": "^4.8.4",
     "openai": "^5.8.3"
   },
+  "optionalDependencies": {
+    "@modelcontextprotocol/sdk": "^1.15.1"
+  },
   "devDependencies": {
+    "jsdoc": "^4.0.4",
     "node-gyp": "^10.0.0",
-    "prebuildify": "^6.0.0"
+    "prebuildify": "^6.0.0",
+    "prebuildify-cross": "^5.0.0"
   },
-  "os": ["darwin", "linux", "win32"],
-  "cpu": ["x64", "arm64"],
+  "os": [
+    "darwin",
+    "linux",
+    "win32"
+  ],
+  "cpu": [
+    "x64",
+    "arm64"
+  ],
   "engines": {
     "node": ">=14.0.0"
   },
-  "gypfile": true
+  "gypfile": true,
+  "files": [
+    "index.js",
+    "lib/",
+    "src/",
+    "deps/",
+    "binding.gyp",
+    "prebuilds/",
+    "docs/"
+  ]
 }

package/src/Makefile

@@ -0,0 +1,87 @@
+# Detect OS
+UNAME_S := $(shell uname -s)
+
+# Compiler
+ifeq ($(UNAME_S),Darwin)
+    CXX = clang++
+else
+    CXX = g++
+endif
+
+# Base flags
+CXXFLAGS = -std=c++17 -g -O0 -fno-omit-frame-pointer -DDEBUG
+
+# OS-specific configuration
+ifeq ($(UNAME_S),Darwin)
+    # macOS configuration
+    INCLUDES = -I. -I../deps/simdjson -I../deps/atomic_queue -I/opt/homebrew/opt/libomp/include
+    LDFLAGS = -L/opt/homebrew/opt/libomp/lib
+    LIBS = -lomp
+    # Add flags for better debugging with lldb
+    CXXFLAGS += -glldb -gdwarf-4
+    # OpenMP flags for macOS
+    CXXFLAGS += -Xpreprocessor -fopenmp
+else
+    # Linux configuration
+    INCLUDES = -I. -I../deps/simdjson -I../deps/atomic_queue
+    LDFLAGS = -L/usr/lib
+    LIBS = -lgomp
+    # OpenMP flags for Linux
+    CXXFLAGS += -fopenmp
+endif
+
+TARGET = test_vector_store
+STRESS_TARGET = test_stress
+SOURCES = test_main.cpp vector_store.cpp ../deps/simdjson/simdjson.cpp
+STRESS_SOURCES = test_stress.cpp vector_store.cpp vector_store_loader.cpp vector_store_loader_mmap.cpp vector_store_loader_adaptive.cpp ../deps/simdjson/simdjson.cpp
+OBJECTS = $(SOURCES:.cpp=.o)
+STRESS_OBJECTS = $(STRESS_SOURCES:.cpp=.o)
+
+all: $(TARGET) $(STRESS_TARGET)
+
+$(TARGET): $(OBJECTS)
+	$(CXX) $(CXXFLAGS) $(OBJECTS) -o $(TARGET) $(LDFLAGS) $(LIBS)
+
+$(STRESS_TARGET): $(STRESS_OBJECTS)
+	$(CXX) $(CXXFLAGS) $(STRESS_OBJECTS) -o $(STRESS_TARGET) $(LDFLAGS) $(LIBS)
+
+%.o: %.cpp
+	$(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@
+
+clean:
+	rm -f $(OBJECTS) $(STRESS_OBJECTS) $(TARGET) $(STRESS_TARGET)
+
+# Force rebuild (useful for CI)
+rebuild: clean all
+
+run: $(TARGET)
+	./$(TARGET) ../test
+
+debug: $(TARGET)
+	lldb ./$(TARGET)
+
+# Sanitizer builds
+test-asan: CXXFLAGS += -fsanitize=address -fno-sanitize-recover=all
+test-asan: LDFLAGS += -fsanitize=address
+test-asan: clean $(STRESS_TARGET)
+	./$(STRESS_TARGET)
+
+test-tsan: CXXFLAGS += -fsanitize=thread -fno-sanitize-recover=all
+test-tsan: LDFLAGS += -fsanitize=thread
+test-tsan: clean $(STRESS_TARGET)
+	./$(STRESS_TARGET)
+
+# Run stress tests with sanitizers by default
+# Use SANITIZER=none to disable, SANITIZER=thread for TSan
+SANITIZER ?= address
+
+ifeq ($(SANITIZER),address)
+stress: test-asan
+else ifeq ($(SANITIZER),thread)
+stress: test-tsan
+else
+stress: $(STRESS_TARGET)
+	./$(STRESS_TARGET)
+endif
+
+.PHONY: all clean run debug test-asan test-tsan stress