gitnexus 1.6.9-rc.26 → 1.6.9-rc.27

package/README.md

@@ -363,6 +363,7 @@ Configure the behavior with two environment variables:
 | -------------------------------------------- | ---------------------------- | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `GITNEXUS_LBUG_EXTENSION_INSTALL`            | `auto`, `load-only`, `never` | `auto`              | `auto` runs one bounded INSTALL if LOAD fails. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely.                                                                                       |
 | `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer             | `15000`             | Wall-clock budget for the out-of-process `INSTALL` child before it is killed.                                                                                                                                                                                                            |
+| `GITNEXUS_FTS_STEMMER`                       | supported LadybugDB stemmer  | `porter`            | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it.              |
 | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD`          | integer `>= -1`              | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. |
 
 ```bash
@@ -371,6 +372,9 @@ GITNEXUS_LBUG_EXTENSION_INSTALL=load-only npx gitnexus analyze
 
 # Slow network: give extension downloads more time
 GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS=30000 npx gitnexus analyze
+
+# CJK-heavy codebase: rebuild keyword indexes without English stemming
+GITNEXUS_FTS_STEMMER=none npx gitnexus analyze --repair-fts
 ```
 
 ### Analysis runs out of memory

package/dist/core/lbug/lbug-adapter.d.ts

@@ -286,6 +286,12 @@ export declare const loadFTSExtension: (targetConn?: lbug.Connection, opts?: Ext
  * unavailable so semantic search can fall back to exact scan.
  */
 export declare const loadVectorExtension: (targetConn?: lbug.Connection, opts?: ExtensionEnsureOptions) => Promise<boolean>;
+/**
+ * Default stemmer for FTS indexes. Single source so the analyze path
+ * (`getSearchFTSStemmer`) and the read-only `createFTSIndex`/`ensureFTSIndex`
+ * defaults can never silently diverge.
+ */
+export declare const DEFAULT_FTS_STEMMER = "porter";
 /**
  * Create a full-text search index on a table
  * @param tableName - The node table name (e.g., 'File', 'CodeSymbol')

package/dist/core/lbug/lbug-adapter.js

@@ -2081,6 +2081,12 @@ export const loadVectorExtension = async (targetConn, opts = {}) => {
         vectorExtensionLoaded = true;
     return loaded;
 };
+/**
+ * Default stemmer for FTS indexes. Single source so the analyze path
+ * (`getSearchFTSStemmer`) and the read-only `createFTSIndex`/`ensureFTSIndex`
+ * defaults can never silently diverge.
+ */
+export const DEFAULT_FTS_STEMMER = 'porter';
 /**
  * Create a full-text search index on a table
  * @param tableName - The node table name (e.g., 'File', 'CodeSymbol')
@@ -2088,7 +2094,7 @@ export const loadVectorExtension = async (targetConn, opts = {}) => {
  * @param properties - List of properties to index (e.g., ['name', 'code'])
  * @param stemmer - Stemming algorithm (default: 'porter')
  */
-export const createFTSIndex = async (tableName, indexName, properties, stemmer = 'porter') => {
+export const createFTSIndex = async (tableName, indexName, properties, stemmer = DEFAULT_FTS_STEMMER) => {
     if (!conn) {
         throw new Error('LadybugDB not initialized. Call initLbug first.');
     }
@@ -2176,7 +2182,7 @@ export const createVectorIndex = async () => {
  * the index is owned by `gitnexus analyze` (writable) and either already
  * exists or will be created on the next analyze.
  */
-export const ensureFTSIndex = async (tableName, indexName, properties, stemmer = 'porter') => {
+export const ensureFTSIndex = async (tableName, indexName, properties, stemmer = DEFAULT_FTS_STEMMER) => {
     const key = ftsIndexKey(tableName, indexName);
     if (ensuredFTSIndexes.has(key))
         return;

package/dist/core/run-analyze.js

@@ -14,7 +14,7 @@ import { execFileSync } from 'child_process';
 import { runPipelineFromRepo } from './ingestion/pipeline.js';
 import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js';
 import { initLbug, loadGraphToLbug, getLbugStats, executeQuery, executeWithReusedStatement, closeLbug, closeLbugBeforeExit, loadCachedEmbeddings, deleteNodesForFile, deleteAllCommunitiesAndProcesses, deleteAllInterprocTaintPaths, deleteAllCallSummaries, queryImporters, loadFTSExtension, } from './lbug/lbug-adapter.js';
-import { createSearchFTSIndexes, verifySearchFTSIndexes } from './search/fts-indexes.js';
+import { createSearchFTSIndexes, initialiseSearchFTSStemmer, verifySearchFTSIndexes, } from './search/fts-indexes.js';
 import { resolveAnalyzeInstallPolicy } from './lbug/extension-loader.js';
 import { startWalCheckpointDriver, } from './lbug/wal-checkpoint-driver.js';
 import { getStoragePaths, resolveBranchPlacement, saveMeta, loadMeta, ensureGitNexusIgnored, registerRepo, isRepoRegistered, cleanupOldKuzuFiles, INCREMENTAL_SCHEMA_VERSION, } from '../storage/repo-manager.js';
@@ -255,6 +255,10 @@ export const pdgModeMismatch = (recorded, options) => {
 export async function runFullAnalysis(repoPath, options, callbacks) {
     const log = (msg) => callbacks.onLog?.(msg);
     const progress = (phase, percent, message) => callbacks.onProgress(phase, percent, message);
+    // Resolve + validate operator-provided FTS config once, before the expensive
+    // parse/load phases. A typo fails here in ms; createSearchFTSIndexes reuses
+    // the cached value via getSearchFTSStemmer.
+    initialiseSearchFTSStemmer();
     // Scope the degraded-parse log throttle to this run. On a reused process
     // (e.g. tests, or any host that calls runFullAnalysis more than once) the
     // module-level counter would otherwise stay saturated and suppress every

package/dist/core/search/fts-indexes.d.ts

@@ -2,5 +2,19 @@ export interface CreateSearchFTSIndexesOptions {
     onIndexStart?: (table: string, indexName: string) => void;
     onIndexReady?: (table: string, indexName: string) => void;
 }
+/**
+ * Resolve + validate `GITNEXUS_FTS_STEMMER` once, up front at analyze startup,
+ * and cache it. An invalid value throws here — in milliseconds — instead of
+ * ~85% into a run (after the expensive parse/scope-resolution work). The cached
+ * value is what {@link getSearchFTSStemmer} returns for the rest of the run, so
+ * config is read and validated in exactly one place.
+ */
+export declare function initialiseSearchFTSStemmer(): string;
+/**
+ * Return the stemmer resolved by {@link initialiseSearchFTSStemmer}. Falls back
+ * to resolving on demand when init was never called (read-only hosts, unit
+ * tests) so validation always applies.
+ */
+export declare function getSearchFTSStemmer(): string;
 export declare function createSearchFTSIndexes(options?: CreateSearchFTSIndexesOptions): Promise<void>;
 export declare function verifySearchFTSIndexes(executeQuery: (cypher: string) => Promise<unknown[]>): Promise<string[]>;

package/dist/core/search/fts-indexes.js

@@ -1,6 +1,71 @@
-import { createFTSIndex, dropFTSIndex } from '../lbug/lbug-adapter.js';
+import { createFTSIndex, dropFTSIndex, DEFAULT_FTS_STEMMER } from '../lbug/lbug-adapter.js';
 import { FTS_INDEXES } from './fts-schema.js';
+// Stemmers shipped by the LadybugDB FTS extension. Mirrors the lowercase token
+// set in the extension bundled with @ladybugdb/core 0.17.x (see package.json).
+// Keep in sync on a LadybugDB minor bump — a value here that the installed
+// extension rejects would pass validation but fail at CREATE_FTS_INDEX.
+const SUPPORTED_FTS_STEMMERS = new Set([
+    'arabic',
+    'basque',
+    'catalan',
+    'danish',
+    'dutch',
+    'english',
+    'finnish',
+    'french',
+    'german',
+    'greek',
+    'hindi',
+    'hungarian',
+    'indonesian',
+    'irish',
+    'italian',
+    'lithuanian',
+    'nepali',
+    'norwegian',
+    'none',
+    'porter',
+    'portuguese',
+    'romanian',
+    'russian',
+    'serbian',
+    'spanish',
+    'swedish',
+    'tamil',
+    'turkish',
+]);
+let resolvedStemmer;
+/** Read + validate `GITNEXUS_FTS_STEMMER`. Throws on an unsupported value. */
+function resolveFTSStemmer() {
+    const raw = process.env.GITNEXUS_FTS_STEMMER?.trim().toLowerCase();
+    if (!raw)
+        return DEFAULT_FTS_STEMMER;
+    if (SUPPORTED_FTS_STEMMERS.has(raw))
+        return raw;
+    throw new Error(`Invalid GITNEXUS_FTS_STEMMER "${process.env.GITNEXUS_FTS_STEMMER}". ` +
+        `Expected one of: ${[...SUPPORTED_FTS_STEMMERS].sort().join(', ')}.`);
+}
+/**
+ * Resolve + validate `GITNEXUS_FTS_STEMMER` once, up front at analyze startup,
+ * and cache it. An invalid value throws here — in milliseconds — instead of
+ * ~85% into a run (after the expensive parse/scope-resolution work). The cached
+ * value is what {@link getSearchFTSStemmer} returns for the rest of the run, so
+ * config is read and validated in exactly one place.
+ */
+export function initialiseSearchFTSStemmer() {
+    resolvedStemmer = resolveFTSStemmer();
+    return resolvedStemmer;
+}
+/**
+ * Return the stemmer resolved by {@link initialiseSearchFTSStemmer}. Falls back
+ * to resolving on demand when init was never called (read-only hosts, unit
+ * tests) so validation always applies.
+ */
+export function getSearchFTSStemmer() {
+    return resolvedStemmer ?? resolveFTSStemmer();
+}
 export async function createSearchFTSIndexes(options) {
+    const stemmer = getSearchFTSStemmer();
     for (const { table, indexName, properties } of FTS_INDEXES) {
         options?.onIndexStart?.(table, indexName);
         // Drop first so the live `properties` always win. `createFTSIndex` is
@@ -15,7 +80,7 @@ export async function createSearchFTSIndexes(options) {
         // runs inside the existing FTS phase. Gate on a stored schema fingerprint if
         // this rebuild cost ever shows up in analyze profiles.
         await dropFTSIndex(table, indexName);
-        await createFTSIndex(table, indexName, [...properties]);
+        await createFTSIndex(table, indexName, [...properties], stemmer);
         options?.onIndexReady?.(table, indexName);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.9-rc.26",
+  "version": "1.6.9-rc.27",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",