@joycodetech/qmd-ja 2.5.3 → 2.5.5

package/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## [Unreleased]
 
+## [2.5.4] - 2026-06-22
+
 ### Documentation
 
 - README: documented collection filtering (`-c` semantics), the `collection

package/dist/cli/qmd.js

@@ -1,4 +1,3 @@
-#!/usr/bin/env node
 import { isBun, openDatabase } from "../db.js";
 import fastGlob from "fast-glob";
 import { execSync, spawn as nodeSpawn } from "child_process";
@@ -7,7 +6,7 @@ import { basename, dirname, join as pathJoin, relative as relativePath, resolve
 import { parseArgs } from "util";
 import { readFileSync, readdirSync, realpathSync, statSync, existsSync, unlinkSync, writeFileSync, openSync, closeSync, mkdirSync, lstatSync, rmSync, symlinkSync, readlinkSync, copyFileSync } from "fs";
 import { createInterface } from "readline/promises";
-import { getPwd, getRealPath, homedir, resolve, enableProductionMode, searchFTS, extractSnippet, getContextForFile, getContextForPath, listCollections, removeCollection, renameCollection, findSimilarFiles, findDocumentByDocid, isDocid, matchFilesByGlob, getHashesNeedingEmbedding, clearAllEmbeddings, insertEmbedding, getStatus, hashContent, extractTitle, formatDocForEmbedding, getEmbeddingFingerprint, chunkDocumentByTokens, clearCache, getCacheKey, getCachedResult, setCachedResult, getIndexHealth, parseVirtualPath, buildVirtualPath, isVirtualPath, resolveVirtualPath, toVirtualPath, insertContent, insertDocument, findActiveDocument, findOrMigrateLegacyDocument, updateDocumentTitle, updateDocument, deactivateDocument, getActiveDocumentPaths, cleanupOrphanedContent, deleteLLMCache, deleteInactiveDocuments, cleanupOrphanedVectors, vacuumDatabase, getCollectionsWithoutContext, getTopLevelPathsWithoutContext, handelize, hybridQuery, vectorSearchQuery, structuredSearch, addLineNumbers, DEFAULT_EMBED_MODEL, DEFAULT_EMBED_MAX_BATCH_BYTES, DEFAULT_EMBED_MAX_DOCS_PER_BATCH, DEFAULT_RERANK_MODEL, DEFAULT_QUERY_MODEL, DEFAULT_GLOB, DEFAULT_MULTI_GET_MAX_BYTES, createStore, getDefaultDbPath, reindexCollection, initializeKuromojiTokenizer, generateEmbeddings, maybeAdoptLegacyEmbeddingFingerprint, syncConfigToDb, } from "../store.js";
+import { getPwd, getRealPath, homedir, resolve, enableProductionMode, searchFTS, extractSnippet, getContextForFile, getContextForPath, listCollections, removeCollection, renameCollection, findSimilarFiles, findDocumentByDocid, isDocid, matchFilesByGlob, getHashesNeedingEmbedding, clearAllEmbeddings, insertEmbedding, getStatus, hashContent, extractTitle, formatDocForEmbedding, getEmbeddingFingerprint, chunkDocumentByTokens, clearCache, getCacheKey, getCachedResult, setCachedResult, getIndexHealth, parseVirtualPath, buildVirtualPath, isVirtualPath, resolveVirtualPath, toVirtualPath, insertContent, insertDocument, findActiveDocument, findOrMigrateLegacyDocument, updateDocumentTitle, updateDocument, deactivateDocument, getActiveDocumentPaths, cleanupOrphanedContent, deleteLLMCache, deleteInactiveDocuments, cleanupOrphanedVectors, vacuumDatabase, getCollectionsWithoutContext, getTopLevelPathsWithoutContext, handelize, hybridQuery, vectorSearchQuery, structuredSearch, addLineNumbers, DEFAULT_EMBED_MODEL, DEFAULT_EMBED_MAX_BATCH_BYTES, DEFAULT_EMBED_MAX_DOCS_PER_BATCH, DEFAULT_RERANK_MODEL, DEFAULT_QUERY_MODEL, DEFAULT_GLOB, DEFAULT_MULTI_GET_MAX_BYTES, createStore, getDefaultDbPath, reindexCollection, initializeKuromojiTokenizer, initializeVaporettoTokenizer, FTS_CJK_NORMALIZED_VERSION, resolveVaporettoModelPath, generateEmbeddings, maybeAdoptLegacyEmbeddingFingerprint, syncConfigToDb, } from "../store.js";
 import { disposeDefaultLlamaCpp, getDefaultLlamaCpp, setDefaultLlamaCpp, LlamaCpp, withLLMSession, pullModels, DEFAULT_MODEL_CACHE_DIR, resolveEmbedModel, resolveGenerateModel, resolveRerankModel, resolveModels, inspectGgufFile, isDarwinMetalMitigationActive } from "../llm.js";
 import { formatSearchResults, formatDocuments, escapeXml, escapeCSV, } from "./formatter.js";
 import { getCollection as getCollectionFromYaml, listCollections as yamlListCollections, getDefaultCollectionNames, addContext as yamlAddContext, removeContext as yamlRemoveContext, removeCollection as yamlRemoveCollectionFn, renameCollection as yamlRenameCollectionFn, setGlobalContext, listAllContexts, setConfigIndexName, loadConfig, saveConfig, setConfigSource, findLocalConfigPath, getLocalDbPath, getConfigPath, configExists, } from "../collections.js";
@@ -3490,6 +3489,27 @@ async function showDoctor() {
     catch (error) {
         doctorCheck("sqlite-vec", false, error instanceof Error ? error.message : String(error));
     }
+    // CJK tokenizer check (qmd-ja: Vaporetto WASM)
+    try {
+        await initializeVaporettoTokenizer();
+        const modelPath = resolveVaporettoModelPath();
+        const modelName = modelPath.split("/").pop() ?? modelPath;
+        let ftsLabel = "not indexed yet";
+        try {
+            const row = db.prepare(`SELECT value FROM store_config WHERE key = 'fts_cjk_normalized_version'`).get();
+            if (row?.value === FTS_CJK_NORMALIZED_VERSION) {
+                ftsLabel = `v${FTS_CJK_NORMALIZED_VERSION} (current)`;
+            }
+            else if (row?.value) {
+                ftsLabel = `v${row.value} -> v${FTS_CJK_NORMALIZED_VERSION} (stale, run qmd-ja update)`;
+            }
+        }
+        catch { /* ignore */ }
+        doctorCheck("CJK tokenizer", true, `Vaporetto WASM — model: ${modelName}, FTS index: ${ftsLabel}`);
+    }
+    catch (error) {
+        doctorCheck("CJK tokenizer", false, `Vaporetto WASM failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
     const configCheck = checkDoctorIndexConfig(nextSteps);
     const configModels = configCheck.config?.models ?? {};
     checkEnvironmentOverrides(activeModels, configModels);
@@ -3600,7 +3620,7 @@ async function showVersion() {
         // Not a git repo or git not available
     }
     const versionStr = commit ? `${pkg.version} (${commit})` : pkg.version;
-    console.log(`qmd ${versionStr}`);
+    console.log(`qmd-ja ${versionStr}`);
 }
 // Main CLI - only run if this is the main module
 const __filename = fileURLToPath(import.meta.url);

package/dist/store.d.ts

@@ -187,6 +187,12 @@ export declare function resolveVirtualPath(db: Database, virtualPath: string): s
  */
 export declare function toVirtualPath(db: Database, absolutePath: string): string | null;
 export declare function verifySqliteVecLoaded(db: Database): void;
+export declare const FTS_CJK_NORMALIZED_VERSION = "3";
+/**
+ * Resolve the Vaporetto model file path relative to this module.
+ * The model is bundled under vendor/vaporetto-node-wasm/../models/ relative to the project root.
+ */
+export declare function resolveVaporettoModelPath(): string;
 /**
  * Pre-initialize the Vaporetto WASM Japanese morphological analyzer.
  * Call this before indexing or search operations involving CJK text.

package/dist/store.js

@@ -595,7 +595,7 @@ const CJK_CHAR_PATTERN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\
 // parts of katakana words (e.g. "ナレッジベース").  Without these, the regex splits
 // on "ー" and vaporetto receives broken sub-strings like "ナレッジベ" and "ス".
 const CJK_RUN_PATTERN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}・ー]+/gu;
-const FTS_CJK_NORMALIZED_VERSION = "3"; // bumped: vaporetto WASM morphological tokenization
+export const FTS_CJK_NORMALIZED_VERSION = "3"; // bumped: vaporetto WASM morphological tokenization
 // --- Vaporetto WASM Japanese morphological analyzer (lazy singleton) ---
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 let _vaporettoTokenizer = null;
@@ -604,7 +604,7 @@ let _vaporettoInitPromise = null;
  * Resolve the Vaporetto model file path relative to this module.
  * The model is bundled under vendor/vaporetto-node-wasm/../models/ relative to the project root.
  */
-function _resolveVaporettoModelPath() {
+export function resolveVaporettoModelPath() {
     // __dirname equivalent for ESM
     const thisDir = dirname(fileURLToPath(import.meta.url));
     // src/ → project root → models/
@@ -626,8 +626,8 @@ export async function initializeVaporettoTokenizer() {
         _vaporettoInitPromise = (async () => {
             const req = createRequire(import.meta.url);
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            const { VaporettoTokenizer } = req(pathJoin(dirname(fileURLToPath(import.meta.url)), "_vendor", "vaporetto-node-wasm", "vaporetto_node_wasm.js"));
-            const modelPath = _resolveVaporettoModelPath();
+            const { VaporettoTokenizer } = req(pathJoin(dirname(fileURLToPath(import.meta.url)), "..", "vendor", "vaporetto-node-wasm", "vaporetto_node_wasm.js"));
+            const modelPath = resolveVaporettoModelPath();
             const modelData = readFileSync(modelPath);
             return new VaporettoTokenizer(modelData);
         })();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@joycodetech/qmd-ja",
-  "version": "2.5.3",
+  "version": "2.5.5",
   "description": "Japanese-enhanced fork of qmd — On-device hybrid search with Vaporetto WASM morphological tokenizer for accurate Japanese BM25 full-text search",
   "type": "module",
   "main": "dist/index.js",
@@ -12,7 +12,7 @@
     }
   },
   "bin": {
-    "qmd": "bin/qmd"
+    "qmd-ja": "bin/qmd-ja"
   },
   "files": [
     "bin/",

package/vendor/vaporetto-node-wasm/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) Koichi Akabe, Shunsuke Kanda, Yusuke Oda, Shinsuke Mori
+(daac-tools/vaporetto — https://github.com/daac-tools/vaporetto)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.