@jcode.labs/ragmir 2.7.0 → 2.9.0

package/dist/security.js

@@ -1,23 +1,34 @@
+import { execFile } from "node:child_process";
 import { existsSync } from "node:fs";
-import { readFile } from "node:fs/promises";
+import { readFile, stat } from "node:fs/promises";
 import path from "node:path";
-import { loadConfig } from "./config.js";
-import { LEGACY_KB_DIR, LEGACY_KB_GITIGNORE_ENTRY, LEGACY_PRIVATE_DIR, LEGACY_PRIVATE_GITIGNORE_ENTRY, LEGACY_PRIVATE_GITIGNORE_FALLBACK_ENTRY, RAGMIR_GITIGNORE_ENTRY, } from "./defaults.js";
+import { findProjectConfig, loadConfig } from "./config.js";
+import { LEGACY_KB_DIR, LEGACY_KB_GITIGNORE_ENTRY, LEGACY_PRIVATE_DIR, LEGACY_PRIVATE_GITIGNORE_ENTRY, RAGMIR_GITIGNORE_ENTRY, } from "./defaults.js";
 export async function securityAudit(cwd = process.cwd()) {
     const config = await loadConfig(cwd);
     const gitignore = await readGitignore(config.projectRoot);
     const warnings = [];
-    const legacyKbIgnored = hasGitignoreEntry(gitignore, LEGACY_KB_GITIGNORE_ENTRY);
-    const ragmirIgnored = hasGitignoreEntry(gitignore, RAGMIR_GITIGNORE_ENTRY);
-    const legacyPrivateIgnored = hasGitignoreEntry(gitignore, LEGACY_PRIVATE_GITIGNORE_ENTRY) ||
-        hasGitignoreEntry(gitignore, LEGACY_PRIVATE_GITIGNORE_FALLBACK_ENTRY);
     const usesLegacyKb = [config.storageDir, config.sourcesFile, config.accessLogPath].some((filePath) => usesProjectDirectory(config.projectRoot, filePath, LEGACY_KB_DIR));
     const usesLegacyPrivate = usesProjectDirectory(config.projectRoot, config.rawDir, LEGACY_PRIVATE_DIR);
-    const storageGitIgnored = isPathIgnored(config.projectRoot, config.storageDir, gitignore);
-    if (config.embeddingProvider === "transformers" && config.transformersAllowRemoteModels) {
+    const [legacyKbIgnored, ragmirIgnored, legacyPrivateIgnored, storageGitIgnored, accessLogGitIgnored,] = await Promise.all([
+        isPathIgnored(config.projectRoot, path.join(config.projectRoot, LEGACY_KB_GITIGNORE_ENTRY), gitignore),
+        isPathIgnored(config.projectRoot, path.join(config.projectRoot, RAGMIR_GITIGNORE_ENTRY), gitignore),
+        isPathIgnored(config.projectRoot, path.join(config.projectRoot, LEGACY_PRIVATE_GITIGNORE_ENTRY), gitignore),
+        isPathIgnored(config.projectRoot, config.storageDir, gitignore),
+        isPathIgnored(config.projectRoot, config.accessLogPath, gitignore),
+    ]);
+    const permissions = await inspectPermissions({
+        configPath: findProjectConfig(cwd).configPath,
+        rawDir: config.rawDir,
+        storageDir: config.storageDir,
+        accessLogPath: config.accessLogPath,
+    });
+    if (config.privacyProfile !== "trusted" &&
+        config.embeddingProvider === "transformers" &&
+        config.transformersAllowRemoteModels) {
         warnings.push("Transformers remote model loading is enabled; model files can be downloaded from Hugging Face.");
     }
-    if (!config.redaction.enabled) {
+    if (config.privacyProfile !== "trusted" && !config.redaction.enabled) {
         warnings.push("Redaction is disabled; secrets and identifiers may be embedded in the index.");
     }
     if (!ragmirIgnored) {
@@ -29,12 +40,27 @@ export async function securityAudit(cwd = process.cwd()) {
     if (usesLegacyPrivate && !legacyPrivateIgnored) {
         warnings.push(`${LEGACY_PRIVATE_GITIGNORE_ENTRY} is not ignored by Git.`);
     }
+    if (!storageGitIgnored) {
+        warnings.push("The configured storageDir is not ignored by Git.");
+    }
+    if (config.accessLog && !accessLogGitIgnored) {
+        warnings.push("The configured accessLogPath is not ignored by Git.");
+    }
+    addPermissionWarning(warnings, permissions.configPrivate, "Ragmir config file");
+    addPermissionWarning(warnings, permissions.rawDirPrivate, "raw document directory");
+    addPermissionWarning(warnings, permissions.storageDirPrivate, "index storage directory");
+    if (config.accessLog) {
+        addPermissionWarning(warnings, permissions.accessLogPrivate, "access log");
+    }
     return {
         projectRoot: config.projectRoot,
         zeroTelemetry: true,
+        privacyProfile: config.privacyProfile,
+        retrievalProfile: config.retrievalProfile,
         providers: {
             embedding: config.embeddingProvider,
             embeddingModel: config.embeddingModel,
+            embeddingModelRevision: config.embeddingModelRevision,
             embeddingModelPath: config.embeddingModelPath,
             transformersAllowRemoteModels: config.transformersAllowRemoteModels,
             llmGeneration: false,
@@ -54,6 +80,7 @@ export async function securityAudit(cwd = process.cwd()) {
             gitIgnored: storageGitIgnored,
             encryptedAtRest: "external-required",
         },
+        permissions,
         mcp: {
             maxTopK: config.mcpMaxTopK,
             destructiveToolsExposed: false,
@@ -72,6 +99,50 @@ export async function securityAudit(cwd = process.cwd()) {
         warnings,
     };
 }
+async function inspectPermissions(paths) {
+    if (process.platform === "win32") {
+        return {
+            checked: false,
+            configPrivate: null,
+            rawDirPrivate: null,
+            storageDirPrivate: null,
+            accessLogPrivate: null,
+        };
+    }
+    const [configPrivate, rawDirPrivate, storageDirPrivate, accessLogPrivate] = await Promise.all([
+        isPrivatePath(paths.configPath),
+        isPrivatePath(paths.rawDir),
+        isPrivatePath(paths.storageDir),
+        isPrivatePath(paths.accessLogPath),
+    ]);
+    return {
+        checked: true,
+        configPrivate,
+        rawDirPrivate,
+        storageDirPrivate,
+        accessLogPrivate,
+    };
+}
+async function isPrivatePath(filePath) {
+    try {
+        const mode = (await stat(filePath)).mode & 0o777;
+        return (mode & 0o077) === 0;
+    }
+    catch (error) {
+        if (isNodeError(error) && error.code === "ENOENT") {
+            return null;
+        }
+        throw error;
+    }
+}
+function addPermissionWarning(warnings, privatePath, label) {
+    if (privatePath === false) {
+        warnings.push(`The ${label} is readable or writable by group/other users; restrict it to owner-only permissions or run \`rgr doctor --fix\` for Ragmir-owned default paths.`);
+    }
+}
+function isNodeError(error) {
+    return error instanceof Error && "code" in error;
+}
 async function readGitignore(projectRoot) {
     const gitignorePath = path.join(projectRoot, ".gitignore");
     if (!existsSync(gitignorePath)) {
@@ -82,15 +153,37 @@ async function readGitignore(projectRoot) {
         .map((line) => line.trim())
         .filter(Boolean));
 }
-function hasGitignoreEntry(lines, entry) {
-    return lines.has(entry);
-}
 function usesProjectDirectory(projectRoot, filePath, directory) {
     const relativePath = normalizeRelativePath(projectRoot, filePath);
     return relativePath === directory || relativePath.startsWith(`${directory}/`);
 }
-function isPathIgnored(projectRoot, filePath, lines) {
+async function isPathIgnored(projectRoot, filePath, lines) {
     const relativePath = normalizeRelativePath(projectRoot, filePath);
+    if (isOutsideProject(relativePath)) {
+        return false;
+    }
+    const gitResult = await checkGitIgnored(projectRoot, relativePath);
+    if (gitResult !== null) {
+        return gitResult;
+    }
+    return isPathIgnoredByEntries(relativePath, lines);
+}
+function checkGitIgnored(projectRoot, relativePath) {
+    return new Promise((resolve) => {
+        execFile("git", ["check-ignore", "--quiet", "--", relativePath], { cwd: projectRoot, windowsHide: true }, (error) => {
+            if (!error) {
+                resolve(true);
+                return;
+            }
+            if (error.code === 1) {
+                resolve(false);
+                return;
+            }
+            resolve(null);
+        });
+    });
+}
+function isPathIgnoredByEntries(relativePath, lines) {
     const segments = relativePath.split("/");
     for (let index = 1; index <= segments.length; index += 1) {
         const prefix = segments.slice(0, index).join("/");
@@ -100,6 +193,9 @@ function isPathIgnored(projectRoot, filePath, lines) {
     }
     return false;
 }
+function isOutsideProject(relativePath) {
+    return relativePath === ".." || relativePath.startsWith("../") || path.isAbsolute(relativePath);
+}
 function normalizeRelativePath(projectRoot, filePath) {
     return path.relative(projectRoot, filePath).split(path.sep).join("/");
 }

package/dist/security.js.map

@@ -1 +1 @@
-{"version":3,"file":"security.js","sourceRoot":"","sources":["../src/security.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EACL,aAAa,EACb,yBAAyB,EACzB,kBAAkB,EAClB,8BAA8B,EAC9B,uCAAuC,EACvC,sBAAsB,GACvB,MAAM,eAAe,CAAA;AAGtB,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IACrD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IACzD,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,MAAM,eAAe,GAAG,iBAAiB,CAAC,SAAS,EAAE,yBAAyB,CAAC,CAAA;IAC/E,MAAM,aAAa,GAAG,iBAAiB,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAA;IAC1E,MAAM,oBAAoB,GACxB,iBAAiB,CAAC,SAAS,EAAE,8BAA8B,CAAC;QAC5D,iBAAiB,CAAC,SAAS,EAAE,uCAAuC,CAAC,CAAA;IACvE,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CACrF,CAAC,QAAQ,EAAE,EAAE,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,EAAE,aAAa,CAAC,CAChF,CAAA;IACD,MAAM,iBAAiB,GAAG,oBAAoB,CAC5C,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,MAAM,EACb,kBAAkB,CACnB,CAAA;IACD,MAAM,iBAAiB,GAAG,aAAa,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;IAEzF,IAAI,MAAM,CAAC,iBAAiB,KAAK,cAAc,IAAI,MAAM,CAAC,6BAA6B,EAAE,CAAC;QACxF,QAAQ,CAAC,IAAI,CACX,gGAAgG,CACjG,CAAA;IACH,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAA;IAC/F,CAAC;IACD,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,QAAQ,CAAC,IAAI,CAAC,GAAG,sBAAsB,yBAAyB,CAAC,CAAA;IACnE,CAAC;IACD,IAAI,YAAY,IAAI,CAAC,eAAe,EAAE,CAAC;QACrC,QAAQ,CAAC,IAAI,CAAC,GAAG,yBAAyB,yBAAyB,CAAC,CAAA;IACtE,CAAC;IACD,IAAI,iBAAiB,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,GAAG,8BAA8B,yBAAyB,CAAC,CAAA;IAC3E,CAAC;IAED,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,aAAa,EAAE,IAAI;QACnB,SAAS,EAAE;YACT,SAAS,EAAE,MAAM,CAAC,iBAAiB;YACnC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;YAC7C,6BAA6B,EAAE,MAAM,CAAC,6BAA6B;YACnE,aAAa,EAAE,KAAK;SACrB;QACD,SAAS,EAAE;YACT,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;YACjC,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;YACjC,cAAc,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;SACzE;QACD,SAAS,EAAE;YACT,OAAO,EAAE,MAAM,CAAC,SAAS;YACzB,IAAI,EAAE,MAAM,CAAC,aAAa;YAC1B,gBAAgB,EAAE,KAAK;SACxB;QACD,OAAO,EAAE;YACP,IAAI,EAAE,MAAM,CAAC,UAAU;YACvB,UAAU,EAAE,iBAAiB;YAC7B,eAAe,EAAE,mBAAmB;SACrC;QACD,GAAG,EAAE;YACH,OAAO,EAAE,MAAM,CAAC,UAAU;YAC1B,uBAAuB,EAAE,KAAK;SAC/B;QACD,SAAS,EAAE;YACT,eAAe;YACf,aAAa;YACb,oBAAoB;SACrB;QACD,eAAe,EAAE;YACf,sFAAsF;YACtF,yFAAyF;YACzF,4FAA4F;YAC5F,8EAA8E;SAC/E;QACD,QAAQ;KACT,CAAA;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,WAAmB;IAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;IAC1D,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/B,OAAO,IAAI,GAAG,EAAE,CAAA;IAClB,CAAC;IAED,OAAO,IAAI,GAAG,CACZ,CAAC,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;SACpC,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,OAAO,CAAC,CACnB,CAAA;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAkB,EAAE,KAAa;IAC1D,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;AACzB,CAAC;AAED,SAAS,oBAAoB,CAAC,WAAmB,EAAE,QAAgB,EAAE,SAAiB;IACpF,MAAM,YAAY,GAAG,qBAAqB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;IACjE,OAAO,YAAY,KAAK,SAAS,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,SAAS,GAAG,CAAC,CAAA;AAC/E,CAAC;AAED,SAAS,aAAa,CAAC,WAAmB,EAAE,QAAgB,EAAE,KAAkB;IAC9E,MAAM,YAAY,GAAG,qBAAqB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;IACjE,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IACxC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACjD,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9E,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,qBAAqB,CAAC,WAAmB,EAAE,QAAgB;IAClE,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACvE,CAAC"}
+{"version":3,"file":"security.js","sourceRoot":"","sources":["../src/security.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAC3D,OAAO,EACL,aAAa,EACb,yBAAyB,EACzB,kBAAkB,EAClB,8BAA8B,EAC9B,sBAAsB,GACvB,MAAM,eAAe,CAAA;AAGtB,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IACrD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;IACzD,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CACrF,CAAC,QAAQ,EAAE,EAAE,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,EAAE,aAAa,CAAC,CAChF,CAAA;IACD,MAAM,iBAAiB,GAAG,oBAAoB,CAC5C,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,MAAM,EACb,kBAAkB,CACnB,CAAA;IACD,MAAM,CACJ,eAAe,EACf,aAAa,EACb,oBAAoB,EACpB,iBAAiB,EACjB,mBAAmB,EACpB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACpB,aAAa,CACX,MAAM,CAAC,WAAW,EAClB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,yBAAyB,CAAC,EACxD,SAAS,CACV;QACD,aAAa,CACX,MAAM,CAAC,WAAW,EAClB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,sBAAsB,CAAC,EACrD,SAAS,CACV;QACD,aAAa,CACX,MAAM,CAAC,WAAW,EAClB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,8BAA8B,CAAC,EAC7D,SAAS,CACV;QACD,aAAa,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC;QAC/D,aAAa,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,aAAa,EAAE,SAAS,CAAC;KACnE,CAAC,CAAA;IACF,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAAC;QAC3C,UAAU,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,UAAU;QAC7C,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;KACpC,CAAC,CAAA;IAEF,IACE,MAAM,CAAC,cAAc,KAAK,SAAS;QACnC,MAAM,CAAC,iBAAiB,KAAK,cAAc;QAC3C,MAAM,CAAC,6BAA6B,EACpC,CAAC;QACD,QAAQ,CAAC,IAAI,CACX,gGAAgG,CACjG,CAAA;IACH,CAAC;IACD,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACrE,QAAQ,CAAC,IAAI,CAAC,8EAA8E,CAAC,CAAA;IAC/F,CAAC;IACD,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,QAAQ,CAAC,IAAI,CAAC,GAAG,sBAAsB,yBAAyB,CAAC,CAAA;IACnE,CAAC;IACD,IAAI,YAAY,IAAI,CAAC,eAAe,EAAE,CAAC;QACrC,QAAQ,CAAC,IAAI,CAAC,GAAG,yBAAyB,yBAAyB,CAAC,CAAA;IACtE,CAAC;IACD,IAAI,iBAAiB,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,GAAG,8BAA8B,yBAAyB,CAAC,CAAA;IAC3E,CAAC;IACD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAA;IACnE,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7C,QAAQ,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAA;IACtE,CAAC;IACD,oBAAoB,CAAC,QAAQ,EAAE,WAAW,CAAC,aAAa,EAAE,oBAAoB,CAAC,CAAA;IAC/E,oBAAoB,CAAC,QAAQ,EAAE,WAAW,CAAC,aAAa,EAAE,wBAAwB,CAAC,CAAA;IACnF,oBAAoB,CAAC,QAAQ,EAAE,WAAW,CAAC,iBAAiB,EAAE,yBAAyB,CAAC,CAAA;IACxF,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,oBAAoB,CAAC,QAAQ,EAAE,WAAW,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAA;IAC5E,CAAC;IAED,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,aAAa,EAAE,IAAI;QACnB,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,SAAS,EAAE;YACT,SAAS,EAAE,MAAM,CAAC,iBAAiB;YACnC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,sBAAsB,EAAE,MAAM,CAAC,sBAAsB;YACrD,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;YAC7C,6BAA6B,EAAE,MAAM,CAAC,6BAA6B;YACnE,aAAa,EAAE,KAAK;SACrB;QACD,SAAS,EAAE;YACT,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;YACjC,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO;YACjC,cAAc,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;SACzE;QACD,SAAS,EAAE;YACT,OAAO,EAAE,MAAM,CAAC,SAAS;YACzB,IAAI,EAAE,MAAM,CAAC,aAAa;YAC1B,gBAAgB,EAAE,KAAK;SACxB;QACD,OAAO,EAAE;YACP,IAAI,EAAE,MAAM,CAAC,UAAU;YACvB,UAAU,EAAE,iBAAiB;YAC7B,eAAe,EAAE,mBAAmB;SACrC;QACD,WAAW;QACX,GAAG,EAAE;YACH,OAAO,EAAE,MAAM,CAAC,UAAU;YAC1B,uBAAuB,EAAE,KAAK;SAC/B;QACD,SAAS,EAAE;YACT,eAAe;YACf,aAAa;YACb,oBAAoB;SACrB;QACD,eAAe,EAAE;YACf,sFAAsF;YACtF,yFAAyF;YACzF,4FAA4F;YAC5F,8EAA8E;SAC/E;QACD,QAAQ;KACT,CAAA;AACH,CAAC;AASD,KAAK,UAAU,kBAAkB,CAC/B,KAAsB;IAEtB,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QACjC,OAAO;YACL,OAAO,EAAE,KAAK;YACd,aAAa,EAAE,IAAI;YACnB,aAAa,EAAE,IAAI;YACnB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;SACvB,CAAA;IACH,CAAC;IACD,MAAM,CAAC,aAAa,EAAE,aAAa,EAAE,iBAAiB,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC5F,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC;QAC/B,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC;QAC3B,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC;QAC/B,aAAa,CAAC,KAAK,CAAC,aAAa,CAAC;KACnC,CAAC,CAAA;IACF,OAAO;QACL,OAAO,EAAE,IAAI;QACb,aAAa;QACb,aAAa;QACb,iBAAiB;QACjB,gBAAgB;KACjB,CAAA;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAA;QAChD,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAA;IAC7B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAAkB,EAClB,WAA2B,EAC3B,KAAa;IAEb,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;QAC1B,QAAQ,CAAC,IAAI,CACX,OAAO,KAAK,kJAAkJ,CAC/J,CAAA;IACH,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAA;AAClD,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,WAAmB;IAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;IAC1D,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/B,OAAO,IAAI,GAAG,EAAE,CAAA;IAClB,CAAC;IAED,OAAO,IAAI,GAAG,CACZ,CAAC,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;SACpC,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,OAAO,CAAC,CACnB,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,WAAmB,EAAE,QAAgB,EAAE,SAAiB;IACpF,MAAM,YAAY,GAAG,qBAAqB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;IACjE,OAAO,YAAY,KAAK,SAAS,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,SAAS,GAAG,CAAC,CAAA;AAC/E,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,WAAmB,EACnB,QAAgB,EAChB,KAAkB;IAElB,MAAM,YAAY,GAAG,qBAAqB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;IACjE,IAAI,gBAAgB,CAAC,YAAY,CAAC,EAAE,CAAC;QACnC,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,WAAW,EAAE,YAAY,CAAC,CAAA;IAClE,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;QACvB,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,OAAO,sBAAsB,CAAC,YAAY,EAAE,KAAK,CAAC,CAAA;AACpD,CAAC;AAED,SAAS,eAAe,CAAC,WAAmB,EAAE,YAAoB;IAChE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,QAAQ,CACN,KAAK,EACL,CAAC,cAAc,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,CAAC,EAC/C,EAAE,GAAG,EAAE,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,EACvC,CAAC,KAAK,EAAE,EAAE;YACR,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,CAAC,IAAI,CAAC,CAAA;gBACb,OAAM;YACR,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,KAAK,CAAC,CAAA;gBACd,OAAM;YACR,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAA;QACf,CAAC,CACF,CAAA;IACH,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,YAAoB,EAAE,KAAkB;IACtE,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IACxC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACjD,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9E,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,gBAAgB,CAAC,YAAoB;IAC5C,OAAO,YAAY,KAAK,IAAI,IAAI,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAA;AACjG,CAAC;AAED,SAAS,qBAAqB,CAAC,WAAmB,EAAE,QAAgB;IAClE,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACvE,CAAC"}

package/dist/store.d.ts

@@ -3,12 +3,15 @@ import type { Config, IndexManifest, VectorRow } from "./types.js";
 export interface EmptyTextFileRecord {
     relativePath: string;
     checksum: string;
+    bytes?: number;
+    mtimeMs?: number;
 }
 export interface IndexWriteResult {
     vectorIndexWarning: string | null;
     lexicalIndexWarning: string | null;
 }
 export declare function writeRows(rows: VectorRow[], config: Config): Promise<IndexWriteResult>;
+export declare function updateRows(rows: VectorRow[], replacePaths: string[], config: Config): Promise<IndexWriteResult>;
 export declare function writeIndexManifest(manifest: IndexManifest, config: Config): Promise<void>;
 export declare function readIndexManifest(config: Config): Promise<IndexManifest | null>;
 export declare function writeEmptyTextFiles(records: EmptyTextFileRecord[], config: Config): Promise<void>;

package/dist/store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAA;AAG3C,OAAO,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAkBlE,MAAM,WAAW,mBAAmB;IAClC,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAA;IACjC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAA;CACnC;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAiC5F;AAqED,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAI/F;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAqBrF;AAqBD,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,mBAAmB,EAAE,EAC9B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAcf;AAED,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAevF;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAOjF;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CASnE;AAED,wBAAsB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG/D"}
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAA;AAI3C,OAAO,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAIlE,MAAM,WAAW,mBAAmB;IAClC,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAA;IACjC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAA;CACnC;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAuB5F;AAED,wBAAsB,UAAU,CAC9B,IAAI,EAAE,SAAS,EAAE,EACjB,YAAY,EAAE,MAAM,EAAE,EACtB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,gBAAgB,CAAC,CAiB3B;AA4BD,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAK/F;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAqBrF;AAmCD,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,mBAAmB,EAAE,EAC9B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAef;AAED,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAevF;AAED,wBAAsB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,CAQjF;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAMnE;AAED,wBAAsB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAG/D"}

package/dist/store.js

@@ -1,85 +1,46 @@
-import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { readFile, rm, writeFile } from "node:fs/promises";
 import path from "node:path";
 import * as lancedb from "@lancedb/lancedb";
-import { VECTOR_DISTANCE_METRIC } from "./defaults.js";
+import { INDEX_MANIFEST_FILENAME } from "./defaults.js";
 import { isRecord } from "./guards.js";
+import { ensurePrivateDirectory, hardenPrivateFile } from "./permissions.js";
 const EMPTY_TEXT_FILES_MANIFEST = "empty-text-files.json";
-const INDEX_MANIFEST = "index-manifest.json";
-/**
- * LanceDB requires a minimum number of rows to train an IVF_PQ vector index.
- * Below this threshold, brute-force (flat) scan is used instead, which is
- * optimal for small corpora and avoids wasted index-training work.
- */
-const MIN_INDEX_ROWS = 256;
-/**
- * IVF partition count heuristic: roughly sqrt(row_count), bounded to keep both
- * small corpora (too many empty partitions) and very large corpora (training
- * cost) well-behaved. See LanceDB production guidance.
- */
-const MIN_IVF_PARTITIONS = 8;
-const MAX_IVF_PARTITIONS = 1024;
 export async function writeRows(rows, config) {
-    await mkdir(config.storageDir, { recursive: true });
+    await ensurePrivateDirectory(config.storageDir);
     const db = await lancedb.connect(config.storageDir);
     if (rows.length === 0) {
         const tableNames = await db.tableNames();
         if (tableNames.includes(config.tableName)) {
             await db.dropTable(config.tableName);
         }
-        await rm(path.join(config.storageDir, INDEX_MANIFEST), { force: true });
+        await rm(path.join(config.storageDir, INDEX_MANIFEST_FILENAME), { force: true });
         return { vectorIndexWarning: null, lexicalIndexWarning: null };
     }
-    const records = rows.map((row) => ({ ...row }));
+    const records = storedRows(rows);
     const table = await db.createTable(config.tableName, records, {
         mode: "overwrite",
     });
-    // Train an IVF_PQ vector index once the corpus is large enough to benefit
-    // from approximate nearest-neighbour search. Below the threshold, LanceDB
-    // falls back to an exact flat scan, which is faster for small corpora and
-    // avoids wasting work training an index on too few vectors.
-    if (rows.length < MIN_INDEX_ROWS) {
-        const lexicalResult = await ensureLexicalIndex(table);
-        return { vectorIndexWarning: null, lexicalIndexWarning: lexicalResult.warning };
-    }
-    const vectorResult = await ensureVectorIndex(table, rows.length);
     const lexicalResult = await ensureLexicalIndex(table);
     return {
-        vectorIndexWarning: vectorResult.warning,
+        vectorIndexWarning: null,
         lexicalIndexWarning: lexicalResult.warning,
     };
 }
-async function ensureVectorIndex(table, rowCount) {
-    const existing = await table.listIndices();
-    const hasVectorIndex = existing.some((index) => index.name === "vector_idx");
-    if (hasVectorIndex) {
-        return { created: false, warning: null };
+export async function updateRows(rows, replacePaths, config) {
+    const table = await openRowsTable(config);
+    if (!table) {
+        return writeRows(rows, config);
     }
-    // numSubVectors must divide the vector dimension evenly. 16 is a safe default
-    // for the 384-dim local-hash and mxbai-xsmall models; LanceDB validates and
-    // will reject a value that does not divide evenly, so larger models still
-    // fall back to flat scan rather than corrupting the index.
-    const numSubVectors = 16;
-    const numPartitions = clampIvfPartitions(Math.round(Math.sqrt(rowCount)));
-    try {
-        await table.createIndex("vector", {
-            config: lancedb.Index.ivfPq({
-                numPartitions,
-                numSubVectors,
-                distanceType: VECTOR_DISTANCE_METRIC,
-            }),
-        });
-        return { created: true, warning: null };
+    for (const paths of batches([...new Set(replacePaths)], 200)) {
+        if (paths.length > 0) {
+            await table.delete(`relativePath IN (${paths.map(sqlString).join(", ")})`);
+        }
     }
-    catch (error) {
-        // Index training can fail on edge-case dimensionality or tiny effective
-        // corpora; the table remains usable via flat scan, but the operator
-        // deserves to know queries will be slower than expected.
-        const detail = error instanceof Error ? error.message : String(error);
-        return {
-            created: false,
-            warning: `Vector index training failed (${detail}). Falling back to flat scan; queries will be slower on large corpora.`,
-        };
+    if (rows.length > 0) {
+        await table.add(storedRows(rows));
     }
+    const lexicalResult = await ensureLexicalIndex(table);
+    return { vectorIndexWarning: null, lexicalIndexWarning: lexicalResult.warning };
 }
 async function ensureLexicalIndex(table) {
     const existing = await table.listIndices();
@@ -89,7 +50,7 @@ async function ensureLexicalIndex(table) {
     }
     try {
         await table.createIndex("text", {
-            config: lancedb.Index.fts(),
+            config: lancedb.Index.fts({ asciiFolding: true, lowercase: true }),
         });
         return { created: true, warning: null };
     }
@@ -101,17 +62,15 @@ async function ensureLexicalIndex(table) {
         };
     }
 }
-function clampIvfPartitions(value) {
-    return Math.min(MAX_IVF_PARTITIONS, Math.max(MIN_IVF_PARTITIONS, value));
-}
 export async function writeIndexManifest(manifest, config) {
-    await mkdir(config.storageDir, { recursive: true });
-    const manifestPath = path.join(config.storageDir, INDEX_MANIFEST);
+    await ensurePrivateDirectory(config.storageDir);
+    const manifestPath = path.join(config.storageDir, INDEX_MANIFEST_FILENAME);
     await writeFile(manifestPath, JSON.stringify(manifest, null, 2), "utf8");
+    await hardenPrivateFile(manifestPath);
 }
 export async function readIndexManifest(config) {
     try {
-        const raw = JSON.parse(await readFile(path.join(config.storageDir, INDEX_MANIFEST), "utf8"));
+        const raw = JSON.parse(await readFile(path.join(config.storageDir, INDEX_MANIFEST_FILENAME), "utf8"));
         if (!isRecord(raw)) {
             return null;
         }
@@ -139,12 +98,23 @@ function isIndexManifest(value) {
         typeof value.ragmirVersion === "string" &&
         (value.embeddingProvider === "local-hash" || value.embeddingProvider === "transformers") &&
         typeof value.embeddingModel === "string" &&
+        (!("indexPolicyFingerprint" in value) || typeof value.indexPolicyFingerprint === "string") &&
         (!("vectorDimension" in value) || typeof value.vectorDimension === "number") &&
         (!("vectorDistanceMetric" in value) || typeof value.vectorDistanceMetric === "string") &&
         typeof value.chunkSize === "number" &&
         typeof value.chunkOverlap === "number" &&
         typeof value.fileCount === "number" &&
-        typeof value.chunkCount === "number");
+        typeof value.chunkCount === "number" &&
+        (!("indexedFiles" in value) ||
+            (Array.isArray(value.indexedFiles) && value.indexedFiles.every(isIndexManifestFile))));
+}
+function isIndexManifestFile(value) {
+    return (isRecord(value) &&
+        typeof value.relativePath === "string" &&
+        typeof value.checksum === "string" &&
+        typeof value.chunkCount === "number" &&
+        (!("bytes" in value) || typeof value.bytes === "number") &&
+        (!("mtimeMs" in value) || typeof value.mtimeMs === "number"));
 }
 export async function writeEmptyTextFiles(records, config) {
     const manifestPath = path.join(config.storageDir, EMPTY_TEXT_FILES_MANIFEST);
@@ -152,9 +122,10 @@ export async function writeEmptyTextFiles(records, config) {
         await rm(manifestPath, { force: true });
         return;
     }
-    await mkdir(config.storageDir, { recursive: true });
+    await ensurePrivateDirectory(config.storageDir);
     const sortedRecords = [...records].sort((a, b) => a.relativePath.localeCompare(b.relativePath));
     await writeFile(manifestPath, JSON.stringify({ version: 1, files: sortedRecords }, null, 2), "utf8");
+    await hardenPrivateFile(manifestPath);
 }
 export async function readEmptyTextFiles(config) {
     try {
@@ -172,6 +143,7 @@ export async function readEmptyTextFiles(config) {
     }
 }
 export async function openRowsTable(config) {
+    await ensurePrivateDirectory(config.storageDir);
     const db = await lancedb.connect(config.storageDir);
     const tableNames = await db.tableNames();
     if (!tableNames.includes(config.tableName)) {
@@ -184,15 +156,38 @@ export async function readRows(config) {
     if (!table) {
         return [];
     }
-    return (await table.query().toArray()).map((row) => ({
-        ...row,
-        vector: normalizeVector(row.vector),
-    }));
+    return (await table.query().toArray()).map(vectorRowFromStored);
 }
 export async function countRows(config) {
     const table = await openRowsTable(config);
     return table ? table.countRows() : 0;
 }
+function storedRows(rows) {
+    return rows.map((row) => ({
+        ...row,
+        pageStart: row.pageStart ?? 0,
+        pageEnd: row.pageEnd ?? 0,
+    }));
+}
+function vectorRowFromStored(row) {
+    const { pageStart, pageEnd, ...rest } = row;
+    return {
+        ...rest,
+        vector: normalizeVector(row.vector),
+        ...(typeof pageStart === "number" && pageStart > 0 ? { pageStart } : {}),
+        ...(typeof pageEnd === "number" && pageEnd > 0 ? { pageEnd } : {}),
+    };
+}
+function sqlString(value) {
+    return `'${value.replaceAll("'", "''")}'`;
+}
+function batches(values, size) {
+    const result = [];
+    for (let index = 0; index < values.length; index += size) {
+        result.push(values.slice(index, index + size));
+    }
+    return result;
+}
 function normalizeVector(vector) {
     if (Array.isArray(vector) && vector.every((value) => typeof value === "number")) {
         return vector;
@@ -214,7 +209,11 @@ function hasIndexedNumberGetter(value) {
         typeof value.get === "function");
 }
 function isEmptyTextFileRecord(value) {
-    return (isRecord(value) && typeof value.relativePath === "string" && typeof value.checksum === "string");
+    return (isRecord(value) &&
+        typeof value.relativePath === "string" &&
+        typeof value.checksum === "string" &&
+        (!("bytes" in value) || typeof value.bytes === "number") &&
+        (!("mtimeMs" in value) || typeof value.mtimeMs === "number"));
 }
 function isNodeError(error) {
     return error instanceof Error && "code" in error;

package/dist/store.js.map

@@ -1 +1 @@
-{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACjE,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAA;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAGtC,MAAM,yBAAyB,GAAG,uBAAuB,CAAA;AACzD,MAAM,cAAc,GAAG,qBAAqB,CAAA;AAC5C;;;;GAIG;AACH,MAAM,cAAc,GAAG,GAAG,CAAA;AAC1B;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAC5B,MAAM,kBAAkB,GAAG,IAAI,CAAA;AAY/B,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAiB,EAAE,MAAc;IAC/D,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACnD,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAEnD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;QACxC,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1C,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QACtC,CAAC;QACD,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,cAAc,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QACvE,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAA;IAChE,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC,CAAA;IAC/C,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE;QAC5D,IAAI,EAAE,WAAW;KAClB,CAAC,CAAA;IAEF,0EAA0E;IAC1E,0EAA0E;IAC1E,0EAA0E;IAC1E,4DAA4D;IAC5D,IAAI,IAAI,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QACjC,MAAM,aAAa,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAA;QACrD,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,mBAAmB,EAAE,aAAa,CAAC,OAAO,EAAE,CAAA;IACjF,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAChE,MAAM,aAAa,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAA;IACrD,OAAO;QACL,kBAAkB,EAAE,YAAY,CAAC,OAAO;QACxC,mBAAmB,EAAE,aAAa,CAAC,OAAO;KAC3C,CAAA;AACH,CAAC;AAOD,KAAK,UAAU,iBAAiB,CAC9B,KAAoB,EACpB,QAAgB;IAEhB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,EAAE,CAAA;IAC1C,MAAM,cAAc,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC,CAAA;IAC5E,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IAC1C,CAAC;IAED,8EAA8E;IAC9E,4EAA4E;IAC5E,0EAA0E;IAC1E,2DAA2D;IAC3D,MAAM,aAAa,GAAG,EAAE,CAAA;IACxB,MAAM,aAAa,GAAG,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;IACzE,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;gBAC1B,aAAa;gBACb,aAAa;gBACb,YAAY,EAAE,sBAAsB;aACrC,CAAC;SACH,CAAC,CAAA;QACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wEAAwE;QACxE,oEAAoE;QACpE,yDAAyD;QACzD,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACrE,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,iCAAiC,MAAM,wEAAwE;SACzH,CAAA;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,KAAoB;IACpD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,EAAE,CAAA;IAC1C,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,CAAC,CAAA;IACxE,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IAC1C,CAAC;IAED,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE;YAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE;SAC5B,CAAC,CAAA;QACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACrE,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,oCAAoC,MAAM,yFAAyF;SAC7I,CAAA;IACH,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAa;IACvC,OAAO,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,KAAK,CAAC,CAAC,CAAA;AAC1E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAuB,EAAE,MAAc;IAC9E,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACnD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,cAAc,CAAC,CAAA;IACjE,MAAM,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;AAC1E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAc;IACpD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAC1D,CAAA;QACZ,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,OAAO,IAAI,CAAA;QACb,CAAC;QACD,OAAO,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAA;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,4EAA4E;QAC5E,yEAAyE;QACzE,wBAAwB;QACxB,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YACjC,OAAO,IAAI,CAAA;QACb,CAAC;QACD,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,KAAK,CAAA;IACd,CAAC;IACD,OAAO,CACL,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ;QACvC,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ;QACvC,CAAC,KAAK,CAAC,iBAAiB,KAAK,YAAY,IAAI,KAAK,CAAC,iBAAiB,KAAK,cAAc,CAAC;QACxF,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ;QACxC,CAAC,CAAC,CAAC,iBAAiB,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,eAAe,KAAK,QAAQ,CAAC;QAC5E,CAAC,CAAC,CAAC,sBAAsB,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,oBAAoB,KAAK,QAAQ,CAAC;QACtF,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ;QACtC,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,CACrC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAA8B,EAC9B,MAAc;IAEd,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAA;IAC5E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,EAAE,CAAC,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QACvC,OAAM;IACR,CAAC;IAED,MAAM,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACnD,MAAM,aAAa,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;IAC/F,MAAM,SAAS,CACb,YAAY,EACZ,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAC7D,MAAM,CACP,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAc;IACrD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CACzB,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,yBAAyB,CAAC,EAAE,MAAM,CAAC,CACrE,CAAA;QACZ,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1D,OAAO,EAAE,CAAA;QACX,CAAC;QACD,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAA;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,EAAE,CAAA;QACX,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc;IAChD,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IACnD,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;IACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAA;IACb,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAc;IAC3C,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,CAAA;IACX,CAAC;IACD,OAAQ,CAAC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAuB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC1E,GAAG,GAAG;QACN,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC;KACpC,CAAC,CAAC,CAAA;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAc;IAC5C,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACzC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AACtC,CAAC;AAMD,SAAS,eAAe,CAAC,MAAe;IACtC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,EAAE,CAAC;QAChF,OAAO,MAAM,CAAA;IACf,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;QACrD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAsC,CAAC,CAAA;IAC3D,CAAC;IACD,IAAI,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAA;IAC/E,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;AAC/D,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAc;IAI5C,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;QAChC,KAAK,IAAI,KAAK;QACd,OAAO,KAAK,CAAC,GAAG,KAAK,UAAU,CAChC,CAAA;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC3C,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAChG,CAAA;AACH,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAA;AAClD,CAAC"}
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAA;AAC3C,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAG5E,MAAM,yBAAyB,GAAG,uBAAuB,CAAA;AAczD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAiB,EAAE,MAAc;IAC/D,MAAM,sBAAsB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAC/C,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAEnD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;QACxC,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1C,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QACtC,CAAC;QACD,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,uBAAuB,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAChF,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAA;IAChE,CAAC;IAED,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,CAAA;IAChC,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE;QAC5D,IAAI,EAAE,WAAW;KAClB,CAAC,CAAA;IAEF,MAAM,aAAa,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAA;IACrD,OAAO;QACL,kBAAkB,EAAE,IAAI;QACxB,mBAAmB,EAAE,aAAa,CAAC,OAAO;KAC3C,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAiB,EACjB,YAAsB,EACtB,MAAc;IAEd,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAChC,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QAC7D,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,KAAK,CAAC,MAAM,CAAC,oBAAoB,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC5E,CAAC;IACH,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAA;IACnC,CAAC;IAED,MAAM,aAAa,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAA;IACrD,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,mBAAmB,EAAE,aAAa,CAAC,OAAO,EAAE,CAAA;AACjF,CAAC;AAOD,KAAK,UAAU,kBAAkB,CAAC,KAAoB;IACpD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,EAAE,CAAA;IAC1C,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,CAAC,CAAA;IACxE,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IAC1C,CAAC;IAED,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE;YAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;SACnE,CAAC,CAAA;QACF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACrE,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,oCAAoC,MAAM,yFAAyF;SAC7I,CAAA;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAuB,EAAE,MAAc;IAC9E,MAAM,sBAAsB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAC/C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;IAC1E,MAAM,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;IACxE,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAc;IACpD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,uBAAuB,CAAC,EAAE,MAAM,CAAC,CACnE,CAAA;QACZ,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,OAAO,IAAI,CAAA;QACb,CAAC;QACD,OAAO,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAA;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,4EAA4E;QAC5E,yEAAyE;QACzE,wBAAwB;QACxB,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YACjC,OAAO,IAAI,CAAA;QACb,CAAC;QACD,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,KAAK,CAAA;IACd,CAAC;IACD,OAAO,CACL,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ;QACvC,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ;QACvC,CAAC,KAAK,CAAC,iBAAiB,KAAK,YAAY,IAAI,KAAK,CAAC,iBAAiB,KAAK,cAAc,CAAC;QACxF,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ;QACxC,CAAC,CAAC,CAAC,wBAAwB,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,sBAAsB,KAAK,QAAQ,CAAC;QAC1F,CAAC,CAAC,CAAC,iBAAiB,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,eAAe,KAAK,QAAQ,CAAC;QAC5E,CAAC,CAAC,CAAC,sBAAsB,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,oBAAoB,KAAK,QAAQ,CAAC;QACtF,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ;QACtC,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ;QACpC,CAAC,CAAC,CAAC,cAAc,IAAI,KAAK,CAAC;YACzB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CACxF,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc;IACzC,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ;QACtC,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ;QAClC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ;QACpC,CAAC,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC;QACxD,CAAC,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAC7D,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAA8B,EAC9B,MAAc;IAEd,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAA;IAC5E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,EAAE,CAAC,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QACvC,OAAM;IACR,CAAC;IAED,MAAM,sBAAsB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAC/C,MAAM,aAAa,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAA;IAC/F,MAAM,SAAS,CACb,YAAY,EACZ,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAC7D,MAAM,CACP,CAAA;IACD,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAc;IACrD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CACzB,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,yBAAyB,CAAC,EAAE,MAAM,CAAC,CACrE,CAAA;QACZ,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1D,OAAO,EAAE,CAAA;QACX,CAAC;QACD,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAA;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClD,OAAO,EAAE,CAAA;QACX,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAc;IAChD,MAAM,sBAAsB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAC/C,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IACnD,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAA;IACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAA;IACb,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAc;IAC3C,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,CAAA;IACX,CAAC;IACD,OAAQ,CAAC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAuB,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAA;AACxF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAc;IAC5C,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAA;IACzC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AACtC,CAAC;AAQD,SAAS,UAAU,CAAC,IAAiB;IACnC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,GAAG,GAAG;QACN,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,CAAC;QAC7B,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,CAAC;KAC1B,CAAC,CAAC,CAAA;AACL,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAoB;IAC/C,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAA;IAC3C,OAAO;QACL,GAAG,IAAI;QACP,MAAM,EAAE,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC;QACnC,GAAG,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,GAAG,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACnE,CAAA;AACH,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAA;AAC3C,CAAC;AAED,SAAS,OAAO,CAAI,MAAW,EAAE,IAAY;IAC3C,MAAM,MAAM,GAAU,EAAE,CAAA;IACxB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,EAAE,CAAC;QACzD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,CAAA;IAChD,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,SAAS,eAAe,CAAC,MAAe;IACtC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,EAAE,CAAC;QAChF,OAAO,MAAM,CAAA;IACf,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;QACrD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAsC,CAAC,CAAA;IAC3D,CAAC;IACD,IAAI,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAA;IAC/E,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;AAC/D,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAc;IAI5C,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;QAChC,KAAK,IAAI,KAAK;QACd,OAAO,KAAK,CAAC,GAAG,KAAK,UAAU,CAChC,CAAA;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC3C,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ;QACtC,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ;QAClC,CAAC,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC;QACxD,CAAC,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAC7D,CAAA;AACH,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,CAAA;AAClD,CAAC"}

package/dist/text.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../src/text.ts"],"names":[],"mappings":"AAAA,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAE/C"}
+{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../src/text.ts"],"names":[],"mappings":"AAAA,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAgB/C"}

package/dist/text.js

@@ -5,6 +5,18 @@ export function normalizeForMatch(text) {
         .replace(/\p{Diacritic}/gu, "");
 }
 export function tokenize(text) {
-    return normalizeForMatch(text).match(/[\p{L}\p{N}]{2,}/gu) ?? [];
+    const normalized = normalizeForMatch(text);
+    const tokens = [];
+    const segmenter = new Intl.Segmenter("und", { granularity: "word" });
+    for (const segment of segmenter.segment(normalized)) {
+        if (!segment.isWordLike) {
+            continue;
+        }
+        const token = segment.segment;
+        if (token.length >= 2 || /[^\p{Script=Latin}\p{N}]/u.test(token)) {
+            tokens.push(token);
+        }
+    }
+    return tokens;
 }
 //# sourceMappingURL=text.js.map

package/dist/text.js.map

@@ -1 +1 @@
-{"version":3,"file":"text.js","sourceRoot":"","sources":["../src/text.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,OAAO,IAAI;SACR,WAAW,EAAE;SACb,SAAS,CAAC,MAAM,CAAC;SACjB,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAA;AACnC,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAA;AAClE,CAAC"}
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../src/text.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,OAAO,IAAI;SACR,WAAW,EAAE;SACb,SAAS,CAAC,MAAM,CAAC;SACjB,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAA;AACnC,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAA;IAEpE,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACpD,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;YACxB,SAAQ;QACV,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAA;QAC7B,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,2BAA2B,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACpB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC"}