@memrosetta/core 0.4.0 → 0.4.1

package/dist/index.js

@@ -88,6 +88,26 @@ var SCHEMA_V5 = `
 ALTER TABLE memories ADD COLUMN use_count INTEGER DEFAULT 0;
 ALTER TABLE memories ADD COLUMN success_count INTEGER DEFAULT 0;
 `;
+var SCHEMA_V6 = `
+CREATE TABLE IF NOT EXISTS migration_version (
+  name        TEXT PRIMARY KEY,
+  applied_at  TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS memory_legacy_scope (
+  memory_id         TEXT PRIMARY KEY,
+  legacy_user_id    TEXT NOT NULL,
+  legacy_namespace  TEXT,
+  migrated_at       TEXT NOT NULL,
+  FOREIGN KEY (memory_id) REFERENCES memories(memory_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_memory_legacy_scope_user
+  ON memory_legacy_scope(legacy_user_id);
+
+CREATE INDEX IF NOT EXISTS idx_memory_legacy_scope_user_ns
+  ON memory_legacy_scope(legacy_user_id, legacy_namespace);
+`;
 function schemaV2(dim) {
   return `CREATE VIRTUAL TABLE IF NOT EXISTS vec_memories USING vec0(embedding float[${dim}]);`;
 }
@@ -122,7 +142,8 @@ function ensureSchema(db, options) {
       db.exec(schemaV2(dim));
       version = 2;
     }
-    version = 5;
+    db.exec(SCHEMA_V6);
+    version = 6;
     db.prepare("INSERT INTO schema_version (version, embedding_dimension) VALUES (?, ?)").run(version, dim);
     return;
   }
@@ -154,6 +175,10 @@ function ensureSchema(db, options) {
     }
     db.prepare("UPDATE schema_version SET version = ?").run(5);
   }
+  if (currentVersion < 6) {
+    db.exec(SCHEMA_V6);
+    db.prepare("UPDATE schema_version SET version = ?").run(6);
+  }
   if (options?.vectorEnabled) {
     const hasVecTable = db.prepare(
       "SELECT name FROM sqlite_master WHERE type='table' AND name='vec_memories'"
@@ -490,7 +515,7 @@ function getRelationsByMemory(stmts, memoryId) {
 }
 
 // src/search.ts
-var FTS5_SPECIAL_CHARS = /["\*\(\):^{}\[\]?!.,;'\\]/g;
+var FTS5_SPECIAL_CHARS = /["\*\(\):^{}\[\]?!.,;'\\/]/g;
 var STOP_WORDS = /* @__PURE__ */ new Set([
   "a",
   "an",
@@ -603,16 +628,32 @@ var STOP_WORDS = /* @__PURE__ */ new Set([
   "got",
   "getting"
 ]);
-function buildFtsQuery(rawQuery) {
-  const allTokens = rawQuery.toLowerCase().split(/\s+/).filter((t) => t.length > 0).map((t) => t.replace(FTS5_SPECIAL_CHARS, "")).filter((t) => t.length > 0);
-  if (allTokens.length === 0) {
-    return "";
+var KOREAN_LOW_SIGNAL_TOKENS = /* @__PURE__ */ new Set([
+  "\uBB50\uC9C0",
+  "\uBB50\uC57C",
+  "\uBB54\uC9C0",
+  "\uC5B4\uB514",
+  "\uC65C",
+  "\uC5B4\uB5BB\uAC8C",
+  "\uC5B8\uC81C",
+  "\uB204\uAD6C",
+  "\uC54C\uB824\uC918"
+]);
+function preprocessQuery(rawQuery) {
+  const normalized = rawQuery.normalize("NFKC").toLowerCase().split(/\s+/).filter((t) => t.length > 0).map((t) => t.replace(FTS5_SPECIAL_CHARS, "")).filter((t) => t.length > 0);
+  if (normalized.length === 0) {
+    return [];
   }
-  const meaningful = allTokens.filter((t) => !STOP_WORDS.has(t));
-  const tokens = meaningful.length > 0 ? meaningful : allTokens;
+  const meaningful = normalized.filter(
+    (t) => !STOP_WORDS.has(t) && !KOREAN_LOW_SIGNAL_TOKENS.has(t)
+  );
+  return meaningful.length > 0 ? meaningful : normalized;
+}
+function buildFtsQuery(rawQuery) {
+  const tokens = preprocessQuery(rawQuery);
   if (tokens.length === 0) return "";
   if (tokens.length === 1) return `"${tokens[0]}"`;
-  if (tokens.length <= 4) {
+  if (tokens.length === 2) {
     return tokens.map((t) => `"${t}"`).join(" AND ");
   }
   return tokens.map((t) => `"${t}"`).join(" OR ");
@@ -1064,9 +1105,7 @@ function applyKeywordBoost(results, queryTokens) {
   }).sort((a, b) => b.score - a.score);
 }
 function extractQueryTokens(rawQuery) {
-  const allTokens = rawQuery.toLowerCase().split(/\s+/).filter((t) => t.length > 0).map((t) => t.replace(FTS5_SPECIAL_CHARS, "")).filter((t) => t.length > 0);
-  const meaningful = allTokens.filter((t) => !STOP_WORDS.has(t));
-  return meaningful.length > 0 ? meaningful : allTokens;
+  return preprocessQuery(rawQuery);
 }
 function updateAccessTracking(db, memoryIds) {
   if (memoryIds.length === 0) return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@memrosetta/core",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",