@open-mercato/search 0.7.1-develop.7113.1.9d83dca10c → 0.7.1-develop.7121.1.734c263bda

package/dist/strategies/token.strategy.js

@@ -7,6 +7,22 @@ function normalizeOrganizationIds(options) {
     options.organizationIds.map((value) => typeof value === "string" ? value.trim() : "").filter((value) => value.length > 0)
   ));
 }
+const CF_ALIAS_PREFIX = "cf_";
+const CF_CANONICAL_PREFIX = "cf:";
+function normalizeCustomFieldKeys(fields) {
+  if (!fields) return fields;
+  const normalized = {};
+  for (const [key, value] of Object.entries(fields)) {
+    if (!key.startsWith(CF_ALIAS_PREFIX)) {
+      normalized[key] = value;
+      continue;
+    }
+    const canonical = `${CF_CANONICAL_PREFIX}${key.slice(CF_ALIAS_PREFIX.length)}`;
+    if (canonical in fields) continue;
+    normalized[canonical] = value;
+  }
+  return normalized;
+}
 class TokenSearchStrategy {
   constructor(db, config) {
     this.db = db;
@@ -70,7 +86,7 @@ class TokenSearchStrategy {
       recordId: record.recordId,
       tenantId: record.tenantId,
       organizationId: record.organizationId,
-      doc: record.fields
+      doc: normalizeCustomFieldKeys(record.fields)
     });
   }
   async delete(entityId, recordId, tenantId) {
@@ -89,7 +105,7 @@ class TokenSearchStrategy {
       recordId: record.recordId,
       tenantId: record.tenantId,
       organizationId: record.organizationId,
-      doc: record.fields
+      doc: normalizeCustomFieldKeys(record.fields)
     }));
     await replaceSearchTokensForBatch(this.db, payloads);
   }

package/dist/strategies/token.strategy.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/strategies/token.strategy.ts"],
-  "sourcesContent": ["import { type Kysely, sql, type SqlBool } from 'kysely'\nimport type {\n  SearchStrategy,\n  SearchStrategyId,\n  SearchOptions,\n  SearchResult,\n  IndexableRecord,\n} from '../types'\nimport type { EntityId } from '@open-mercato/shared/modules/entities'\n\n/**\n * Configuration for TokenSearchStrategy.\n */\nexport type TokenStrategyConfig = {\n  /** Minimum number of query tokens that must match (0-1 ratio, default 0.5) */\n  minMatchRatio?: number\n  /** Default limit for search results */\n  defaultLimit?: number\n}\n\nfunction normalizeOrganizationIds(options: SearchOptions): string[] | null {\n  const single = typeof options.organizationId === 'string' ? options.organizationId.trim() : ''\n  if (single) return [single]\n  if (!Array.isArray(options.organizationIds)) return null\n  return Array.from(new Set(\n    options.organizationIds\n      .map((value) => (typeof value === 'string' ? value.trim() : ''))\n      .filter((value) => value.length > 0),\n  ))\n}\n\n/**\n * TokenSearchStrategy provides hash-based search using the existing search_tokens table.\n * This strategy is always available and serves as a fallback when other strategies fail.\n *\n * It tokenizes queries into hashes and matches against pre-indexed token hashes,\n * enabling search on encrypted fields without exposing plaintext to external services.\n */\nexport class TokenSearchStrategy implements SearchStrategy {\n  readonly id: SearchStrategyId = 'tokens'\n  readonly name = 'Token Search'\n  readonly priority = 10 // Lowest priority, always available as fallback\n\n  private readonly minMatchRatio: number\n  private readonly defaultLimit: number\n\n  constructor(\n    private readonly db: Kysely<any>,\n    config?: TokenStrategyConfig,\n  ) {\n    this.minMatchRatio = config?.minMatchRatio ?? 0.5\n    this.defaultLimit = config?.defaultLimit ?? 50\n  }\n\n  async isAvailable(): Promise<boolean> {\n    return true // Always available\n  }\n\n  async ensureReady(): Promise<void> {\n    // No initialization needed\n  }\n\n  async search(query: string, options: SearchOptions): Promise<SearchResult[]> {\n    const organizationIds = normalizeOrganizationIds(options)\n    if (organizationIds && organizationIds.length === 0) return []\n\n    // Dynamically import tokenization to avoid circular dependencies\n    const { tokenizeText } = await import('@open-mercato/shared/lib/search/tokenize')\n    const { resolveSearchConfig } = await import('@open-mercato/shared/lib/search/config')\n    const { listSearchTokenExcludedEntityTypes } = await import(\n      '@open-mercato/core/modules/query_index/lib/search-entity-policy'\n    )\n\n    const config = resolveSearchConfig()\n    if (!config.enabled) return []\n\n    // The rows themselves stay in `search_tokens` \u2014 list routes and the query engines' encrypted\n    // like/ilike rewrite depend on them \u2014 so the exclusion is enforced here, at read time.\n    const excludedEntityTypes = listSearchTokenExcludedEntityTypes()\n    const requestedEntityTypes = options.entityTypes?.length\n      ? options.entityTypes.filter((entityType) => !excludedEntityTypes.includes(entityType))\n      : undefined\n    if (options.entityTypes?.length && !requestedEntityTypes?.length) return []\n\n    const { hashes } = tokenizeText(query, config)\n    if (hashes.length === 0) return []\n\n    const minMatches = Math.max(1, Math.ceil(hashes.length * this.minMatchRatio))\n    const limit = options.limit ?? this.defaultLimit\n\n    let queryBuilder = this.db\n      .selectFrom('search_tokens' as any)\n      .select([\n        'entity_type' as any,\n        'entity_id' as any,\n        'organization_id' as any,\n        sql<string>`count(*)`.as('match_count'),\n      ])\n      .where('token_hash' as any, 'in', hashes)\n      .where('tenant_id' as any, '=', options.tenantId)\n      .groupBy(['entity_type' as any, 'entity_id' as any, 'organization_id' as any])\n      .having(sql<SqlBool>`count(distinct token_hash) >= ${minMatches}`)\n      .orderBy(sql`count(distinct token_hash) desc`)\n      .limit(limit)\n\n    if (organizationIds) {\n      queryBuilder = queryBuilder.where('organization_id' as any, 'in', organizationIds)\n    }\n\n    if (requestedEntityTypes?.length) {\n      queryBuilder = queryBuilder.where('entity_type' as any, 'in', requestedEntityTypes)\n    } else if (excludedEntityTypes.length) {\n      queryBuilder = queryBuilder.where('entity_type' as any, 'not in', excludedEntityTypes)\n    }\n\n    const rows = await queryBuilder.execute() as Array<{\n      entity_type: string\n      entity_id: string\n      organization_id: string | null\n      match_count: string | number\n    }>\n\n    return rows.map((row) => {\n      const matchCount = typeof row.match_count === 'string'\n        ? parseInt(row.match_count, 10)\n        : row.match_count\n      // Calculate score based on match ratio\n      const score = matchCount / hashes.length\n\n      return {\n        entityId: row.entity_type as EntityId,\n        recordId: row.entity_id,\n        score,\n        source: this.id,\n        organizationId: row.organization_id ?? null,\n      }\n    })\n  }\n\n  async index(record: IndexableRecord): Promise<void> {\n    // Dynamically import to avoid circular dependencies\n    const { replaceSearchTokensForRecord } = await import(\n      '@open-mercato/core/modules/query_index/lib/search-tokens'\n    )\n\n    await replaceSearchTokensForRecord(this.db, {\n      entityType: record.entityId,\n      recordId: record.recordId,\n      tenantId: record.tenantId,\n      organizationId: record.organizationId,\n      doc: record.fields,\n    })\n  }\n\n  async delete(entityId: EntityId, recordId: string, tenantId: string): Promise<void> {\n    // Dynamically import to avoid circular dependencies\n    const { deleteSearchTokensForRecord } = await import(\n      '@open-mercato/core/modules/query_index/lib/search-tokens'\n    )\n\n    await deleteSearchTokensForRecord(this.db, {\n      entityType: entityId,\n      recordId,\n      tenantId,\n    })\n  }\n\n  async bulkIndex(records: IndexableRecord[]): Promise<void> {\n    if (records.length === 0) return\n\n    const { replaceSearchTokensForBatch } = await import(\n      '@open-mercato/core/modules/query_index/lib/search-tokens'\n    )\n\n    const payloads = records.map((record) => ({\n      entityType: record.entityId,\n      recordId: record.recordId,\n      tenantId: record.tenantId,\n      organizationId: record.organizationId,\n      doc: record.fields as Record<string, unknown>,\n    }))\n\n    await replaceSearchTokensForBatch(this.db, payloads)\n  }\n\n  async purge(entityId: EntityId, tenantId: string, organizationId?: string | null): Promise<void> {\n    const normalizedOrganizationId =\n      typeof organizationId === 'string' && organizationId.trim().length > 0 ? organizationId.trim() : null\n    let query = this.db\n      .deleteFrom('search_tokens' as any)\n      .where('entity_type' as any, '=', entityId)\n      .where('tenant_id' as any, '=', tenantId)\n    if (normalizedOrganizationId !== null) {\n      query = query.where('organization_id' as any, '=', normalizedOrganizationId)\n    }\n    await query.execute()\n  }\n}\n"],
-  "mappings": "AAAA,SAAsB,WAAyB;AAoB/C,SAAS,yBAAyB,SAAyC;AACzE,QAAM,SAAS,OAAO,QAAQ,mBAAmB,WAAW,QAAQ,eAAe,KAAK,IAAI;AAC5F,MAAI,OAAQ,QAAO,CAAC,MAAM;AAC1B,MAAI,CAAC,MAAM,QAAQ,QAAQ,eAAe,EAAG,QAAO;AACpD,SAAO,MAAM,KAAK,IAAI;AAAA,IACpB,QAAQ,gBACL,IAAI,CAAC,UAAW,OAAO,UAAU,WAAW,MAAM,KAAK,IAAI,EAAG,EAC9D,OAAO,CAAC,UAAU,MAAM,SAAS,CAAC;AAAA,EACvC,CAAC;AACH;AASO,MAAM,oBAA8C;AAAA,EAQzD,YACmB,IACjB,QACA;AAFiB;AARnB,SAAS,KAAuB;AAChC,SAAS,OAAO;AAChB,SAAS,WAAW;AASlB,SAAK,gBAAgB,QAAQ,iBAAiB;AAC9C,SAAK,eAAe,QAAQ,gBAAgB;AAAA,EAC9C;AAAA,EAEA,MAAM,cAAgC;AACpC,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,cAA6B;AAAA,EAEnC;AAAA,EAEA,MAAM,OAAO,OAAe,SAAiD;AAC3E,UAAM,kBAAkB,yBAAyB,OAAO;AACxD,QAAI,mBAAmB,gBAAgB,WAAW,EAAG,QAAO,CAAC;AAG7D,UAAM,EAAE,aAAa,IAAI,MAAM,OAAO,0CAA0C;AAChF,UAAM,EAAE,oBAAoB,IAAI,MAAM,OAAO,wCAAwC;AACrF,UAAM,EAAE,mCAAmC,IAAI,MAAM,OACnD,iEACF;AAEA,UAAM,SAAS,oBAAoB;AACnC,QAAI,CAAC,OAAO,QAAS,QAAO,CAAC;AAI7B,UAAM,sBAAsB,mCAAmC;AAC/D,UAAM,uBAAuB,QAAQ,aAAa,SAC9C,QAAQ,YAAY,OAAO,CAAC,eAAe,CAAC,oBAAoB,SAAS,UAAU,CAAC,IACpF;AACJ,QAAI,QAAQ,aAAa,UAAU,CAAC,sBAAsB,OAAQ,QAAO,CAAC;AAE1E,UAAM,EAAE,OAAO,IAAI,aAAa,OAAO,MAAM;AAC7C,QAAI,OAAO,WAAW,EAAG,QAAO,CAAC;AAEjC,UAAM,aAAa,KAAK,IAAI,GAAG,KAAK,KAAK,OAAO,SAAS,KAAK,aAAa,CAAC;AAC5E,UAAM,QAAQ,QAAQ,SAAS,KAAK;AAEpC,QAAI,eAAe,KAAK,GACrB,WAAW,eAAsB,EACjC,OAAO;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,cAAsB,GAAG,aAAa;AAAA,IACxC,CAAC,EACA,MAAM,cAAqB,MAAM,MAAM,EACvC,MAAM,aAAoB,KAAK,QAAQ,QAAQ,EAC/C,QAAQ,CAAC,eAAsB,aAAoB,iBAAwB,CAAC,EAC5E,OAAO,oCAA6C,UAAU,EAAE,EAChE,QAAQ,oCAAoC,EAC5C,MAAM,KAAK;AAEd,QAAI,iBAAiB;AACnB,qBAAe,aAAa,MAAM,mBAA0B,MAAM,eAAe;AAAA,IACnF;AAEA,QAAI,sBAAsB,QAAQ;AAChC,qBAAe,aAAa,MAAM,eAAsB,MAAM,oBAAoB;AAAA,IACpF,WAAW,oBAAoB,QAAQ;AACrC,qBAAe,aAAa,MAAM,eAAsB,UAAU,mBAAmB;AAAA,IACvF;AAEA,UAAM,OAAO,MAAM,aAAa,QAAQ;AAOxC,WAAO,KAAK,IAAI,CAAC,QAAQ;AACvB,YAAM,aAAa,OAAO,IAAI,gBAAgB,WAC1C,SAAS,IAAI,aAAa,EAAE,IAC5B,IAAI;AAER,YAAM,QAAQ,aAAa,OAAO;AAElC,aAAO;AAAA,QACL,UAAU,IAAI;AAAA,QACd,UAAU,IAAI;AAAA,QACd;AAAA,QACA,QAAQ,KAAK;AAAA,QACb,gBAAgB,IAAI,mBAAmB;AAAA,MACzC;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,MAAM,QAAwC;AAElD,UAAM,EAAE,6BAA6B,IAAI,MAAM,OAC7C,0DACF;AAEA,UAAM,6BAA6B,KAAK,IAAI;AAAA,MAC1C,YAAY,OAAO;AAAA,MACnB,UAAU,OAAO;AAAA,MACjB,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO;AAAA,MACvB,KAAK,OAAO;AAAA,IACd,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,OAAO,UAAoB,UAAkB,UAAiC;AAElF,UAAM,EAAE,4BAA4B,IAAI,MAAM,OAC5C,0DACF;AAEA,UAAM,4BAA4B,KAAK,IAAI;AAAA,MACzC,YAAY;AAAA,MACZ;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,SAA2C;AACzD,QAAI,QAAQ,WAAW,EAAG;AAE1B,UAAM,EAAE,4BAA4B,IAAI,MAAM,OAC5C,0DACF;AAEA,UAAM,WAAW,QAAQ,IAAI,CAAC,YAAY;AAAA,MACxC,YAAY,OAAO;AAAA,MACnB,UAAU,OAAO;AAAA,MACjB,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO;AAAA,MACvB,KAAK,OAAO;AAAA,IACd,EAAE;AAEF,UAAM,4BAA4B,KAAK,IAAI,QAAQ;AAAA,EACrD;AAAA,EAEA,MAAM,MAAM,UAAoB,UAAkB,gBAA+C;AAC/F,UAAM,2BACJ,OAAO,mBAAmB,YAAY,eAAe,KAAK,EAAE,SAAS,IAAI,eAAe,KAAK,IAAI;AACnG,QAAI,QAAQ,KAAK,GACd,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,QAAQ,EACzC,MAAM,aAAoB,KAAK,QAAQ;AAC1C,QAAI,6BAA6B,MAAM;AACrC,cAAQ,MAAM,MAAM,mBAA0B,KAAK,wBAAwB;AAAA,IAC7E;AACA,UAAM,MAAM,QAAQ;AAAA,EACtB;AACF;",
+  "sourcesContent": ["import { type Kysely, sql, type SqlBool } from 'kysely'\nimport type {\n  SearchStrategy,\n  SearchStrategyId,\n  SearchOptions,\n  SearchResult,\n  IndexableRecord,\n} from '../types'\nimport type { EntityId } from '@open-mercato/shared/modules/entities'\n\n/**\n * Configuration for TokenSearchStrategy.\n */\nexport type TokenStrategyConfig = {\n  /** Minimum number of query tokens that must match (0-1 ratio, default 0.5) */\n  minMatchRatio?: number\n  /** Default limit for search results */\n  defaultLimit?: number\n}\n\nfunction normalizeOrganizationIds(options: SearchOptions): string[] | null {\n  const single = typeof options.organizationId === 'string' ? options.organizationId.trim() : ''\n  if (single) return [single]\n  if (!Array.isArray(options.organizationIds)) return null\n  return Array.from(new Set(\n    options.organizationIds\n      .map((value) => (typeof value === 'string' ? value.trim() : ''))\n      .filter((value) => value.length > 0),\n  ))\n}\n\nconst CF_ALIAS_PREFIX = 'cf_'\nconst CF_CANONICAL_PREFIX = 'cf:'\n\n/**\n * Rewrites the query engine's aliased custom-field keys back to the spelling `search_tokens` is\n * meant to carry.\n *\n * The engine cannot label a column `cf:<key>` \u2014 `:` is not a valid SQL identifier \u2014 so it sanitizes\n * the alias down to `cf_<key>`. Core's token writer builds from `entity_indexes.doc`, which keeps\n * `cf:<key>`, and both writers replace a record's tokens by deleting only the `(entity_id, field)`\n * pairs their own document carries. Under two spellings neither deletes the other's custom-field\n * rows: every custom field is tokenized twice under names that carry the same hashes, while the\n * base-field rows the two documents share are alternately deleted and re-inserted on every write.\n *\n * `cf:` is the side to converge on because it is the side that is read \u2014 the query engine's search\n * predicate and every caller of `findEntityIdsBySearchTokens` ask for `cf:<key>`.\n *\n * Deliberately scoped to the rows this strategy writes rather than applied to\n * `IndexableRecord.fields` upstream: the same object is handed to the fulltext driver, and\n * Meilisearch rejects an attribute name containing `:`.\n *\n * The reversal assumes word-character keys. The engine's alias sanitizer is\n * `[^a-zA-Z0-9_] -> _` and a custom-field key is an unconstrained `z.string()`, so a key named\n * `order-ref` arrives as `cf_order_ref` and is rewritten to `cf:order_ref` \u2014 a name nothing reads,\n * which leaves that one field's double-write unfixed. Inverting the sanitizer would need the\n * field-definition key list, which this strategy has not got.\n */\nfunction normalizeCustomFieldKeys(\n  fields: Record<string, unknown> | null | undefined,\n): Record<string, unknown> | null | undefined {\n  if (!fields) return fields\n  const normalized: Record<string, unknown> = {}\n  for (const [key, value] of Object.entries(fields)) {\n    if (!key.startsWith(CF_ALIAS_PREFIX)) {\n      normalized[key] = value\n      continue\n    }\n    const canonical = `${CF_CANONICAL_PREFIX}${key.slice(CF_ALIAS_PREFIX.length)}`\n    // A document carrying both spellings meant the explicit one; the alias is the sanitizer's\n    // output for the same field.\n    if (canonical in fields) continue\n    normalized[canonical] = value\n  }\n  return normalized\n}\n\n/**\n * TokenSearchStrategy provides hash-based search using the existing search_tokens table.\n * This strategy is always available and serves as a fallback when other strategies fail.\n *\n * It tokenizes queries into hashes and matches against pre-indexed token hashes,\n * enabling search on encrypted fields without exposing plaintext to external services.\n */\nexport class TokenSearchStrategy implements SearchStrategy {\n  readonly id: SearchStrategyId = 'tokens'\n  readonly name = 'Token Search'\n  readonly priority = 10 // Lowest priority, always available as fallback\n\n  private readonly minMatchRatio: number\n  private readonly defaultLimit: number\n\n  constructor(\n    private readonly db: Kysely<any>,\n    config?: TokenStrategyConfig,\n  ) {\n    this.minMatchRatio = config?.minMatchRatio ?? 0.5\n    this.defaultLimit = config?.defaultLimit ?? 50\n  }\n\n  async isAvailable(): Promise<boolean> {\n    return true // Always available\n  }\n\n  async ensureReady(): Promise<void> {\n    // No initialization needed\n  }\n\n  async search(query: string, options: SearchOptions): Promise<SearchResult[]> {\n    const organizationIds = normalizeOrganizationIds(options)\n    if (organizationIds && organizationIds.length === 0) return []\n\n    // Dynamically import tokenization to avoid circular dependencies\n    const { tokenizeText } = await import('@open-mercato/shared/lib/search/tokenize')\n    const { resolveSearchConfig } = await import('@open-mercato/shared/lib/search/config')\n    const { listSearchTokenExcludedEntityTypes } = await import(\n      '@open-mercato/core/modules/query_index/lib/search-entity-policy'\n    )\n\n    const config = resolveSearchConfig()\n    if (!config.enabled) return []\n\n    // The rows themselves stay in `search_tokens` \u2014 list routes and the query engines' encrypted\n    // like/ilike rewrite depend on them \u2014 so the exclusion is enforced here, at read time.\n    const excludedEntityTypes = listSearchTokenExcludedEntityTypes()\n    const requestedEntityTypes = options.entityTypes?.length\n      ? options.entityTypes.filter((entityType) => !excludedEntityTypes.includes(entityType))\n      : undefined\n    if (options.entityTypes?.length && !requestedEntityTypes?.length) return []\n\n    const { hashes } = tokenizeText(query, config)\n    if (hashes.length === 0) return []\n\n    const minMatches = Math.max(1, Math.ceil(hashes.length * this.minMatchRatio))\n    const limit = options.limit ?? this.defaultLimit\n\n    let queryBuilder = this.db\n      .selectFrom('search_tokens' as any)\n      .select([\n        'entity_type' as any,\n        'entity_id' as any,\n        'organization_id' as any,\n        sql<string>`count(*)`.as('match_count'),\n      ])\n      .where('token_hash' as any, 'in', hashes)\n      .where('tenant_id' as any, '=', options.tenantId)\n      .groupBy(['entity_type' as any, 'entity_id' as any, 'organization_id' as any])\n      .having(sql<SqlBool>`count(distinct token_hash) >= ${minMatches}`)\n      .orderBy(sql`count(distinct token_hash) desc`)\n      .limit(limit)\n\n    if (organizationIds) {\n      queryBuilder = queryBuilder.where('organization_id' as any, 'in', organizationIds)\n    }\n\n    if (requestedEntityTypes?.length) {\n      queryBuilder = queryBuilder.where('entity_type' as any, 'in', requestedEntityTypes)\n    } else if (excludedEntityTypes.length) {\n      queryBuilder = queryBuilder.where('entity_type' as any, 'not in', excludedEntityTypes)\n    }\n\n    const rows = await queryBuilder.execute() as Array<{\n      entity_type: string\n      entity_id: string\n      organization_id: string | null\n      match_count: string | number\n    }>\n\n    return rows.map((row) => {\n      const matchCount = typeof row.match_count === 'string'\n        ? parseInt(row.match_count, 10)\n        : row.match_count\n      // Calculate score based on match ratio\n      const score = matchCount / hashes.length\n\n      return {\n        entityId: row.entity_type as EntityId,\n        recordId: row.entity_id,\n        score,\n        source: this.id,\n        organizationId: row.organization_id ?? null,\n      }\n    })\n  }\n\n  async index(record: IndexableRecord): Promise<void> {\n    // Dynamically import to avoid circular dependencies\n    const { replaceSearchTokensForRecord } = await import(\n      '@open-mercato/core/modules/query_index/lib/search-tokens'\n    )\n\n    await replaceSearchTokensForRecord(this.db, {\n      entityType: record.entityId,\n      recordId: record.recordId,\n      tenantId: record.tenantId,\n      organizationId: record.organizationId,\n      doc: normalizeCustomFieldKeys(record.fields),\n    })\n  }\n\n  async delete(entityId: EntityId, recordId: string, tenantId: string): Promise<void> {\n    // Dynamically import to avoid circular dependencies\n    const { deleteSearchTokensForRecord } = await import(\n      '@open-mercato/core/modules/query_index/lib/search-tokens'\n    )\n\n    await deleteSearchTokensForRecord(this.db, {\n      entityType: entityId,\n      recordId,\n      tenantId,\n    })\n  }\n\n  async bulkIndex(records: IndexableRecord[]): Promise<void> {\n    if (records.length === 0) return\n\n    const { replaceSearchTokensForBatch } = await import(\n      '@open-mercato/core/modules/query_index/lib/search-tokens'\n    )\n\n    const payloads = records.map((record) => ({\n      entityType: record.entityId,\n      recordId: record.recordId,\n      tenantId: record.tenantId,\n      organizationId: record.organizationId,\n      doc: normalizeCustomFieldKeys(record.fields) as Record<string, unknown>,\n    }))\n\n    await replaceSearchTokensForBatch(this.db, payloads)\n  }\n\n  async purge(entityId: EntityId, tenantId: string, organizationId?: string | null): Promise<void> {\n    const normalizedOrganizationId =\n      typeof organizationId === 'string' && organizationId.trim().length > 0 ? organizationId.trim() : null\n    let query = this.db\n      .deleteFrom('search_tokens' as any)\n      .where('entity_type' as any, '=', entityId)\n      .where('tenant_id' as any, '=', tenantId)\n    if (normalizedOrganizationId !== null) {\n      query = query.where('organization_id' as any, '=', normalizedOrganizationId)\n    }\n    await query.execute()\n  }\n}\n"],
+  "mappings": "AAAA,SAAsB,WAAyB;AAoB/C,SAAS,yBAAyB,SAAyC;AACzE,QAAM,SAAS,OAAO,QAAQ,mBAAmB,WAAW,QAAQ,eAAe,KAAK,IAAI;AAC5F,MAAI,OAAQ,QAAO,CAAC,MAAM;AAC1B,MAAI,CAAC,MAAM,QAAQ,QAAQ,eAAe,EAAG,QAAO;AACpD,SAAO,MAAM,KAAK,IAAI;AAAA,IACpB,QAAQ,gBACL,IAAI,CAAC,UAAW,OAAO,UAAU,WAAW,MAAM,KAAK,IAAI,EAAG,EAC9D,OAAO,CAAC,UAAU,MAAM,SAAS,CAAC;AAAA,EACvC,CAAC;AACH;AAEA,MAAM,kBAAkB;AACxB,MAAM,sBAAsB;AA0B5B,SAAS,yBACP,QAC4C;AAC5C,MAAI,CAAC,OAAQ,QAAO;AACpB,QAAM,aAAsC,CAAC;AAC7C,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,MAAM,GAAG;AACjD,QAAI,CAAC,IAAI,WAAW,eAAe,GAAG;AACpC,iBAAW,GAAG,IAAI;AAClB;AAAA,IACF;AACA,UAAM,YAAY,GAAG,mBAAmB,GAAG,IAAI,MAAM,gBAAgB,MAAM,CAAC;AAG5E,QAAI,aAAa,OAAQ;AACzB,eAAW,SAAS,IAAI;AAAA,EAC1B;AACA,SAAO;AACT;AASO,MAAM,oBAA8C;AAAA,EAQzD,YACmB,IACjB,QACA;AAFiB;AARnB,SAAS,KAAuB;AAChC,SAAS,OAAO;AAChB,SAAS,WAAW;AASlB,SAAK,gBAAgB,QAAQ,iBAAiB;AAC9C,SAAK,eAAe,QAAQ,gBAAgB;AAAA,EAC9C;AAAA,EAEA,MAAM,cAAgC;AACpC,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,cAA6B;AAAA,EAEnC;AAAA,EAEA,MAAM,OAAO,OAAe,SAAiD;AAC3E,UAAM,kBAAkB,yBAAyB,OAAO;AACxD,QAAI,mBAAmB,gBAAgB,WAAW,EAAG,QAAO,CAAC;AAG7D,UAAM,EAAE,aAAa,IAAI,MAAM,OAAO,0CAA0C;AAChF,UAAM,EAAE,oBAAoB,IAAI,MAAM,OAAO,wCAAwC;AACrF,UAAM,EAAE,mCAAmC,IAAI,MAAM,OACnD,iEACF;AAEA,UAAM,SAAS,oBAAoB;AACnC,QAAI,CAAC,OAAO,QAAS,QAAO,CAAC;AAI7B,UAAM,sBAAsB,mCAAmC;AAC/D,UAAM,uBAAuB,QAAQ,aAAa,SAC9C,QAAQ,YAAY,OAAO,CAAC,eAAe,CAAC,oBAAoB,SAAS,UAAU,CAAC,IACpF;AACJ,QAAI,QAAQ,aAAa,UAAU,CAAC,sBAAsB,OAAQ,QAAO,CAAC;AAE1E,UAAM,EAAE,OAAO,IAAI,aAAa,OAAO,MAAM;AAC7C,QAAI,OAAO,WAAW,EAAG,QAAO,CAAC;AAEjC,UAAM,aAAa,KAAK,IAAI,GAAG,KAAK,KAAK,OAAO,SAAS,KAAK,aAAa,CAAC;AAC5E,UAAM,QAAQ,QAAQ,SAAS,KAAK;AAEpC,QAAI,eAAe,KAAK,GACrB,WAAW,eAAsB,EACjC,OAAO;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,MACA,cAAsB,GAAG,aAAa;AAAA,IACxC,CAAC,EACA,MAAM,cAAqB,MAAM,MAAM,EACvC,MAAM,aAAoB,KAAK,QAAQ,QAAQ,EAC/C,QAAQ,CAAC,eAAsB,aAAoB,iBAAwB,CAAC,EAC5E,OAAO,oCAA6C,UAAU,EAAE,EAChE,QAAQ,oCAAoC,EAC5C,MAAM,KAAK;AAEd,QAAI,iBAAiB;AACnB,qBAAe,aAAa,MAAM,mBAA0B,MAAM,eAAe;AAAA,IACnF;AAEA,QAAI,sBAAsB,QAAQ;AAChC,qBAAe,aAAa,MAAM,eAAsB,MAAM,oBAAoB;AAAA,IACpF,WAAW,oBAAoB,QAAQ;AACrC,qBAAe,aAAa,MAAM,eAAsB,UAAU,mBAAmB;AAAA,IACvF;AAEA,UAAM,OAAO,MAAM,aAAa,QAAQ;AAOxC,WAAO,KAAK,IAAI,CAAC,QAAQ;AACvB,YAAM,aAAa,OAAO,IAAI,gBAAgB,WAC1C,SAAS,IAAI,aAAa,EAAE,IAC5B,IAAI;AAER,YAAM,QAAQ,aAAa,OAAO;AAElC,aAAO;AAAA,QACL,UAAU,IAAI;AAAA,QACd,UAAU,IAAI;AAAA,QACd;AAAA,QACA,QAAQ,KAAK;AAAA,QACb,gBAAgB,IAAI,mBAAmB;AAAA,MACzC;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,MAAM,QAAwC;AAElD,UAAM,EAAE,6BAA6B,IAAI,MAAM,OAC7C,0DACF;AAEA,UAAM,6BAA6B,KAAK,IAAI;AAAA,MAC1C,YAAY,OAAO;AAAA,MACnB,UAAU,OAAO;AAAA,MACjB,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO;AAAA,MACvB,KAAK,yBAAyB,OAAO,MAAM;AAAA,IAC7C,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,OAAO,UAAoB,UAAkB,UAAiC;AAElF,UAAM,EAAE,4BAA4B,IAAI,MAAM,OAC5C,0DACF;AAEA,UAAM,4BAA4B,KAAK,IAAI;AAAA,MACzC,YAAY;AAAA,MACZ;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,UAAU,SAA2C;AACzD,QAAI,QAAQ,WAAW,EAAG;AAE1B,UAAM,EAAE,4BAA4B,IAAI,MAAM,OAC5C,0DACF;AAEA,UAAM,WAAW,QAAQ,IAAI,CAAC,YAAY;AAAA,MACxC,YAAY,OAAO;AAAA,MACnB,UAAU,OAAO;AAAA,MACjB,UAAU,OAAO;AAAA,MACjB,gBAAgB,OAAO;AAAA,MACvB,KAAK,yBAAyB,OAAO,MAAM;AAAA,IAC7C,EAAE;AAEF,UAAM,4BAA4B,KAAK,IAAI,QAAQ;AAAA,EACrD;AAAA,EAEA,MAAM,MAAM,UAAoB,UAAkB,gBAA+C;AAC/F,UAAM,2BACJ,OAAO,mBAAmB,YAAY,eAAe,KAAK,EAAE,SAAS,IAAI,eAAe,KAAK,IAAI;AACnG,QAAI,QAAQ,KAAK,GACd,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,QAAQ,EACzC,MAAM,aAAoB,KAAK,QAAQ;AAC1C,QAAI,6BAA6B,MAAM;AACrC,cAAQ,MAAM,MAAM,mBAA0B,KAAK,wBAAwB;AAAA,IAC7E;AACA,UAAM,MAAM,QAAQ;AAAA,EACtB;AACF;",
   "names": []
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@open-mercato/search",
-  "version": "0.7.1-develop.7113.1.9d83dca10c",
+  "version": "0.7.1-develop.7121.1.734c263bda",
   "license": "MIT",
   "type": "module",
   "main": "./dist/index.js",
@@ -127,9 +127,9 @@
     "zod": "^4.4.3"
   },
   "peerDependencies": {
-    "@open-mercato/core": "0.7.1-develop.7113.1.9d83dca10c",
-    "@open-mercato/queue": "0.7.1-develop.7113.1.9d83dca10c",
-    "@open-mercato/shared": "0.7.1-develop.7113.1.9d83dca10c"
+    "@open-mercato/core": "0.7.1-develop.7121.1.734c263bda",
+    "@open-mercato/queue": "0.7.1-develop.7121.1.734c263bda",
+    "@open-mercato/shared": "0.7.1-develop.7121.1.734c263bda"
   },
   "devDependencies": {
     "@types/jest": "^30.0.0",

package/src/__tests__/token-strategy-custom-field-keys.test.ts

@@ -0,0 +1,278 @@
+import type { Kysely } from 'kysely'
+import { replaceSearchTokensForRecord } from '@open-mercato/core/modules/query_index/lib/search-tokens'
+import { TokenSearchStrategy } from '../strategies/token.strategy'
+import type { IndexableRecord } from '../types'
+
+/**
+ * The search module reaches `search_tokens` through the query engine, which cannot label a column
+ * `cf:<key>` and returns the sanitized alias `cf_<key>` instead. Core's own writer builds from
+ * `entity_indexes.doc` and keeps `cf:<key>`. Both replace a record's tokens by deleting only the
+ * `(entity_id, field)` pairs their own document carries, so under two spellings neither deletes the
+ * other's custom-field rows.
+ *
+ * These tests drive the real core writer against an in-memory `search_tokens` so the two writers
+ * meet in one table, which is the only place the divergence is observable.
+ */
+
+type StoredRow = {
+  id: number
+  entity_type: string
+  entity_id: string
+  organization_id: string | null
+  tenant_id: string | null
+  field: string
+  token_hash: string
+  token: string | null
+}
+
+type RawBuilderLike = { toOperationNode: () => { sqlFragments: string[]; parameters: Array<{ value?: unknown }> } }
+type Matcher = (row: StoredRow) => boolean
+
+const columnOf = (row: StoredRow, column: string): unknown => (row as unknown as Record<string, unknown>)[column]
+
+function createSearchTokenStore() {
+  const rows: StoredRow[] = []
+  let nextId = 1
+
+  const expressionBuilder = () => {
+    const eb: any = (column: string, operator: string, value: unknown): Matcher => {
+      if (operator !== '=') throw new Error(`[internal] unsupported eb operator: ${operator}`)
+      return (row) => String(columnOf(row, column)) === String(value)
+    }
+    eb.and = (matchers: Matcher[]): Matcher => (row) => matchers.every((matches) => matches(row))
+    eb.or = (matchers: Matcher[]): Matcher => (row) => matchers.some((matches) => matches(row))
+    return eb
+  }
+
+  const buildPredicate = (args: unknown[]): Matcher => {
+    if (args.length === 1) {
+      const arg = args[0]
+      if (typeof arg === 'function') return (arg as (eb: unknown) => Matcher)(expressionBuilder())
+      const node = (arg as RawBuilderLike).toOperationNode()
+      const column = String(node.sqlFragments[0]).trim().split(/\s+/)[0]
+      const expected = node.parameters[0]?.value ?? null
+      return (row) => (columnOf(row, column) ?? null) === expected
+    }
+    const [column, operator, value] = args as [string, string, unknown]
+    if (operator === '=') return (row) => String(columnOf(row, column)) === String(value)
+    if (operator === 'in') {
+      const allowed = new Set((value as unknown[]).map(String))
+      return (row) => allowed.has(String(columnOf(row, column)))
+    }
+    throw new Error(`[internal] unsupported where operator: ${operator}`)
+  }
+
+  const selectChain = () => {
+    const predicates: Matcher[] = []
+    let columns: string[] | null = null
+    let groupedBy: string | null = null
+    let limit: number | null = null
+    const chain: any = {
+      // The batch path's count probe selects an aggregate builder inside a column array and pairs
+      // it with groupBy(); the per-record probe passes the aggregate on its own. Neither carries a
+      // parseable column name, so both branches key off what else was supplied.
+      select: (selection: unknown) => {
+        columns = Array.isArray(selection) ? selection.filter((col) => typeof col === 'string').map(String) : null
+        return chain
+      },
+      where: (...args: unknown[]) => {
+        predicates.push(buildPredicate(args))
+        return chain
+      },
+      groupBy: (column: unknown) => {
+        groupedBy = String(column)
+        return chain
+      },
+      limit: (count: number) => {
+        limit = count
+        return chain
+      },
+      execute: async () => {
+        const matched = rows.filter((row) => predicates.every((matches) => matches(row)))
+        if (groupedBy) {
+          const counts = new Map<string, number>()
+          for (const row of matched) {
+            const groupKey = String(columnOf(row, groupedBy))
+            counts.set(groupKey, (counts.get(groupKey) ?? 0) + 1)
+          }
+          return Array.from(counts.entries()).map(([groupKey, count]) => ({
+            [groupedBy as string]: groupKey,
+            token_count: String(count),
+          }))
+        }
+        if (!columns) return [{ token_count: String(matched.length) }]
+        const limited = limit === null ? matched : matched.slice(0, limit)
+        return limited.map((row) => Object.fromEntries(columns!.map((column) => [column, columnOf(row, column)])))
+      },
+    }
+    return chain
+  }
+
+  const deleteChain = () => {
+    const predicates: Matcher[] = []
+    const chain: any = {
+      where: (...args: unknown[]) => {
+        predicates.push(buildPredicate(args))
+        return chain
+      },
+      execute: async () => {
+        const kept = rows.filter((row) => !predicates.every((matches) => matches(row)))
+        rows.length = 0
+        rows.push(...kept)
+        return []
+      },
+    }
+    return chain
+  }
+
+  const insertChain = () => {
+    const chain: any = {
+      values: (values: any[]) => {
+        for (const value of values) {
+          rows.push({
+            id: nextId++,
+            entity_type: String(value.entity_type),
+            entity_id: String(value.entity_id),
+            organization_id: value.organization_id ?? null,
+            tenant_id: value.tenant_id ?? null,
+            field: String(value.field),
+            token_hash: String(value.token_hash),
+            token: value.token ?? null,
+          })
+        }
+        return chain
+      },
+      execute: async () => [],
+    }
+    return chain
+  }
+
+  const executor = {
+    selectFrom: selectChain,
+    deleteFrom: deleteChain,
+    insertInto: insertChain,
+  }
+
+  const db = {
+    ...executor,
+    transaction: () => ({
+      execute: async (callback: (trx: unknown) => Promise<void>) => callback(executor),
+    }),
+  } as unknown as Kysely<any>
+
+  return {
+    db,
+    rows,
+    fields: () => Array.from(new Set(rows.map((row) => row.field))).sort(),
+    rowIds: () => rows.map((row) => row.id).sort((a, b) => a - b),
+  }
+}
+
+const ENTITY_ID = 'sales:sales_order'
+const SCOPE = { tenantId: 'tenant-1', organizationId: 'org-1' }
+
+const indexableRecord = (fields: Record<string, unknown>): IndexableRecord => ({
+  entityId: ENTITY_ID,
+  recordId: 'order-1',
+  tenantId: SCOPE.tenantId,
+  organizationId: SCOPE.organizationId,
+  fields,
+})
+
+// What the query engine hands back: base columns plus custom fields under the sanitized alias.
+const QUERY_ENGINE_RECORD = { id: 'order-1', title: 'alpha widget', cf_priority: 'urgent' }
+// What core stores in `entity_indexes.doc` for the same record.
+const INDEX_DOCUMENT = { id: 'order-1', title: 'alpha widget', 'cf:priority': 'urgent' }
+
+const writeCoreTokens = (db: Kysely<any>) =>
+  replaceSearchTokensForRecord(db, {
+    entityType: ENTITY_ID,
+    recordId: 'order-1',
+    tenantId: SCOPE.tenantId,
+    organizationId: SCOPE.organizationId,
+    doc: INDEX_DOCUMENT,
+  })
+
+describe('TokenSearchStrategy writes custom fields under the spelling core uses', () => {
+  const previousEnv = {
+    enabled: process.env.OM_SEARCH_ENABLED,
+    partials: process.env.OM_SEARCH_ENABLE_PARTIAL,
+  }
+
+  beforeAll(() => {
+    process.env.OM_SEARCH_ENABLED = 'true'
+    // Partials off keeps the token set small; the field naming is independent of token fan-out.
+    process.env.OM_SEARCH_ENABLE_PARTIAL = 'false'
+  })
+
+  afterAll(() => {
+    if (previousEnv.enabled === undefined) delete process.env.OM_SEARCH_ENABLED
+    else process.env.OM_SEARCH_ENABLED = previousEnv.enabled
+    if (previousEnv.partials === undefined) delete process.env.OM_SEARCH_ENABLE_PARTIAL
+    else process.env.OM_SEARCH_ENABLE_PARTIAL = previousEnv.partials
+  })
+
+  it('stores the aliased custom-field key as cf:<key>, leaving base columns alone', async () => {
+    const store = createSearchTokenStore()
+    const strategy = new TokenSearchStrategy(store.db)
+
+    await strategy.index(indexableRecord(QUERY_ENGINE_RECORD))
+
+    expect(store.rows.length).toBeGreaterThan(0)
+    expect(store.fields()).toEqual(['cf:priority', 'title'])
+  })
+
+  it('normalizes the same way on the batch path', async () => {
+    const store = createSearchTokenStore()
+    const strategy = new TokenSearchStrategy(store.db)
+
+    await strategy.bulkIndex([indexableRecord(QUERY_ENGINE_RECORD)])
+
+    expect(store.rows.length).toBeGreaterThan(0)
+    expect(store.fields()).toEqual(['cf:priority', 'title'])
+  })
+
+  it('keeps an explicit cf: value when a document carries both spellings of one field', async () => {
+    const store = createSearchTokenStore()
+    const strategy = new TokenSearchStrategy(store.db)
+
+    await strategy.index(indexableRecord({ 'cf:priority': 'urgent', cf_priority: 'stale' }))
+
+    expect(store.fields()).toEqual(['cf:priority'])
+    const stored = new Set(store.rows.map((row) => row.token_hash))
+    const urgentOnly = createSearchTokenStore()
+    await new TokenSearchStrategy(urgentOnly.db).index(indexableRecord({ 'cf:priority': 'urgent' }))
+    expect(stored).toEqual(new Set(urgentOnly.rows.map((row) => row.token_hash)))
+  })
+
+  it('indexes a record once when both writers run over it, rather than under two field names', async () => {
+    const store = createSearchTokenStore()
+    const strategy = new TokenSearchStrategy(store.db)
+
+    await writeCoreTokens(store.db)
+    const afterCore = store.rowIds()
+    expect(afterCore.length).toBeGreaterThan(0)
+
+    await strategy.index(indexableRecord(QUERY_ENGINE_RECORD))
+
+    expect(store.fields()).toEqual(['cf:priority', 'title'])
+    // The second writer found exactly the rows it wanted to write and left them in place: two
+    // consecutive indexes of an unchanged record write nothing on either path.
+    expect(store.rowIds()).toEqual(afterCore)
+  })
+
+  it('leaves the record alone when the two writers alternate over it', async () => {
+    const store = createSearchTokenStore()
+    const strategy = new TokenSearchStrategy(store.db)
+
+    await strategy.index(indexableRecord(QUERY_ENGINE_RECORD))
+    const afterFirstWrite = store.rowIds()
+    expect(afterFirstWrite.length).toBeGreaterThan(0)
+
+    await writeCoreTokens(store.db)
+    await strategy.index(indexableRecord(QUERY_ENGINE_RECORD))
+    await writeCoreTokens(store.db)
+
+    expect(store.rowIds()).toEqual(afterFirstWrite)
+  })
+})

package/src/strategies/token.strategy.ts

@@ -29,6 +29,52 @@ function normalizeOrganizationIds(options: SearchOptions): string[] | null {
   ))
 }
 
+const CF_ALIAS_PREFIX = 'cf_'
+const CF_CANONICAL_PREFIX = 'cf:'
+
+/**
+ * Rewrites the query engine's aliased custom-field keys back to the spelling `search_tokens` is
+ * meant to carry.
+ *
+ * The engine cannot label a column `cf:<key>` — `:` is not a valid SQL identifier — so it sanitizes
+ * the alias down to `cf_<key>`. Core's token writer builds from `entity_indexes.doc`, which keeps
+ * `cf:<key>`, and both writers replace a record's tokens by deleting only the `(entity_id, field)`
+ * pairs their own document carries. Under two spellings neither deletes the other's custom-field
+ * rows: every custom field is tokenized twice under names that carry the same hashes, while the
+ * base-field rows the two documents share are alternately deleted and re-inserted on every write.
+ *
+ * `cf:` is the side to converge on because it is the side that is read — the query engine's search
+ * predicate and every caller of `findEntityIdsBySearchTokens` ask for `cf:<key>`.
+ *
+ * Deliberately scoped to the rows this strategy writes rather than applied to
+ * `IndexableRecord.fields` upstream: the same object is handed to the fulltext driver, and
+ * Meilisearch rejects an attribute name containing `:`.
+ *
+ * The reversal assumes word-character keys. The engine's alias sanitizer is
+ * `[^a-zA-Z0-9_] -> _` and a custom-field key is an unconstrained `z.string()`, so a key named
+ * `order-ref` arrives as `cf_order_ref` and is rewritten to `cf:order_ref` — a name nothing reads,
+ * which leaves that one field's double-write unfixed. Inverting the sanitizer would need the
+ * field-definition key list, which this strategy has not got.
+ */
+function normalizeCustomFieldKeys(
+  fields: Record<string, unknown> | null | undefined,
+): Record<string, unknown> | null | undefined {
+  if (!fields) return fields
+  const normalized: Record<string, unknown> = {}
+  for (const [key, value] of Object.entries(fields)) {
+    if (!key.startsWith(CF_ALIAS_PREFIX)) {
+      normalized[key] = value
+      continue
+    }
+    const canonical = `${CF_CANONICAL_PREFIX}${key.slice(CF_ALIAS_PREFIX.length)}`
+    // A document carrying both spellings meant the explicit one; the alias is the sanitizer's
+    // output for the same field.
+    if (canonical in fields) continue
+    normalized[canonical] = value
+  }
+  return normalized
+}
+
 /**
  * TokenSearchStrategy provides hash-based search using the existing search_tokens table.
  * This strategy is always available and serves as a fallback when other strategies fail.
@@ -148,7 +194,7 @@ export class TokenSearchStrategy implements SearchStrategy {
       recordId: record.recordId,
       tenantId: record.tenantId,
       organizationId: record.organizationId,
-      doc: record.fields,
+      doc: normalizeCustomFieldKeys(record.fields),
     })
   }
 
@@ -177,7 +223,7 @@ export class TokenSearchStrategy implements SearchStrategy {
       recordId: record.recordId,
       tenantId: record.tenantId,
       organizationId: record.organizationId,
-      doc: record.fields as Record<string, unknown>,
+      doc: normalizeCustomFieldKeys(record.fields) as Record<string, unknown>,
     }))
 
     await replaceSearchTokensForBatch(this.db, payloads)