@open-mercato/core 0.6.8-develop.7012.1.1dbd6f5fbd → 0.6.8-develop.7013.1.48f7adca80

package/dist/modules/query_index/lib/search-tokens.js

@@ -157,6 +157,34 @@ async function deleteSearchTokensForRecord(db, params, options) {
   const executor = options?.trx ?? db;
   await executor.deleteFrom("search_tokens").where("entity_type", "=", params.entityType).where("entity_id", "=", String(params.recordId)).where(sql`organization_id is not distinct from ${organizationId}`).where(sql`tenant_id is not distinct from ${tenantId}`).execute();
 }
+const SIGNATURE_SEPARATOR = String.fromCharCode(0);
+function tokenSignature(row) {
+  return [
+    String(row.field ?? ""),
+    String(row.token_hash ?? ""),
+    row.token == null ? "" : String(row.token)
+  ].join(SIGNATURE_SEPARATOR);
+}
+function tallyEquals(a, b) {
+  const left = a ?? /* @__PURE__ */ new Map();
+  const right = b ?? /* @__PURE__ */ new Map();
+  if (left.size !== right.size) return false;
+  for (const [key, count] of left.entries()) {
+    if (right.get(key) !== count) return false;
+  }
+  return true;
+}
+function tallyTokenRows(rows, keyOf) {
+  const tallies = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    const key = keyOf(row);
+    const tally = tallies.get(key) ?? /* @__PURE__ */ new Map();
+    const signature = tokenSignature(row);
+    tally.set(signature, (tally.get(signature) ?? 0) + 1);
+    tallies.set(key, tally);
+  }
+  return tallies;
+}
 async function replaceSearchTokensForBatch(db, payloads) {
   if (!payloads.length) return;
   const config = resolveSearchConfig();
@@ -179,12 +207,60 @@ async function replaceSearchTokensForBatch(db, payloads) {
     bucket.ids.add(String(payload.recordId));
     scopeBuckets.set(key, bucket);
   }
+  const recordKeyOf = (row) => `${scopeKey(row.organization_id ?? null, row.tenant_id ?? null)}|${String(row.entity_id)}`;
+  const builtTally = tallyTokenRows(rows, recordKeyOf);
+  const changedIdsByBucket = /* @__PURE__ */ new Map();
+  for (const [key, bucket] of scopeBuckets.entries()) {
+    const ids = Array.from(bucket.ids);
+    const builtCountById = /* @__PURE__ */ new Map();
+    for (const id of ids) {
+      let total = 0;
+      const tally = builtTally.get(`${key}|${id}`);
+      if (tally) for (const count of tally.values()) total += count;
+      builtCountById.set(id, total);
+    }
+    const storedCounts = await db.selectFrom("search_tokens").select(["entity_id", sql`count(*)`.as("token_count")]).where("entity_type", "=", payloads[0].entityType).where(sql`organization_id is not distinct from ${bucket.organizationId}`).where(sql`tenant_id is not distinct from ${bucket.tenantId}`).where("entity_id", "in", ids).groupBy("entity_id").execute();
+    const storedCountById = /* @__PURE__ */ new Map();
+    for (const row of storedCounts) {
+      storedCountById.set(String(row.entity_id), Number(row.token_count));
+    }
+    const changed = /* @__PURE__ */ new Set();
+    const contentCandidates = ids.filter((id) => {
+      const builtCount = builtCountById.get(id) ?? 0;
+      if ((storedCountById.get(id) ?? 0) !== builtCount) {
+        changed.add(id);
+        return false;
+      }
+      return builtCount > 0;
+    });
+    if (contentCandidates.length) {
+      const rowBudget = contentCandidates.reduce((sum, id) => sum + (builtCountById.get(id) ?? 0), 0);
+      const stored = await db.selectFrom("search_tokens").select(["entity_id", "field", "token_hash", "token"]).where("entity_type", "=", payloads[0].entityType).where(sql`organization_id is not distinct from ${bucket.organizationId}`).where(sql`tenant_id is not distinct from ${bucket.tenantId}`).where("entity_id", "in", contentCandidates).limit(rowBudget).execute();
+      const storedTally = tallyTokenRows(stored, (row) => String(row.entity_id));
+      for (const id of contentCandidates) {
+        if (!tallyEquals(builtTally.get(`${key}|${id}`), storedTally.get(id))) changed.add(id);
+      }
+    }
+    changedIdsByBucket.set(key, changed);
+  }
+  const changedRecordKeys = /* @__PURE__ */ new Set();
+  for (const [key, changed] of changedIdsByBucket.entries()) {
+    for (const id of changed) changedRecordKeys.add(`${key}|${id}`);
+  }
+  debug("batch.skip", {
+    entityType: payloads[0].entityType,
+    recordCount: payloads.length,
+    changedCount: changedRecordKeys.size
+  });
+  if (!changedRecordKeys.size) return;
   await db.transaction().execute(async (trx) => {
-    for (const [, bucket] of scopeBuckets.entries()) {
-      const deleteQuery = trx.deleteFrom("search_tokens").where("entity_type", "=", payloads[0].entityType).where(sql`organization_id is not distinct from ${bucket.organizationId}`).where(sql`tenant_id is not distinct from ${bucket.tenantId}`).where("entity_id", "in", Array.from(bucket.ids));
+    for (const [key, bucket] of scopeBuckets.entries()) {
+      const changed = changedIdsByBucket.get(key);
+      if (!changed?.size) continue;
+      const deleteQuery = trx.deleteFrom("search_tokens").where("entity_type", "=", payloads[0].entityType).where(sql`organization_id is not distinct from ${bucket.organizationId}`).where(sql`tenant_id is not distinct from ${bucket.tenantId}`).where("entity_id", "in", Array.from(changed));
       await deleteQuery.execute();
     }
-    const payloadWithTimestamps = rows.map((row) => ({ ...row, created_at: sql`now()` }));
+    const payloadWithTimestamps = rows.filter((row) => changedRecordKeys.has(recordKeyOf(row))).map((row) => ({ ...row, created_at: sql`now()` }));
     for (const batch of chunk(payloadWithTimestamps, INSERT_BATCH_SIZE)) {
       await trx.insertInto("search_tokens").values(batch).execute();
     }

package/dist/modules/query_index/lib/search-tokens.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../../src/modules/query_index/lib/search-tokens.ts"],
-  "sourcesContent": ["import { type Kysely, type Transaction, sql } from 'kysely'\nimport {\n  isSearchFieldBlocklisted,\n  resolveSearchConfig,\n  resolveSearchTokenLimits,\n  type SearchConfig,\n} from '@open-mercato/shared/lib/search/config'\nimport { tokenizeText } from '@open-mercato/shared/lib/search/tokenize'\nimport { parseBooleanToken } from '@open-mercato/shared/lib/boolean'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst logger = createLogger('query_index').child({ component: 'search-tokens' })\n\nconst INSERT_BATCH_SIZE = 500\n\nfunction chunk<T>(items: T[], size: number): T[][] {\n  if (size <= 0) return [items]\n  const out: T[][] = []\n  for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size))\n  return out\n}\n\nexport type SearchTokenRow = {\n  entity_type: string\n  entity_id: string\n  organization_id: string | null\n  tenant_id: string | null\n  field: string\n  token_hash: string\n  token?: string | null\n}\n\ntype BuildTokenOptions = {\n  entityType: string\n  recordId: string\n  organizationId?: string | null\n  tenantId?: string | null\n  doc?: Record<string, unknown> | null\n  config?: SearchConfig\n}\n\nconst DEFAULT_SCOPE = { organizationId: null, tenantId: null }\ntype EntityFieldPair = [string, string]\ntype SearchTokenExecutor = Kysely<any> | Transaction<any>\n\nexport const isSearchDebugEnabled = (): boolean => {\n  return parseBooleanToken(process.env.OM_SEARCH_DEBUG ?? '') === true\n}\n\nconst debug = (event: string, payload: Record<string, unknown>) => {\n  if (!isSearchDebugEnabled()) return\n  try {\n    logger.debug('Search token event', { event, payload })\n  } catch {\n    // ignore\n  }\n}\n\nfunction collectTextValues(value: unknown): string[] {\n  if (typeof value === 'string') return [value]\n  if (Array.isArray(value)) {\n    const out: string[] = []\n    for (const entry of value) {\n      if (typeof entry === 'string') out.push(entry)\n    }\n    return out\n  }\n  return []\n}\n\nfunction shouldIndexField(\n  field: string,\n  value: unknown,\n  config: SearchConfig,\n  entityType: string | null,\n): boolean {\n  if (typeof value !== 'string' && !Array.isArray(value)) return false\n  const lower = field.toLowerCase()\n  if (lower === 'id' || lower.endsWith('_id') || lower.endsWith('.id')) return false\n  if (lower.endsWith('_at')) return false\n  if (['created_at', 'updated_at', 'deleted_at', 'tenant_id', 'organization_id'].includes(lower)) return false\n  if (isSearchFieldBlocklisted(field, entityType, config)) return false\n  return collectTextValues(value).some((text) => text.length > 0)\n}\n\nexport function buildSearchTokenRows(params: BuildTokenOptions): SearchTokenRow[] {\n  const config = params.config ?? resolveSearchConfig()\n  if (!config.enabled) return []\n  if (!params.doc) return []\n  const tokens: SearchTokenRow[] = []\n  const capturePairs = isSearchDebugEnabled() && params.entityType === 'customers:customer_deal'\n  const debugPairs: Array<{ field: string; hash: string }> = []\n  const scope = {\n    organizationId: params.organizationId ?? DEFAULT_SCOPE.organizationId,\n    tenantId: params.tenantId ?? DEFAULT_SCOPE.tenantId,\n  }\n  const limits = resolveSearchTokenLimits(config)\n  const recordLimit = limits.maxTokensPerRecord > 0 ? limits.maxTokensPerRecord : Number.POSITIVE_INFINITY\n  const fieldLimit = limits.maxTokensPerField > 0 ? limits.maxTokensPerField : Number.POSITIVE_INFINITY\n\n  for (const [field, rawValue] of Object.entries(params.doc)) {\n    if (tokens.length >= recordLimit) break\n    if (!shouldIndexField(field, rawValue, config, params.entityType)) continue\n    const values = collectTextValues(rawValue)\n    const seen = new Set<string>()\n    let fieldTokenCount = 0\n    for (const text of values) {\n      if (tokens.length >= recordLimit || fieldTokenCount >= fieldLimit) break\n      const remainingLimit = Math.min(recordLimit - tokens.length, fieldLimit - fieldTokenCount)\n      const candidateLimit = fieldTokenCount + remainingLimit\n      const tokenConfig = Number.isFinite(candidateLimit)\n        ? { ...config, maxTokensPerField: candidateLimit }\n        : config\n      const { tokens: textTokens, hashes } = tokenizeText(text, tokenConfig)\n      for (let i = 0; i < textTokens.length; i += 1) {\n        if (tokens.length >= recordLimit || fieldTokenCount >= fieldLimit) break\n        const token = textTokens[i]\n        const hash = hashes[i]\n        const dedupeKey = `${field}|${hash}`\n        if (seen.has(dedupeKey)) continue\n        seen.add(dedupeKey)\n        fieldTokenCount += 1\n        debug('token.generated', { entityType: params.entityType, recordId: params.recordId, field, hash })\n        tokens.push({\n          entity_type: params.entityType,\n          entity_id: String(params.recordId),\n          organization_id: scope.organizationId,\n          tenant_id: scope.tenantId,\n          field,\n          token_hash: hash,\n          token: config.storeRawTokens ? token : null,\n        })\n        if (capturePairs) {\n          debugPairs.push({ field, hash })\n        }\n      }\n    }\n  }\n  if (capturePairs) {\n    debug('deal.tokens', {\n      entityType: params.entityType,\n      recordId: params.recordId,\n      tokenCount: debugPairs.length,\n      tokens: debugPairs,\n    })\n  }\n  debug('doc.completed', { entityType: params.entityType, recordId: params.recordId, tokenCount: tokens.length })\n\n  return tokens\n}\n\nfunction buildFieldPairs(recordId: string, doc?: Record<string, unknown> | null): EntityFieldPair[] {\n  if (!doc) return []\n  const pairs: EntityFieldPair[] = []\n  const dedupe = new Set<string>()\n  for (const field of Object.keys(doc)) {\n    const key = `${recordId}|${field}`\n    if (dedupe.has(key)) continue\n    dedupe.add(key)\n    pairs.push([recordId, field])\n  }\n  return pairs\n}\n\nexport async function replaceSearchTokensForRecord(\n  db: Kysely<any>,\n  params: BuildTokenOptions,\n  options?: { trx?: SearchTokenExecutor },\n): Promise<void> {\n  const rows = buildSearchTokenRows(params)\n  const config = params.config ?? resolveSearchConfig()\n  if (!config.enabled) return\n  const organizationId = params.organizationId ?? null\n  const tenantId = params.tenantId ?? null\n  const fieldPairs = buildFieldPairs(String(params.recordId), params.doc)\n\n  const writeTokens = async (executor: SearchTokenExecutor): Promise<void> => {\n    let deleteQuery = executor\n      .deleteFrom('search_tokens' as any)\n      .where('entity_type' as any, '=', params.entityType)\n      .where(sql<boolean>`organization_id is not distinct from ${organizationId}`)\n      .where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)\n    if (fieldPairs.length) {\n      deleteQuery = deleteQuery.where((eb: any) => eb.or(\n        fieldPairs.map(([rid, field]) => eb.and([\n          eb('entity_id' as any, '=', rid),\n          eb('field' as any, '=', field),\n        ])),\n      ))\n    } else {\n      deleteQuery = deleteQuery.where('entity_id' as any, '=', String(params.recordId))\n    }\n    await deleteQuery.execute()\n    if (!rows.length) return\n    const payloads = rows.map((row) => ({ ...row, created_at: sql`now()` }))\n    for (const batch of chunk(payloads, INSERT_BATCH_SIZE)) {\n      await executor.insertInto('search_tokens' as any).values(batch as any).execute()\n    }\n  }\n\n  if (options?.trx) {\n    await writeTokens(options.trx)\n    return\n  }\n\n  await db.transaction().execute(writeTokens)\n}\n\nexport async function deleteSearchTokensForRecord(\n  db: Kysely<any>,\n  params: { entityType: string; recordId: string; organizationId?: string | null; tenantId?: string | null },\n  options?: { trx?: SearchTokenExecutor },\n): Promise<void> {\n  const organizationId = params.organizationId ?? null\n  const tenantId = params.tenantId ?? null\n  const executor = options?.trx ?? db\n  await executor\n    .deleteFrom('search_tokens' as any)\n    .where('entity_type' as any, '=', params.entityType)\n    .where('entity_id' as any, '=', String(params.recordId))\n    .where(sql<boolean>`organization_id is not distinct from ${organizationId}`)\n    .where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)\n    .execute()\n}\n\nexport async function replaceSearchTokensForBatch(\n  db: Kysely<any>,\n  payloads: Array<BuildTokenOptions & { doc: Record<string, unknown> }>\n): Promise<void> {\n  if (!payloads.length) return\n  const config = resolveSearchConfig()\n  if (!config.enabled) return\n\n  const rows = payloads.flatMap((payload) => buildSearchTokenRows({ ...payload, config }))\n  if (!rows.length) {\n    const entityType = payloads[0]?.entityType\n    if (!entityType) return\n    const ids = payloads.map((p) => String(p.recordId))\n    await db\n      .deleteFrom('search_tokens' as any)\n      .where('entity_type' as any, '=', entityType)\n      .where('entity_id' as any, 'in', ids)\n      .execute()\n    return\n  }\n\n  const scopeKey = (org: string | null, tenant: string | null) => `${org ?? '__null__'}|${tenant ?? '__null__'}`\n  const scopeBuckets = new Map<string, { organizationId: string | null; tenantId: string | null; ids: Set<string> }>()\n\n  for (const payload of payloads) {\n    const org = payload.organizationId ?? null\n    const tenant = payload.tenantId ?? null\n    const key = scopeKey(org, tenant)\n    const bucket = scopeBuckets.get(key) ?? { organizationId: org, tenantId: tenant, ids: new Set<string>() }\n    bucket.ids.add(String(payload.recordId))\n    scopeBuckets.set(key, bucket)\n  }\n\n  await db.transaction().execute(async (trx) => {\n    for (const [, bucket] of scopeBuckets.entries()) {\n      // Delete by entity_id: a batch replaces all of a record's tokens, and a per-field OR over the\n      // whole batch overflows the query compiler's call stack on large batches.\n      const deleteQuery = trx\n        .deleteFrom('search_tokens' as any)\n        .where('entity_type' as any, '=', payloads[0].entityType)\n        .where(sql<boolean>`organization_id is not distinct from ${bucket.organizationId}`)\n        .where(sql<boolean>`tenant_id is not distinct from ${bucket.tenantId}`)\n        .where('entity_id' as any, 'in', Array.from(bucket.ids))\n      await deleteQuery.execute()\n    }\n    const payloadWithTimestamps = rows.map((row) => ({ ...row, created_at: sql`now()` }))\n    for (const batch of chunk(payloadWithTimestamps, INSERT_BATCH_SIZE)) {\n      await trx.insertInto('search_tokens' as any).values(batch as any).execute()\n    }\n  })\n}\n"],
-  "mappings": "AAAA,SAAwC,WAAW;AACnD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AACP,SAAS,oBAAoB;AAC7B,SAAS,yBAAyB;AAClC,SAAS,oBAAoB;AAE7B,MAAM,SAAS,aAAa,aAAa,EAAE,MAAM,EAAE,WAAW,gBAAgB,CAAC;AAE/E,MAAM,oBAAoB;AAE1B,SAAS,MAAS,OAAY,MAAqB;AACjD,MAAI,QAAQ,EAAG,QAAO,CAAC,KAAK;AAC5B,QAAM,MAAa,CAAC;AACpB,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,KAAM,KAAI,KAAK,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC;AAC9E,SAAO;AACT;AAqBA,MAAM,gBAAgB,EAAE,gBAAgB,MAAM,UAAU,KAAK;AAItD,MAAM,uBAAuB,MAAe;AACjD,SAAO,kBAAkB,QAAQ,IAAI,mBAAmB,EAAE,MAAM;AAClE;AAEA,MAAM,QAAQ,CAAC,OAAe,YAAqC;AACjE,MAAI,CAAC,qBAAqB,EAAG;AAC7B,MAAI;AACF,WAAO,MAAM,sBAAsB,EAAE,OAAO,QAAQ,CAAC;AAAA,EACvD,QAAQ;AAAA,EAER;AACF;AAEA,SAAS,kBAAkB,OAA0B;AACnD,MAAI,OAAO,UAAU,SAAU,QAAO,CAAC,KAAK;AAC5C,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,UAAM,MAAgB,CAAC;AACvB,eAAW,SAAS,OAAO;AACzB,UAAI,OAAO,UAAU,SAAU,KAAI,KAAK,KAAK;AAAA,IAC/C;AACA,WAAO;AAAA,EACT;AACA,SAAO,CAAC;AACV;AAEA,SAAS,iBACP,OACA,OACA,QACA,YACS;AACT,MAAI,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO;AAC/D,QAAM,QAAQ,MAAM,YAAY;AAChC,MAAI,UAAU,QAAQ,MAAM,SAAS,KAAK,KAAK,MAAM,SAAS,KAAK,EAAG,QAAO;AAC7E,MAAI,MAAM,SAAS,KAAK,EAAG,QAAO;AAClC,MAAI,CAAC,cAAc,cAAc,cAAc,aAAa,iBAAiB,EAAE,SAAS,KAAK,EAAG,QAAO;AACvG,MAAI,yBAAyB,OAAO,YAAY,MAAM,EAAG,QAAO;AAChE,SAAO,kBAAkB,KAAK,EAAE,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC;AAChE;AAEO,SAAS,qBAAqB,QAA6C;AAChF,QAAM,SAAS,OAAO,UAAU,oBAAoB;AACpD,MAAI,CAAC,OAAO,QAAS,QAAO,CAAC;AAC7B,MAAI,CAAC,OAAO,IAAK,QAAO,CAAC;AACzB,QAAM,SAA2B,CAAC;AAClC,QAAM,eAAe,qBAAqB,KAAK,OAAO,eAAe;AACrE,QAAM,aAAqD,CAAC;AAC5D,QAAM,QAAQ;AAAA,IACZ,gBAAgB,OAAO,kBAAkB,cAAc;AAAA,IACvD,UAAU,OAAO,YAAY,cAAc;AAAA,EAC7C;AACA,QAAM,SAAS,yBAAyB,MAAM;AAC9C,QAAM,cAAc,OAAO,qBAAqB,IAAI,OAAO,qBAAqB,OAAO;AACvF,QAAM,aAAa,OAAO,oBAAoB,IAAI,OAAO,oBAAoB,OAAO;AAEpF,aAAW,CAAC,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,GAAG,GAAG;AAC1D,QAAI,OAAO,UAAU,YAAa;AAClC,QAAI,CAAC,iBAAiB,OAAO,UAAU,QAAQ,OAAO,UAAU,EAAG;AACnE,UAAM,SAAS,kBAAkB,QAAQ;AACzC,UAAM,OAAO,oBAAI,IAAY;AAC7B,QAAI,kBAAkB;AACtB,eAAW,QAAQ,QAAQ;AACzB,UAAI,OAAO,UAAU,eAAe,mBAAmB,WAAY;AACnE,YAAM,iBAAiB,KAAK,IAAI,cAAc,OAAO,QAAQ,aAAa,eAAe;AACzF,YAAM,iBAAiB,kBAAkB;AACzC,YAAM,cAAc,OAAO,SAAS,cAAc,IAC9C,EAAE,GAAG,QAAQ,mBAAmB,eAAe,IAC/C;AACJ,YAAM,EAAE,QAAQ,YAAY,OAAO,IAAI,aAAa,MAAM,WAAW;AACrE,eAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK,GAAG;AAC7C,YAAI,OAAO,UAAU,eAAe,mBAAmB,WAAY;AACnE,cAAM,QAAQ,WAAW,CAAC;AAC1B,cAAM,OAAO,OAAO,CAAC;AACrB,cAAM,YAAY,GAAG,KAAK,IAAI,IAAI;AAClC,YAAI,KAAK,IAAI,SAAS,EAAG;AACzB,aAAK,IAAI,SAAS;AAClB,2BAAmB;AACnB,cAAM,mBAAmB,EAAE,YAAY,OAAO,YAAY,UAAU,OAAO,UAAU,OAAO,KAAK,CAAC;AAClG,eAAO,KAAK;AAAA,UACV,aAAa,OAAO;AAAA,UACpB,WAAW,OAAO,OAAO,QAAQ;AAAA,UACjC,iBAAiB,MAAM;AAAA,UACvB,WAAW,MAAM;AAAA,UACjB;AAAA,UACA,YAAY;AAAA,UACZ,OAAO,OAAO,iBAAiB,QAAQ;AAAA,QACzC,CAAC;AACD,YAAI,cAAc;AAChB,qBAAW,KAAK,EAAE,OAAO,KAAK,CAAC;AAAA,QACjC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,MAAI,cAAc;AAChB,UAAM,eAAe;AAAA,MACnB,YAAY,OAAO;AAAA,MACnB,UAAU,OAAO;AAAA,MACjB,YAAY,WAAW;AAAA,MACvB,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACA,QAAM,iBAAiB,EAAE,YAAY,OAAO,YAAY,UAAU,OAAO,UAAU,YAAY,OAAO,OAAO,CAAC;AAE9G,SAAO;AACT;AAEA,SAAS,gBAAgB,UAAkB,KAAyD;AAClG,MAAI,CAAC,IAAK,QAAO,CAAC;AAClB,QAAM,QAA2B,CAAC;AAClC,QAAM,SAAS,oBAAI,IAAY;AAC/B,aAAW,SAAS,OAAO,KAAK,GAAG,GAAG;AACpC,UAAM,MAAM,GAAG,QAAQ,IAAI,KAAK;AAChC,QAAI,OAAO,IAAI,GAAG,EAAG;AACrB,WAAO,IAAI,GAAG;AACd,UAAM,KAAK,CAAC,UAAU,KAAK,CAAC;AAAA,EAC9B;AACA,SAAO;AACT;AAEA,eAAsB,6BACpB,IACA,QACA,SACe;AACf,QAAM,OAAO,qBAAqB,MAAM;AACxC,QAAM,SAAS,OAAO,UAAU,oBAAoB;AACpD,MAAI,CAAC,OAAO,QAAS;AACrB,QAAM,iBAAiB,OAAO,kBAAkB;AAChD,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,aAAa,gBAAgB,OAAO,OAAO,QAAQ,GAAG,OAAO,GAAG;AAEtE,QAAM,cAAc,OAAO,aAAiD;AAC1E,QAAI,cAAc,SACf,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,OAAO,UAAU,EAClD,MAAM,2CAAoD,cAAc,EAAE,EAC1E,MAAM,qCAA8C,QAAQ,EAAE;AACjE,QAAI,WAAW,QAAQ;AACrB,oBAAc,YAAY,MAAM,CAAC,OAAY,GAAG;AAAA,QAC9C,WAAW,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,IAAI;AAAA,UACtC,GAAG,aAAoB,KAAK,GAAG;AAAA,UAC/B,GAAG,SAAgB,KAAK,KAAK;AAAA,QAC/B,CAAC,CAAC;AAAA,MACJ,CAAC;AAAA,IACH,OAAO;AACL,oBAAc,YAAY,MAAM,aAAoB,KAAK,OAAO,OAAO,QAAQ,CAAC;AAAA,IAClF;AACA,UAAM,YAAY,QAAQ;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAClB,UAAM,WAAW,KAAK,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,YAAY,WAAW,EAAE;AACvE,eAAW,SAAS,MAAM,UAAU,iBAAiB,GAAG;AACtD,YAAM,SAAS,WAAW,eAAsB,EAAE,OAAO,KAAY,EAAE,QAAQ;AAAA,IACjF;AAAA,EACF;AAEA,MAAI,SAAS,KAAK;AAChB,UAAM,YAAY,QAAQ,GAAG;AAC7B;AAAA,EACF;AAEA,QAAM,GAAG,YAAY,EAAE,QAAQ,WAAW;AAC5C;AAEA,eAAsB,4BACpB,IACA,QACA,SACe;AACf,QAAM,iBAAiB,OAAO,kBAAkB;AAChD,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,WAAW,SAAS,OAAO;AACjC,QAAM,SACH,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,OAAO,UAAU,EAClD,MAAM,aAAoB,KAAK,OAAO,OAAO,QAAQ,CAAC,EACtD,MAAM,2CAAoD,cAAc,EAAE,EAC1E,MAAM,qCAA8C,QAAQ,EAAE,EAC9D,QAAQ;AACb;AAEA,eAAsB,4BACpB,IACA,UACe;AACf,MAAI,CAAC,SAAS,OAAQ;AACtB,QAAM,SAAS,oBAAoB;AACnC,MAAI,CAAC,OAAO,QAAS;AAErB,QAAM,OAAO,SAAS,QAAQ,CAAC,YAAY,qBAAqB,EAAE,GAAG,SAAS,OAAO,CAAC,CAAC;AACvF,MAAI,CAAC,KAAK,QAAQ;AAChB,UAAM,aAAa,SAAS,CAAC,GAAG;AAChC,QAAI,CAAC,WAAY;AACjB,UAAM,MAAM,SAAS,IAAI,CAAC,MAAM,OAAO,EAAE,QAAQ,CAAC;AAClD,UAAM,GACH,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,UAAU,EAC3C,MAAM,aAAoB,MAAM,GAAG,EACnC,QAAQ;AACX;AAAA,EACF;AAEA,QAAM,WAAW,CAAC,KAAoB,WAA0B,GAAG,OAAO,UAAU,IAAI,UAAU,UAAU;AAC5G,QAAM,eAAe,oBAAI,IAA0F;AAEnH,aAAW,WAAW,UAAU;AAC9B,UAAM,MAAM,QAAQ,kBAAkB;AACtC,UAAM,SAAS,QAAQ,YAAY;AACnC,UAAM,MAAM,SAAS,KAAK,MAAM;AAChC,UAAM,SAAS,aAAa,IAAI,GAAG,KAAK,EAAE,gBAAgB,KAAK,UAAU,QAAQ,KAAK,oBAAI,IAAY,EAAE;AACxG,WAAO,IAAI,IAAI,OAAO,QAAQ,QAAQ,CAAC;AACvC,iBAAa,IAAI,KAAK,MAAM;AAAA,EAC9B;AAEA,QAAM,GAAG,YAAY,EAAE,QAAQ,OAAO,QAAQ;AAC5C,eAAW,CAAC,EAAE,MAAM,KAAK,aAAa,QAAQ,GAAG;AAG/C,YAAM,cAAc,IACjB,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,SAAS,CAAC,EAAE,UAAU,EACvD,MAAM,2CAAoD,OAAO,cAAc,EAAE,EACjF,MAAM,qCAA8C,OAAO,QAAQ,EAAE,EACrE,MAAM,aAAoB,MAAM,MAAM,KAAK,OAAO,GAAG,CAAC;AACzD,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,UAAM,wBAAwB,KAAK,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,YAAY,WAAW,EAAE;AACpF,eAAW,SAAS,MAAM,uBAAuB,iBAAiB,GAAG;AACnE,YAAM,IAAI,WAAW,eAAsB,EAAE,OAAO,KAAY,EAAE,QAAQ;AAAA,IAC5E;AAAA,EACF,CAAC;AACH;",
+  "sourcesContent": ["import { type Kysely, type Transaction, sql } from 'kysely'\nimport {\n  isSearchFieldBlocklisted,\n  resolveSearchConfig,\n  resolveSearchTokenLimits,\n  type SearchConfig,\n} from '@open-mercato/shared/lib/search/config'\nimport { tokenizeText } from '@open-mercato/shared/lib/search/tokenize'\nimport { parseBooleanToken } from '@open-mercato/shared/lib/boolean'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst logger = createLogger('query_index').child({ component: 'search-tokens' })\n\nconst INSERT_BATCH_SIZE = 500\n\nfunction chunk<T>(items: T[], size: number): T[][] {\n  if (size <= 0) return [items]\n  const out: T[][] = []\n  for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size))\n  return out\n}\n\nexport type SearchTokenRow = {\n  entity_type: string\n  entity_id: string\n  organization_id: string | null\n  tenant_id: string | null\n  field: string\n  token_hash: string\n  token?: string | null\n}\n\ntype BuildTokenOptions = {\n  entityType: string\n  recordId: string\n  organizationId?: string | null\n  tenantId?: string | null\n  doc?: Record<string, unknown> | null\n  config?: SearchConfig\n}\n\nconst DEFAULT_SCOPE = { organizationId: null, tenantId: null }\ntype EntityFieldPair = [string, string]\ntype SearchTokenExecutor = Kysely<any> | Transaction<any>\n\nexport const isSearchDebugEnabled = (): boolean => {\n  return parseBooleanToken(process.env.OM_SEARCH_DEBUG ?? '') === true\n}\n\nconst debug = (event: string, payload: Record<string, unknown>) => {\n  if (!isSearchDebugEnabled()) return\n  try {\n    logger.debug('Search token event', { event, payload })\n  } catch {\n    // ignore\n  }\n}\n\nfunction collectTextValues(value: unknown): string[] {\n  if (typeof value === 'string') return [value]\n  if (Array.isArray(value)) {\n    const out: string[] = []\n    for (const entry of value) {\n      if (typeof entry === 'string') out.push(entry)\n    }\n    return out\n  }\n  return []\n}\n\nfunction shouldIndexField(\n  field: string,\n  value: unknown,\n  config: SearchConfig,\n  entityType: string | null,\n): boolean {\n  if (typeof value !== 'string' && !Array.isArray(value)) return false\n  const lower = field.toLowerCase()\n  if (lower === 'id' || lower.endsWith('_id') || lower.endsWith('.id')) return false\n  if (lower.endsWith('_at')) return false\n  if (['created_at', 'updated_at', 'deleted_at', 'tenant_id', 'organization_id'].includes(lower)) return false\n  if (isSearchFieldBlocklisted(field, entityType, config)) return false\n  return collectTextValues(value).some((text) => text.length > 0)\n}\n\nexport function buildSearchTokenRows(params: BuildTokenOptions): SearchTokenRow[] {\n  const config = params.config ?? resolveSearchConfig()\n  if (!config.enabled) return []\n  if (!params.doc) return []\n  const tokens: SearchTokenRow[] = []\n  const capturePairs = isSearchDebugEnabled() && params.entityType === 'customers:customer_deal'\n  const debugPairs: Array<{ field: string; hash: string }> = []\n  const scope = {\n    organizationId: params.organizationId ?? DEFAULT_SCOPE.organizationId,\n    tenantId: params.tenantId ?? DEFAULT_SCOPE.tenantId,\n  }\n  const limits = resolveSearchTokenLimits(config)\n  const recordLimit = limits.maxTokensPerRecord > 0 ? limits.maxTokensPerRecord : Number.POSITIVE_INFINITY\n  const fieldLimit = limits.maxTokensPerField > 0 ? limits.maxTokensPerField : Number.POSITIVE_INFINITY\n\n  for (const [field, rawValue] of Object.entries(params.doc)) {\n    if (tokens.length >= recordLimit) break\n    if (!shouldIndexField(field, rawValue, config, params.entityType)) continue\n    const values = collectTextValues(rawValue)\n    const seen = new Set<string>()\n    let fieldTokenCount = 0\n    for (const text of values) {\n      if (tokens.length >= recordLimit || fieldTokenCount >= fieldLimit) break\n      const remainingLimit = Math.min(recordLimit - tokens.length, fieldLimit - fieldTokenCount)\n      const candidateLimit = fieldTokenCount + remainingLimit\n      const tokenConfig = Number.isFinite(candidateLimit)\n        ? { ...config, maxTokensPerField: candidateLimit }\n        : config\n      const { tokens: textTokens, hashes } = tokenizeText(text, tokenConfig)\n      for (let i = 0; i < textTokens.length; i += 1) {\n        if (tokens.length >= recordLimit || fieldTokenCount >= fieldLimit) break\n        const token = textTokens[i]\n        const hash = hashes[i]\n        const dedupeKey = `${field}|${hash}`\n        if (seen.has(dedupeKey)) continue\n        seen.add(dedupeKey)\n        fieldTokenCount += 1\n        debug('token.generated', { entityType: params.entityType, recordId: params.recordId, field, hash })\n        tokens.push({\n          entity_type: params.entityType,\n          entity_id: String(params.recordId),\n          organization_id: scope.organizationId,\n          tenant_id: scope.tenantId,\n          field,\n          token_hash: hash,\n          token: config.storeRawTokens ? token : null,\n        })\n        if (capturePairs) {\n          debugPairs.push({ field, hash })\n        }\n      }\n    }\n  }\n  if (capturePairs) {\n    debug('deal.tokens', {\n      entityType: params.entityType,\n      recordId: params.recordId,\n      tokenCount: debugPairs.length,\n      tokens: debugPairs,\n    })\n  }\n  debug('doc.completed', { entityType: params.entityType, recordId: params.recordId, tokenCount: tokens.length })\n\n  return tokens\n}\n\nfunction buildFieldPairs(recordId: string, doc?: Record<string, unknown> | null): EntityFieldPair[] {\n  if (!doc) return []\n  const pairs: EntityFieldPair[] = []\n  const dedupe = new Set<string>()\n  for (const field of Object.keys(doc)) {\n    const key = `${recordId}|${field}`\n    if (dedupe.has(key)) continue\n    dedupe.add(key)\n    pairs.push([recordId, field])\n  }\n  return pairs\n}\n\nexport async function replaceSearchTokensForRecord(\n  db: Kysely<any>,\n  params: BuildTokenOptions,\n  options?: { trx?: SearchTokenExecutor },\n): Promise<void> {\n  const rows = buildSearchTokenRows(params)\n  const config = params.config ?? resolveSearchConfig()\n  if (!config.enabled) return\n  const organizationId = params.organizationId ?? null\n  const tenantId = params.tenantId ?? null\n  const fieldPairs = buildFieldPairs(String(params.recordId), params.doc)\n\n  const writeTokens = async (executor: SearchTokenExecutor): Promise<void> => {\n    let deleteQuery = executor\n      .deleteFrom('search_tokens' as any)\n      .where('entity_type' as any, '=', params.entityType)\n      .where(sql<boolean>`organization_id is not distinct from ${organizationId}`)\n      .where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)\n    if (fieldPairs.length) {\n      deleteQuery = deleteQuery.where((eb: any) => eb.or(\n        fieldPairs.map(([rid, field]) => eb.and([\n          eb('entity_id' as any, '=', rid),\n          eb('field' as any, '=', field),\n        ])),\n      ))\n    } else {\n      deleteQuery = deleteQuery.where('entity_id' as any, '=', String(params.recordId))\n    }\n    await deleteQuery.execute()\n    if (!rows.length) return\n    const payloads = rows.map((row) => ({ ...row, created_at: sql`now()` }))\n    for (const batch of chunk(payloads, INSERT_BATCH_SIZE)) {\n      await executor.insertInto('search_tokens' as any).values(batch as any).execute()\n    }\n  }\n\n  if (options?.trx) {\n    await writeTokens(options.trx)\n    return\n  }\n\n  await db.transaction().execute(writeTokens)\n}\n\nexport async function deleteSearchTokensForRecord(\n  db: Kysely<any>,\n  params: { entityType: string; recordId: string; organizationId?: string | null; tenantId?: string | null },\n  options?: { trx?: SearchTokenExecutor },\n): Promise<void> {\n  const organizationId = params.organizationId ?? null\n  const tenantId = params.tenantId ?? null\n  const executor = options?.trx ?? db\n  await executor\n    .deleteFrom('search_tokens' as any)\n    .where('entity_type' as any, '=', params.entityType)\n    .where('entity_id' as any, '=', String(params.recordId))\n    .where(sql<boolean>`organization_id is not distinct from ${organizationId}`)\n    .where(sql<boolean>`tenant_id is not distinct from ${tenantId}`)\n    .execute()\n}\n\n// NUL, not a printable separator: a field name may itself contain a space, so `a b` + hash `c`\n// would otherwise sign identically to field `a` + hash `b c`.\nconst SIGNATURE_SEPARATOR = String.fromCharCode(0)\n\n// Identifies one token row for comparison. `token` is NULL unless `storeRawTokens` is on, and a\n// stored NULL has to sign the same as the `null` a freshly built row carries \u2014 otherwise every\n// record compares as changed and the skip never fires.\nfunction tokenSignature(row: { field?: unknown; token_hash?: unknown; token?: unknown }): string {\n  return [\n    String(row.field ?? ''),\n    String(row.token_hash ?? ''),\n    row.token == null ? '' : String(row.token),\n  ].join(SIGNATURE_SEPARATOR)\n}\n\n// Multiplicities, not sets: #4681 reports token rows duplicated by the concurrent-replacement\n// defect, and a set comparison reads such a record as already correct and preserves the duplicates\n// forever. Counting sends it through a full rewrite, which collapses them.\nfunction tallyEquals(a: Map<string, number> | undefined, b: Map<string, number> | undefined): boolean {\n  const left = a ?? new Map<string, number>()\n  const right = b ?? new Map<string, number>()\n  if (left.size !== right.size) return false\n  for (const [key, count] of left.entries()) {\n    if (right.get(key) !== count) return false\n  }\n  return true\n}\n\nfunction tallyTokenRows<TRow extends { field?: unknown; token_hash?: unknown; token?: unknown }>(\n  rows: Iterable<TRow>,\n  keyOf: (row: TRow) => string\n): Map<string, Map<string, number>> {\n  const tallies = new Map<string, Map<string, number>>()\n  for (const row of rows) {\n    const key = keyOf(row)\n    const tally = tallies.get(key) ?? new Map<string, number>()\n    const signature = tokenSignature(row)\n    tally.set(signature, (tally.get(signature) ?? 0) + 1)\n    tallies.set(key, tally)\n  }\n  return tallies\n}\n\nexport async function replaceSearchTokensForBatch(\n  db: Kysely<any>,\n  payloads: Array<BuildTokenOptions & { doc: Record<string, unknown> }>\n): Promise<void> {\n  if (!payloads.length) return\n  const config = resolveSearchConfig()\n  if (!config.enabled) return\n\n  const rows = payloads.flatMap((payload) => buildSearchTokenRows({ ...payload, config }))\n  if (!rows.length) {\n    const entityType = payloads[0]?.entityType\n    if (!entityType) return\n    const ids = payloads.map((p) => String(p.recordId))\n    await db\n      .deleteFrom('search_tokens' as any)\n      .where('entity_type' as any, '=', entityType)\n      .where('entity_id' as any, 'in', ids)\n      .execute()\n    return\n  }\n\n  const scopeKey = (org: string | null, tenant: string | null) => `${org ?? '__null__'}|${tenant ?? '__null__'}`\n  const scopeBuckets = new Map<string, { organizationId: string | null; tenantId: string | null; ids: Set<string> }>()\n\n  for (const payload of payloads) {\n    const org = payload.organizationId ?? null\n    const tenant = payload.tenantId ?? null\n    const key = scopeKey(org, tenant)\n    const bucket = scopeBuckets.get(key) ?? { organizationId: org, tenantId: tenant, ids: new Set<string>() }\n    bucket.ids.add(String(payload.recordId))\n    scopeBuckets.set(key, bucket)\n  }\n\n  const recordKeyOf = (row: SearchTokenRow) =>\n    `${scopeKey(row.organization_id ?? null, row.tenant_id ?? null)}|${String(row.entity_id)}`\n  const builtTally = tallyTokenRows(rows, recordKeyOf)\n\n  // Read outside the transaction, deliberately. The comparison decides only whether to skip a\n  // rewrite, so a concurrent writer costs us at most a rewrite we declined \u2014 declined because the\n  // table already held exactly the rows this call wanted to write. One ordering is worth naming\n  // though: if the read matches and a concurrent writer then commits tokens built from a *staler*\n  // doc, the unconditional rewrite this call used to perform would have overwritten them by\n  // accident. It no longer does, so those stale rows survive until the record's next write. That\n  // is a repair we lose, not a guarantee we break.\n  const changedIdsByBucket = new Map<string, Set<string>>()\n  for (const [key, bucket] of scopeBuckets.entries()) {\n    const ids = Array.from(bucket.ids)\n    const builtCountById = new Map<string, number>()\n    for (const id of ids) {\n      let total = 0\n      const tally = builtTally.get(`${key}|${id}`)\n      if (tally) for (const count of tally.values()) total += count\n      builtCountById.set(id, total)\n    }\n\n    // Count probe first. Its result is one row per record in the batch, so it is bounded by the\n    // batch size \u2014 unlike a bare row read, which would be bounded only by how many token rows the\n    // table already holds for these ids, a quantity this function does not control and (per #4681)\n    // has no reason to trust.\n    const storedCounts = await db\n      .selectFrom('search_tokens' as any)\n      .select(['entity_id' as any, sql<number>`count(*)`.as('token_count') as any])\n      .where('entity_type' as any, '=', payloads[0].entityType)\n      .where(sql<boolean>`organization_id is not distinct from ${bucket.organizationId}`)\n      .where(sql<boolean>`tenant_id is not distinct from ${bucket.tenantId}`)\n      .where('entity_id' as any, 'in', ids)\n      .groupBy('entity_id' as any)\n      .execute()\n    const storedCountById = new Map<string, number>()\n    for (const row of storedCounts as any[]) {\n      storedCountById.set(String(row.entity_id), Number(row.token_count))\n    }\n\n    const changed = new Set<string>()\n    // A record whose stored row count already differs is changed, whatever the rows say \u2014 the\n    // duplicate case from #4681 resolves here without ever materializing the duplicated rows.\n    const contentCandidates = ids.filter((id) => {\n      const builtCount = builtCountById.get(id) ?? 0\n      if ((storedCountById.get(id) ?? 0) !== builtCount) {\n        changed.add(id)\n        return false\n      }\n      return builtCount > 0\n    })\n\n    if (contentCandidates.length) {\n      const rowBudget = contentCandidates.reduce((sum, id) => sum + (builtCountById.get(id) ?? 0), 0)\n      const stored = await db\n        .selectFrom('search_tokens' as any)\n        .select(['entity_id' as any, 'field' as any, 'token_hash' as any, 'token' as any])\n        .where('entity_type' as any, '=', payloads[0].entityType)\n        .where(sql<boolean>`organization_id is not distinct from ${bucket.organizationId}`)\n        .where(sql<boolean>`tenant_id is not distinct from ${bucket.tenantId}`)\n        .where('entity_id' as any, 'in', contentCandidates)\n        // Counts already match, so this cannot truncate \u2014 it bounds the damage if a concurrent\n        // writer inserts between the probe and this read. A truncated read compares as changed,\n        // which costs a rewrite rather than a wrong skip.\n        .limit(rowBudget)\n        .execute()\n      const storedTally = tallyTokenRows(stored as any[], (row) => String(row.entity_id))\n      for (const id of contentCandidates) {\n        if (!tallyEquals(builtTally.get(`${key}|${id}`), storedTally.get(id))) changed.add(id)\n      }\n    }\n    changedIdsByBucket.set(key, changed)\n  }\n\n  const changedRecordKeys = new Set<string>()\n  for (const [key, changed] of changedIdsByBucket.entries()) {\n    for (const id of changed) changedRecordKeys.add(`${key}|${id}`)\n  }\n  debug('batch.skip', {\n    entityType: payloads[0].entityType,\n    recordCount: payloads.length,\n    changedCount: changedRecordKeys.size,\n  })\n  if (!changedRecordKeys.size) return\n\n  await db.transaction().execute(async (trx) => {\n    for (const [key, bucket] of scopeBuckets.entries()) {\n      const changed = changedIdsByBucket.get(key)\n      if (!changed?.size) continue\n      // Delete by entity_id: a batch replaces all of a record's tokens, and a per-field OR over the\n      // whole batch overflows the query compiler's call stack on large batches.\n      const deleteQuery = trx\n        .deleteFrom('search_tokens' as any)\n        .where('entity_type' as any, '=', payloads[0].entityType)\n        .where(sql<boolean>`organization_id is not distinct from ${bucket.organizationId}`)\n        .where(sql<boolean>`tenant_id is not distinct from ${bucket.tenantId}`)\n        .where('entity_id' as any, 'in', Array.from(changed))\n      await deleteQuery.execute()\n    }\n    const payloadWithTimestamps = rows\n      .filter((row) => changedRecordKeys.has(recordKeyOf(row)))\n      .map((row) => ({ ...row, created_at: sql`now()` }))\n    for (const batch of chunk(payloadWithTimestamps, INSERT_BATCH_SIZE)) {\n      await trx.insertInto('search_tokens' as any).values(batch as any).execute()\n    }\n  })\n}\n"],
+  "mappings": "AAAA,SAAwC,WAAW;AACnD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AACP,SAAS,oBAAoB;AAC7B,SAAS,yBAAyB;AAClC,SAAS,oBAAoB;AAE7B,MAAM,SAAS,aAAa,aAAa,EAAE,MAAM,EAAE,WAAW,gBAAgB,CAAC;AAE/E,MAAM,oBAAoB;AAE1B,SAAS,MAAS,OAAY,MAAqB;AACjD,MAAI,QAAQ,EAAG,QAAO,CAAC,KAAK;AAC5B,QAAM,MAAa,CAAC;AACpB,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,KAAM,KAAI,KAAK,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC;AAC9E,SAAO;AACT;AAqBA,MAAM,gBAAgB,EAAE,gBAAgB,MAAM,UAAU,KAAK;AAItD,MAAM,uBAAuB,MAAe;AACjD,SAAO,kBAAkB,QAAQ,IAAI,mBAAmB,EAAE,MAAM;AAClE;AAEA,MAAM,QAAQ,CAAC,OAAe,YAAqC;AACjE,MAAI,CAAC,qBAAqB,EAAG;AAC7B,MAAI;AACF,WAAO,MAAM,sBAAsB,EAAE,OAAO,QAAQ,CAAC;AAAA,EACvD,QAAQ;AAAA,EAER;AACF;AAEA,SAAS,kBAAkB,OAA0B;AACnD,MAAI,OAAO,UAAU,SAAU,QAAO,CAAC,KAAK;AAC5C,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,UAAM,MAAgB,CAAC;AACvB,eAAW,SAAS,OAAO;AACzB,UAAI,OAAO,UAAU,SAAU,KAAI,KAAK,KAAK;AAAA,IAC/C;AACA,WAAO;AAAA,EACT;AACA,SAAO,CAAC;AACV;AAEA,SAAS,iBACP,OACA,OACA,QACA,YACS;AACT,MAAI,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO;AAC/D,QAAM,QAAQ,MAAM,YAAY;AAChC,MAAI,UAAU,QAAQ,MAAM,SAAS,KAAK,KAAK,MAAM,SAAS,KAAK,EAAG,QAAO;AAC7E,MAAI,MAAM,SAAS,KAAK,EAAG,QAAO;AAClC,MAAI,CAAC,cAAc,cAAc,cAAc,aAAa,iBAAiB,EAAE,SAAS,KAAK,EAAG,QAAO;AACvG,MAAI,yBAAyB,OAAO,YAAY,MAAM,EAAG,QAAO;AAChE,SAAO,kBAAkB,KAAK,EAAE,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC;AAChE;AAEO,SAAS,qBAAqB,QAA6C;AAChF,QAAM,SAAS,OAAO,UAAU,oBAAoB;AACpD,MAAI,CAAC,OAAO,QAAS,QAAO,CAAC;AAC7B,MAAI,CAAC,OAAO,IAAK,QAAO,CAAC;AACzB,QAAM,SAA2B,CAAC;AAClC,QAAM,eAAe,qBAAqB,KAAK,OAAO,eAAe;AACrE,QAAM,aAAqD,CAAC;AAC5D,QAAM,QAAQ;AAAA,IACZ,gBAAgB,OAAO,kBAAkB,cAAc;AAAA,IACvD,UAAU,OAAO,YAAY,cAAc;AAAA,EAC7C;AACA,QAAM,SAAS,yBAAyB,MAAM;AAC9C,QAAM,cAAc,OAAO,qBAAqB,IAAI,OAAO,qBAAqB,OAAO;AACvF,QAAM,aAAa,OAAO,oBAAoB,IAAI,OAAO,oBAAoB,OAAO;AAEpF,aAAW,CAAC,OAAO,QAAQ,KAAK,OAAO,QAAQ,OAAO,GAAG,GAAG;AAC1D,QAAI,OAAO,UAAU,YAAa;AAClC,QAAI,CAAC,iBAAiB,OAAO,UAAU,QAAQ,OAAO,UAAU,EAAG;AACnE,UAAM,SAAS,kBAAkB,QAAQ;AACzC,UAAM,OAAO,oBAAI,IAAY;AAC7B,QAAI,kBAAkB;AACtB,eAAW,QAAQ,QAAQ;AACzB,UAAI,OAAO,UAAU,eAAe,mBAAmB,WAAY;AACnE,YAAM,iBAAiB,KAAK,IAAI,cAAc,OAAO,QAAQ,aAAa,eAAe;AACzF,YAAM,iBAAiB,kBAAkB;AACzC,YAAM,cAAc,OAAO,SAAS,cAAc,IAC9C,EAAE,GAAG,QAAQ,mBAAmB,eAAe,IAC/C;AACJ,YAAM,EAAE,QAAQ,YAAY,OAAO,IAAI,aAAa,MAAM,WAAW;AACrE,eAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK,GAAG;AAC7C,YAAI,OAAO,UAAU,eAAe,mBAAmB,WAAY;AACnE,cAAM,QAAQ,WAAW,CAAC;AAC1B,cAAM,OAAO,OAAO,CAAC;AACrB,cAAM,YAAY,GAAG,KAAK,IAAI,IAAI;AAClC,YAAI,KAAK,IAAI,SAAS,EAAG;AACzB,aAAK,IAAI,SAAS;AAClB,2BAAmB;AACnB,cAAM,mBAAmB,EAAE,YAAY,OAAO,YAAY,UAAU,OAAO,UAAU,OAAO,KAAK,CAAC;AAClG,eAAO,KAAK;AAAA,UACV,aAAa,OAAO;AAAA,UACpB,WAAW,OAAO,OAAO,QAAQ;AAAA,UACjC,iBAAiB,MAAM;AAAA,UACvB,WAAW,MAAM;AAAA,UACjB;AAAA,UACA,YAAY;AAAA,UACZ,OAAO,OAAO,iBAAiB,QAAQ;AAAA,QACzC,CAAC;AACD,YAAI,cAAc;AAChB,qBAAW,KAAK,EAAE,OAAO,KAAK,CAAC;AAAA,QACjC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,MAAI,cAAc;AAChB,UAAM,eAAe;AAAA,MACnB,YAAY,OAAO;AAAA,MACnB,UAAU,OAAO;AAAA,MACjB,YAAY,WAAW;AAAA,MACvB,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACA,QAAM,iBAAiB,EAAE,YAAY,OAAO,YAAY,UAAU,OAAO,UAAU,YAAY,OAAO,OAAO,CAAC;AAE9G,SAAO;AACT;AAEA,SAAS,gBAAgB,UAAkB,KAAyD;AAClG,MAAI,CAAC,IAAK,QAAO,CAAC;AAClB,QAAM,QAA2B,CAAC;AAClC,QAAM,SAAS,oBAAI,IAAY;AAC/B,aAAW,SAAS,OAAO,KAAK,GAAG,GAAG;AACpC,UAAM,MAAM,GAAG,QAAQ,IAAI,KAAK;AAChC,QAAI,OAAO,IAAI,GAAG,EAAG;AACrB,WAAO,IAAI,GAAG;AACd,UAAM,KAAK,CAAC,UAAU,KAAK,CAAC;AAAA,EAC9B;AACA,SAAO;AACT;AAEA,eAAsB,6BACpB,IACA,QACA,SACe;AACf,QAAM,OAAO,qBAAqB,MAAM;AACxC,QAAM,SAAS,OAAO,UAAU,oBAAoB;AACpD,MAAI,CAAC,OAAO,QAAS;AACrB,QAAM,iBAAiB,OAAO,kBAAkB;AAChD,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,aAAa,gBAAgB,OAAO,OAAO,QAAQ,GAAG,OAAO,GAAG;AAEtE,QAAM,cAAc,OAAO,aAAiD;AAC1E,QAAI,cAAc,SACf,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,OAAO,UAAU,EAClD,MAAM,2CAAoD,cAAc,EAAE,EAC1E,MAAM,qCAA8C,QAAQ,EAAE;AACjE,QAAI,WAAW,QAAQ;AACrB,oBAAc,YAAY,MAAM,CAAC,OAAY,GAAG;AAAA,QAC9C,WAAW,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,GAAG,IAAI;AAAA,UACtC,GAAG,aAAoB,KAAK,GAAG;AAAA,UAC/B,GAAG,SAAgB,KAAK,KAAK;AAAA,QAC/B,CAAC,CAAC;AAAA,MACJ,CAAC;AAAA,IACH,OAAO;AACL,oBAAc,YAAY,MAAM,aAAoB,KAAK,OAAO,OAAO,QAAQ,CAAC;AAAA,IAClF;AACA,UAAM,YAAY,QAAQ;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAClB,UAAM,WAAW,KAAK,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,YAAY,WAAW,EAAE;AACvE,eAAW,SAAS,MAAM,UAAU,iBAAiB,GAAG;AACtD,YAAM,SAAS,WAAW,eAAsB,EAAE,OAAO,KAAY,EAAE,QAAQ;AAAA,IACjF;AAAA,EACF;AAEA,MAAI,SAAS,KAAK;AAChB,UAAM,YAAY,QAAQ,GAAG;AAC7B;AAAA,EACF;AAEA,QAAM,GAAG,YAAY,EAAE,QAAQ,WAAW;AAC5C;AAEA,eAAsB,4BACpB,IACA,QACA,SACe;AACf,QAAM,iBAAiB,OAAO,kBAAkB;AAChD,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,WAAW,SAAS,OAAO;AACjC,QAAM,SACH,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,OAAO,UAAU,EAClD,MAAM,aAAoB,KAAK,OAAO,OAAO,QAAQ,CAAC,EACtD,MAAM,2CAAoD,cAAc,EAAE,EAC1E,MAAM,qCAA8C,QAAQ,EAAE,EAC9D,QAAQ;AACb;AAIA,MAAM,sBAAsB,OAAO,aAAa,CAAC;AAKjD,SAAS,eAAe,KAAyE;AAC/F,SAAO;AAAA,IACL,OAAO,IAAI,SAAS,EAAE;AAAA,IACtB,OAAO,IAAI,cAAc,EAAE;AAAA,IAC3B,IAAI,SAAS,OAAO,KAAK,OAAO,IAAI,KAAK;AAAA,EAC3C,EAAE,KAAK,mBAAmB;AAC5B;AAKA,SAAS,YAAY,GAAoC,GAA6C;AACpG,QAAM,OAAO,KAAK,oBAAI,IAAoB;AAC1C,QAAM,QAAQ,KAAK,oBAAI,IAAoB;AAC3C,MAAI,KAAK,SAAS,MAAM,KAAM,QAAO;AACrC,aAAW,CAAC,KAAK,KAAK,KAAK,KAAK,QAAQ,GAAG;AACzC,QAAI,MAAM,IAAI,GAAG,MAAM,MAAO,QAAO;AAAA,EACvC;AACA,SAAO;AACT;AAEA,SAAS,eACP,MACA,OACkC;AAClC,QAAM,UAAU,oBAAI,IAAiC;AACrD,aAAW,OAAO,MAAM;AACtB,UAAM,MAAM,MAAM,GAAG;AACrB,UAAM,QAAQ,QAAQ,IAAI,GAAG,KAAK,oBAAI,IAAoB;AAC1D,UAAM,YAAY,eAAe,GAAG;AACpC,UAAM,IAAI,YAAY,MAAM,IAAI,SAAS,KAAK,KAAK,CAAC;AACpD,YAAQ,IAAI,KAAK,KAAK;AAAA,EACxB;AACA,SAAO;AACT;AAEA,eAAsB,4BACpB,IACA,UACe;AACf,MAAI,CAAC,SAAS,OAAQ;AACtB,QAAM,SAAS,oBAAoB;AACnC,MAAI,CAAC,OAAO,QAAS;AAErB,QAAM,OAAO,SAAS,QAAQ,CAAC,YAAY,qBAAqB,EAAE,GAAG,SAAS,OAAO,CAAC,CAAC;AACvF,MAAI,CAAC,KAAK,QAAQ;AAChB,UAAM,aAAa,SAAS,CAAC,GAAG;AAChC,QAAI,CAAC,WAAY;AACjB,UAAM,MAAM,SAAS,IAAI,CAAC,MAAM,OAAO,EAAE,QAAQ,CAAC;AAClD,UAAM,GACH,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,UAAU,EAC3C,MAAM,aAAoB,MAAM,GAAG,EACnC,QAAQ;AACX;AAAA,EACF;AAEA,QAAM,WAAW,CAAC,KAAoB,WAA0B,GAAG,OAAO,UAAU,IAAI,UAAU,UAAU;AAC5G,QAAM,eAAe,oBAAI,IAA0F;AAEnH,aAAW,WAAW,UAAU;AAC9B,UAAM,MAAM,QAAQ,kBAAkB;AACtC,UAAM,SAAS,QAAQ,YAAY;AACnC,UAAM,MAAM,SAAS,KAAK,MAAM;AAChC,UAAM,SAAS,aAAa,IAAI,GAAG,KAAK,EAAE,gBAAgB,KAAK,UAAU,QAAQ,KAAK,oBAAI,IAAY,EAAE;AACxG,WAAO,IAAI,IAAI,OAAO,QAAQ,QAAQ,CAAC;AACvC,iBAAa,IAAI,KAAK,MAAM;AAAA,EAC9B;AAEA,QAAM,cAAc,CAAC,QACnB,GAAG,SAAS,IAAI,mBAAmB,MAAM,IAAI,aAAa,IAAI,CAAC,IAAI,OAAO,IAAI,SAAS,CAAC;AAC1F,QAAM,aAAa,eAAe,MAAM,WAAW;AASnD,QAAM,qBAAqB,oBAAI,IAAyB;AACxD,aAAW,CAAC,KAAK,MAAM,KAAK,aAAa,QAAQ,GAAG;AAClD,UAAM,MAAM,MAAM,KAAK,OAAO,GAAG;AACjC,UAAM,iBAAiB,oBAAI,IAAoB;AAC/C,eAAW,MAAM,KAAK;AACpB,UAAI,QAAQ;AACZ,YAAM,QAAQ,WAAW,IAAI,GAAG,GAAG,IAAI,EAAE,EAAE;AAC3C,UAAI,MAAO,YAAW,SAAS,MAAM,OAAO,EAAG,UAAS;AACxD,qBAAe,IAAI,IAAI,KAAK;AAAA,IAC9B;AAMA,UAAM,eAAe,MAAM,GACxB,WAAW,eAAsB,EACjC,OAAO,CAAC,aAAoB,cAAsB,GAAG,aAAa,CAAQ,CAAC,EAC3E,MAAM,eAAsB,KAAK,SAAS,CAAC,EAAE,UAAU,EACvD,MAAM,2CAAoD,OAAO,cAAc,EAAE,EACjF,MAAM,qCAA8C,OAAO,QAAQ,EAAE,EACrE,MAAM,aAAoB,MAAM,GAAG,EACnC,QAAQ,WAAkB,EAC1B,QAAQ;AACX,UAAM,kBAAkB,oBAAI,IAAoB;AAChD,eAAW,OAAO,cAAuB;AACvC,sBAAgB,IAAI,OAAO,IAAI,SAAS,GAAG,OAAO,IAAI,WAAW,CAAC;AAAA,IACpE;AAEA,UAAM,UAAU,oBAAI,IAAY;AAGhC,UAAM,oBAAoB,IAAI,OAAO,CAAC,OAAO;AAC3C,YAAM,aAAa,eAAe,IAAI,EAAE,KAAK;AAC7C,WAAK,gBAAgB,IAAI,EAAE,KAAK,OAAO,YAAY;AACjD,gBAAQ,IAAI,EAAE;AACd,eAAO;AAAA,MACT;AACA,aAAO,aAAa;AAAA,IACtB,CAAC;AAED,QAAI,kBAAkB,QAAQ;AAC5B,YAAM,YAAY,kBAAkB,OAAO,CAAC,KAAK,OAAO,OAAO,eAAe,IAAI,EAAE,KAAK,IAAI,CAAC;AAC9F,YAAM,SAAS,MAAM,GAClB,WAAW,eAAsB,EACjC,OAAO,CAAC,aAAoB,SAAgB,cAAqB,OAAc,CAAC,EAChF,MAAM,eAAsB,KAAK,SAAS,CAAC,EAAE,UAAU,EACvD,MAAM,2CAAoD,OAAO,cAAc,EAAE,EACjF,MAAM,qCAA8C,OAAO,QAAQ,EAAE,EACrE,MAAM,aAAoB,MAAM,iBAAiB,EAIjD,MAAM,SAAS,EACf,QAAQ;AACX,YAAM,cAAc,eAAe,QAAiB,CAAC,QAAQ,OAAO,IAAI,SAAS,CAAC;AAClF,iBAAW,MAAM,mBAAmB;AAClC,YAAI,CAAC,YAAY,WAAW,IAAI,GAAG,GAAG,IAAI,EAAE,EAAE,GAAG,YAAY,IAAI,EAAE,CAAC,EAAG,SAAQ,IAAI,EAAE;AAAA,MACvF;AAAA,IACF;AACA,uBAAmB,IAAI,KAAK,OAAO;AAAA,EACrC;AAEA,QAAM,oBAAoB,oBAAI,IAAY;AAC1C,aAAW,CAAC,KAAK,OAAO,KAAK,mBAAmB,QAAQ,GAAG;AACzD,eAAW,MAAM,QAAS,mBAAkB,IAAI,GAAG,GAAG,IAAI,EAAE,EAAE;AAAA,EAChE;AACA,QAAM,cAAc;AAAA,IAClB,YAAY,SAAS,CAAC,EAAE;AAAA,IACxB,aAAa,SAAS;AAAA,IACtB,cAAc,kBAAkB;AAAA,EAClC,CAAC;AACD,MAAI,CAAC,kBAAkB,KAAM;AAE7B,QAAM,GAAG,YAAY,EAAE,QAAQ,OAAO,QAAQ;AAC5C,eAAW,CAAC,KAAK,MAAM,KAAK,aAAa,QAAQ,GAAG;AAClD,YAAM,UAAU,mBAAmB,IAAI,GAAG;AAC1C,UAAI,CAAC,SAAS,KAAM;AAGpB,YAAM,cAAc,IACjB,WAAW,eAAsB,EACjC,MAAM,eAAsB,KAAK,SAAS,CAAC,EAAE,UAAU,EACvD,MAAM,2CAAoD,OAAO,cAAc,EAAE,EACjF,MAAM,qCAA8C,OAAO,QAAQ,EAAE,EACrE,MAAM,aAAoB,MAAM,MAAM,KAAK,OAAO,CAAC;AACtD,YAAM,YAAY,QAAQ;AAAA,IAC5B;AACA,UAAM,wBAAwB,KAC3B,OAAO,CAAC,QAAQ,kBAAkB,IAAI,YAAY,GAAG,CAAC,CAAC,EACvD,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,YAAY,WAAW,EAAE;AACpD,eAAW,SAAS,MAAM,uBAAuB,iBAAiB,GAAG;AACnE,YAAM,IAAI,WAAW,eAAsB,EAAE,OAAO,KAAY,EAAE,QAAQ;AAAA,IAC5E;AAAA,EACF,CAAC;AACH;",
   "names": []
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@open-mercato/core",
-  "version": "0.6.8-develop.7012.1.1dbd6f5fbd",
+  "version": "0.6.8-develop.7013.1.48f7adca80",
   "license": "MIT",
   "type": "module",
   "main": "./dist/index.js",
@@ -254,16 +254,16 @@
     "zod": "^4.4.3"
   },
   "peerDependencies": {
-    "@open-mercato/ai-assistant": "0.6.8-develop.7012.1.1dbd6f5fbd",
-    "@open-mercato/shared": "0.6.8-develop.7012.1.1dbd6f5fbd",
-    "@open-mercato/ui": "0.6.8-develop.7012.1.1dbd6f5fbd",
+    "@open-mercato/ai-assistant": "0.6.8-develop.7013.1.48f7adca80",
+    "@open-mercato/shared": "0.6.8-develop.7013.1.48f7adca80",
+    "@open-mercato/ui": "0.6.8-develop.7013.1.48f7adca80",
     "react": "^19.0.0",
     "react-dom": "^19.0.0"
   },
   "devDependencies": {
-    "@open-mercato/ai-assistant": "0.6.8-develop.7012.1.1dbd6f5fbd",
-    "@open-mercato/shared": "0.6.8-develop.7012.1.1dbd6f5fbd",
-    "@open-mercato/ui": "0.6.8-develop.7012.1.1dbd6f5fbd",
+    "@open-mercato/ai-assistant": "0.6.8-develop.7013.1.48f7adca80",
+    "@open-mercato/shared": "0.6.8-develop.7013.1.48f7adca80",
+    "@open-mercato/ui": "0.6.8-develop.7013.1.48f7adca80",
     "@testing-library/dom": "^10.4.1",
     "@testing-library/jest-dom": "^7.0.0",
     "@testing-library/react": "^16.3.1",

package/src/modules/query_index/lib/search-tokens.ts

@@ -223,6 +223,49 @@ export async function deleteSearchTokensForRecord(
     .execute()
 }
 
+// NUL, not a printable separator: a field name may itself contain a space, so `a b` + hash `c`
+// would otherwise sign identically to field `a` + hash `b c`.
+const SIGNATURE_SEPARATOR = String.fromCharCode(0)
+
+// Identifies one token row for comparison. `token` is NULL unless `storeRawTokens` is on, and a
+// stored NULL has to sign the same as the `null` a freshly built row carries — otherwise every
+// record compares as changed and the skip never fires.
+function tokenSignature(row: { field?: unknown; token_hash?: unknown; token?: unknown }): string {
+  return [
+    String(row.field ?? ''),
+    String(row.token_hash ?? ''),
+    row.token == null ? '' : String(row.token),
+  ].join(SIGNATURE_SEPARATOR)
+}
+
+// Multiplicities, not sets: #4681 reports token rows duplicated by the concurrent-replacement
+// defect, and a set comparison reads such a record as already correct and preserves the duplicates
+// forever. Counting sends it through a full rewrite, which collapses them.
+function tallyEquals(a: Map<string, number> | undefined, b: Map<string, number> | undefined): boolean {
+  const left = a ?? new Map<string, number>()
+  const right = b ?? new Map<string, number>()
+  if (left.size !== right.size) return false
+  for (const [key, count] of left.entries()) {
+    if (right.get(key) !== count) return false
+  }
+  return true
+}
+
+function tallyTokenRows<TRow extends { field?: unknown; token_hash?: unknown; token?: unknown }>(
+  rows: Iterable<TRow>,
+  keyOf: (row: TRow) => string
+): Map<string, Map<string, number>> {
+  const tallies = new Map<string, Map<string, number>>()
+  for (const row of rows) {
+    const key = keyOf(row)
+    const tally = tallies.get(key) ?? new Map<string, number>()
+    const signature = tokenSignature(row)
+    tally.set(signature, (tally.get(signature) ?? 0) + 1)
+    tallies.set(key, tally)
+  }
+  return tallies
+}
+
 export async function replaceSearchTokensForBatch(
   db: Kysely<any>,
   payloads: Array<BuildTokenOptions & { doc: Record<string, unknown> }>
@@ -256,8 +299,95 @@ export async function replaceSearchTokensForBatch(
     scopeBuckets.set(key, bucket)
   }
 
+  const recordKeyOf = (row: SearchTokenRow) =>
+    `${scopeKey(row.organization_id ?? null, row.tenant_id ?? null)}|${String(row.entity_id)}`
+  const builtTally = tallyTokenRows(rows, recordKeyOf)
+
+  // Read outside the transaction, deliberately. The comparison decides only whether to skip a
+  // rewrite, so a concurrent writer costs us at most a rewrite we declined — declined because the
+  // table already held exactly the rows this call wanted to write. One ordering is worth naming
+  // though: if the read matches and a concurrent writer then commits tokens built from a *staler*
+  // doc, the unconditional rewrite this call used to perform would have overwritten them by
+  // accident. It no longer does, so those stale rows survive until the record's next write. That
+  // is a repair we lose, not a guarantee we break.
+  const changedIdsByBucket = new Map<string, Set<string>>()
+  for (const [key, bucket] of scopeBuckets.entries()) {
+    const ids = Array.from(bucket.ids)
+    const builtCountById = new Map<string, number>()
+    for (const id of ids) {
+      let total = 0
+      const tally = builtTally.get(`${key}|${id}`)
+      if (tally) for (const count of tally.values()) total += count
+      builtCountById.set(id, total)
+    }
+
+    // Count probe first. Its result is one row per record in the batch, so it is bounded by the
+    // batch size — unlike a bare row read, which would be bounded only by how many token rows the
+    // table already holds for these ids, a quantity this function does not control and (per #4681)
+    // has no reason to trust.
+    const storedCounts = await db
+      .selectFrom('search_tokens' as any)
+      .select(['entity_id' as any, sql<number>`count(*)`.as('token_count') as any])
+      .where('entity_type' as any, '=', payloads[0].entityType)
+      .where(sql<boolean>`organization_id is not distinct from ${bucket.organizationId}`)
+      .where(sql<boolean>`tenant_id is not distinct from ${bucket.tenantId}`)
+      .where('entity_id' as any, 'in', ids)
+      .groupBy('entity_id' as any)
+      .execute()
+    const storedCountById = new Map<string, number>()
+    for (const row of storedCounts as any[]) {
+      storedCountById.set(String(row.entity_id), Number(row.token_count))
+    }
+
+    const changed = new Set<string>()
+    // A record whose stored row count already differs is changed, whatever the rows say — the
+    // duplicate case from #4681 resolves here without ever materializing the duplicated rows.
+    const contentCandidates = ids.filter((id) => {
+      const builtCount = builtCountById.get(id) ?? 0
+      if ((storedCountById.get(id) ?? 0) !== builtCount) {
+        changed.add(id)
+        return false
+      }
+      return builtCount > 0
+    })
+
+    if (contentCandidates.length) {
+      const rowBudget = contentCandidates.reduce((sum, id) => sum + (builtCountById.get(id) ?? 0), 0)
+      const stored = await db
+        .selectFrom('search_tokens' as any)
+        .select(['entity_id' as any, 'field' as any, 'token_hash' as any, 'token' as any])
+        .where('entity_type' as any, '=', payloads[0].entityType)
+        .where(sql<boolean>`organization_id is not distinct from ${bucket.organizationId}`)
+        .where(sql<boolean>`tenant_id is not distinct from ${bucket.tenantId}`)
+        .where('entity_id' as any, 'in', contentCandidates)
+        // Counts already match, so this cannot truncate — it bounds the damage if a concurrent
+        // writer inserts between the probe and this read. A truncated read compares as changed,
+        // which costs a rewrite rather than a wrong skip.
+        .limit(rowBudget)
+        .execute()
+      const storedTally = tallyTokenRows(stored as any[], (row) => String(row.entity_id))
+      for (const id of contentCandidates) {
+        if (!tallyEquals(builtTally.get(`${key}|${id}`), storedTally.get(id))) changed.add(id)
+      }
+    }
+    changedIdsByBucket.set(key, changed)
+  }
+
+  const changedRecordKeys = new Set<string>()
+  for (const [key, changed] of changedIdsByBucket.entries()) {
+    for (const id of changed) changedRecordKeys.add(`${key}|${id}`)
+  }
+  debug('batch.skip', {
+    entityType: payloads[0].entityType,
+    recordCount: payloads.length,
+    changedCount: changedRecordKeys.size,
+  })
+  if (!changedRecordKeys.size) return
+
   await db.transaction().execute(async (trx) => {
-    for (const [, bucket] of scopeBuckets.entries()) {
+    for (const [key, bucket] of scopeBuckets.entries()) {
+      const changed = changedIdsByBucket.get(key)
+      if (!changed?.size) continue
       // Delete by entity_id: a batch replaces all of a record's tokens, and a per-field OR over the
       // whole batch overflows the query compiler's call stack on large batches.
       const deleteQuery = trx
@@ -265,10 +395,12 @@ export async function replaceSearchTokensForBatch(
         .where('entity_type' as any, '=', payloads[0].entityType)
         .where(sql<boolean>`organization_id is not distinct from ${bucket.organizationId}`)
         .where(sql<boolean>`tenant_id is not distinct from ${bucket.tenantId}`)
-        .where('entity_id' as any, 'in', Array.from(bucket.ids))
+        .where('entity_id' as any, 'in', Array.from(changed))
       await deleteQuery.execute()
     }
-    const payloadWithTimestamps = rows.map((row) => ({ ...row, created_at: sql`now()` }))
+    const payloadWithTimestamps = rows
+      .filter((row) => changedRecordKeys.has(recordKeyOf(row)))
+      .map((row) => ({ ...row, created_at: sql`now()` }))
     for (const batch of chunk(payloadWithTimestamps, INSERT_BATCH_SIZE)) {
       await trx.insertInto('search_tokens' as any).values(batch as any).execute()
     }