@open-mercato/core 0.6.7-develop.6653.1.b5bc7194a0 → 0.6.7-develop.6661.1.0043ed1d03

package/src/modules/query_index/lib/batch.ts

@@ -1,11 +1,20 @@
 import { type Kysely, sql } from 'kysely'
 import { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'
+import { isUniqueViolation } from '@open-mercato/shared/lib/db/pg-errors'
 import { buildIndexDocument, type IndexCustomFieldValue } from './document'
 import { replaceSearchTokensForBatch, isSearchDebugEnabled } from './search-tokens'
 import { createLogger } from '@open-mercato/shared/lib/logger'
 
 const logger = createLogger('query_index').child({ component: 'reindex-batch' })
 
+/**
+ * Cap on how many per-row failures a single batch persists to `indexer_error_logs`.
+ * Under a systemic failure (pool exhausted, disk full, KMS down) every row fails, and
+ * writing one error row per record would hammer the database that is already failing.
+ * The count of suppressed failures still reaches the caller via `failedRecordIds`.
+ */
+const MAX_RECORDED_ROW_ERRORS_PER_BATCH = 50
+
 export type AnyRow = Record<string, any> & { id: string | number }
 
 export type ScopeOverrides = {
@@ -13,6 +22,59 @@ export type ScopeOverrides = {
   tenantId?: string
 }
 
+/**
+ * What a batch actually wrote, as opposed to what it was asked to write. Callers MUST
+ * reconcile `written` against `attempted` — a batch that loses rows and reports success
+ * is how records go permanently missing from the index (issue GSM-266).
+ */
+export type UpsertIndexBatchResult = {
+  attempted: number
+  written: number
+  failedRecordIds: string[]
+  searchTokenFailures: number
+}
+
+export class QueryIndexBatchWriteError extends Error {
+  readonly entityType: string
+  readonly attempted: number
+  readonly written: number
+  readonly failedRecordIds: string[]
+
+  constructor(entityType: string, result: UpsertIndexBatchResult) {
+    const preview = result.failedRecordIds.slice(0, 10)
+    const suffix = result.failedRecordIds.length > preview.length ? ', ...' : ''
+    super(
+      `[internal] Query index write incomplete for ${entityType}: wrote ${result.written} of ${result.attempted} records` +
+        (preview.length ? ` (failed: ${preview.join(', ')}${suffix})` : ''),
+    )
+    this.name = 'QueryIndexBatchWriteError'
+    this.entityType = entityType
+    this.attempted = result.attempted
+    this.written = result.written
+    this.failedRecordIds = result.failedRecordIds
+  }
+}
+
+export function assertIndexBatchWritesLanded(entityType: string, result: UpsertIndexBatchResult): void {
+  if (result.written >= result.attempted && result.failedRecordIds.length === 0) return
+  throw new QueryIndexBatchWriteError(entityType, result)
+}
+
+export function createEmptyUpsertIndexBatchResult(): UpsertIndexBatchResult {
+  return { attempted: 0, written: 0, failedRecordIds: [], searchTokenFailures: 0 }
+}
+
+export function mergeUpsertIndexBatchResults(
+  target: UpsertIndexBatchResult,
+  addition: UpsertIndexBatchResult,
+): UpsertIndexBatchResult {
+  target.attempted += addition.attempted
+  target.written += addition.written
+  target.searchTokenFailures += addition.searchTokenFailures
+  target.failedRecordIds.push(...addition.failedRecordIds)
+  return target
+}
+
 type CustomFieldRow = {
   record_id: string
   field_key: string
@@ -48,16 +110,73 @@ function normalizeScopedValue(value: unknown): string | null {
   return String(value)
 }
 
+type IndexRowPayload = {
+  entity_type: string
+  entity_id: string
+  organization_id: string | null
+  tenant_id: string | null
+  doc: Record<string, unknown>
+  tokenDoc: Record<string, unknown>
+  index_version: number
+}
+
+async function updateIndexRow(db: Kysely<any>, payload: IndexRowPayload): Promise<number> {
+  let updateQuery = db
+    .updateTable('entity_indexes' as any)
+    .set({
+      doc: sql`${JSON.stringify(payload.doc)}::jsonb`,
+      index_version: payload.index_version,
+      organization_id: payload.organization_id ?? null,
+      tenant_id: payload.tenant_id ?? null,
+      updated_at: sql`now()`,
+      deleted_at: null,
+    } as any)
+    .where('entity_type' as any, '=', payload.entity_type)
+    .where('entity_id' as any, '=', payload.entity_id)
+  updateQuery = payload.organization_id == null
+    ? updateQuery.where('organization_id' as any, 'is', null as any)
+    : updateQuery.where('organization_id' as any, '=', payload.organization_id)
+  const result = await updateQuery.executeTakeFirst() as { numUpdatedRows?: bigint | number } | undefined
+  return Number(result?.numUpdatedRows ?? 0)
+}
+
 export async function upsertIndexBatch(
   db: Kysely<any>,
   entityType: string,
   rows: AnyRow[],
   scope: ScopeOverrides,
   options: IndexBatchOptions = {},
-): Promise<void> {
-  if (!rows.length) return
+): Promise<UpsertIndexBatchResult> {
+  if (!rows.length) return createEmptyUpsertIndexBatchResult()
   const recordIds = rows.map((row) => normalizeId(row.id))
 
+  const failedRecordIds: string[] = []
+  let searchTokenFailures = 0
+  let recordedRowErrors = 0
+
+  const recordRowFailure = async (
+    recordId: string,
+    error: unknown,
+    handler: string,
+    rowScope: { organizationId: string | null; tenantId: string | null },
+  ): Promise<void> => {
+    failedRecordIds.push(recordId)
+    if (recordedRowErrors >= MAX_RECORDED_ROW_ERRORS_PER_BATCH) return
+    recordedRowErrors += 1
+    await recordIndexerError(
+      { db },
+      {
+        source: 'query_index',
+        handler,
+        error,
+        entityType,
+        recordId,
+        tenantId: rowScope.tenantId,
+        organizationId: rowScope.organizationId,
+      },
+    ).catch(() => undefined)
+  }
+
   const shouldMergeCustomerEntity =
     entityType === 'customers:customer_person_profile' || entityType === 'customers:customer_company_profile'
 
@@ -106,15 +225,7 @@ export async function upsertIndexBatch(
     else customFieldMap.set(key, [fieldRow])
   }
 
-  const basePayloads: Array<{
-    entity_type: string
-    entity_id: string
-    organization_id: string | null
-    tenant_id: string | null
-    doc: Record<string, unknown>
-    tokenDoc: Record<string, unknown>
-    index_version: number
-  }> = []
+  const basePayloads: IndexRowPayload[] = []
 
   const debugEnabled = isSearchDebugEnabled()
 
@@ -168,8 +279,15 @@ export async function upsertIndexBatch(
           doc = encrypted
           tokenDoc = encrypted
         }
-      } catch {
-        // best-effort; ignore encrypt errors during indexing
+      } catch (encryptError) {
+        // Falling through would write the plaintext document into entity_indexes,
+        // which must stay encrypted at rest. Skip the row instead so the previously
+        // encrypted row survives, and let the caller fail the run.
+        await recordRowFailure(recordId, encryptError, 'query_index:reindex-batch:encrypt', {
+          organizationId: scopeOrg ?? null,
+          tenantId: scopeTenant ?? null,
+        })
+        continue
       }
     }
     if (typeof options.decryptDoc === 'function') {
@@ -181,8 +299,22 @@ export async function upsertIndexBatch(
         if (decrypted && typeof decrypted === 'object') {
           tokenDoc = decrypted
         }
-      } catch {
-        // best-effort; ignore decrypt errors during indexing
+      } catch (decryptError) {
+        // Only affects the search tokens built below; the indexed document itself is
+        // unharmed, so the row still counts as written.
+        logger.warn('Failed to decrypt index document for search tokens', { entityType, recordId })
+        await recordIndexerError(
+          { db },
+          {
+            source: 'fulltext',
+            handler: 'query_index:reindex-batch:decrypt',
+            error: decryptError,
+            entityType,
+            recordId,
+            tenantId: scopeTenant ?? null,
+            organizationId: scopeOrg ?? null,
+          },
+        ).catch(() => undefined)
       }
     }
     basePayloads.push({
@@ -232,7 +364,37 @@ export async function upsertIndexBatch(
     doc: payload.tokenDoc,
   }))
 
+  const writeSearchTokens = async (payloads = tokenPayloads): Promise<void> => {
+    if (!payloads.length) return
+    try {
+      await replaceSearchTokensForBatch(db, payloads)
+    } catch (searchTokenError) {
+      // Record instead of swallowing: a failed token write leaves fulltext search stale.
+      // Not counted as a write failure — replaceSearchTokensForBatch is transactional, so
+      // tokens end up stale rather than missing, and the record's next write rebuilds them.
+      searchTokenFailures += 1
+      await recordIndexerError(
+        { db },
+        { source: 'fulltext', handler: 'query_index:reindex-batch', error: searchTokenError, entityType, tenantId: scope.tenantId ?? null, organizationId: scope.orgId ?? null },
+      ).catch(() => undefined)
+    }
+  }
+
+  const buildResult = (written: number): UpsertIndexBatchResult => ({
+    attempted: rows.length,
+    written,
+    failedRecordIds,
+    searchTokenFailures,
+  })
+
+  if (!basePayloads.length) return buildResult(0)
+
+  let bulkWriteFailed = false
   try {
+    // One statement, so it is atomic on its own. Note the arbiter is
+    // (entity_type, entity_id, organization_id_coalesced): two payloads sharing a
+    // conflict key in the same batch would raise 21000 for the whole statement and
+    // fall through to the per-row path below.
     await db
       .insertInto('entity_indexes' as any)
       .values(insertRows as any)
@@ -247,15 +409,27 @@ export async function upsertIndexBatch(
           updated_at: sql`now()`,
         } as any))
       .execute()
-    try {
-      await replaceSearchTokensForBatch(db, tokenPayloads)
-    } catch (searchTokenError) {
-      // Record instead of swallowing: a failed token write leaves fulltext search stale.
-      await recordIndexerError(
-        { db },
-        { source: 'fulltext', handler: 'query_index:reindex-batch', error: searchTokenError, entityType, tenantId: scope.tenantId ?? null, organizationId: scope.orgId ?? null },
-      ).catch(() => undefined)
-    }
+  } catch (bulkError) {
+    bulkWriteFailed = true
+    logger.warn('Bulk index upsert failed; falling back to per-row writes', {
+      entityType,
+      records: basePayloads.length,
+    })
+    await recordIndexerError(
+      { db },
+      {
+        source: 'query_index',
+        handler: 'query_index:reindex-batch:bulk',
+        error: bulkError,
+        entityType,
+        tenantId: scope.tenantId ?? null,
+        organizationId: scope.orgId ?? null,
+      },
+    ).catch(() => undefined)
+  }
+
+  if (!bulkWriteFailed) {
+    await writeSearchTokens()
     if (debugEnabled) {
       logger.debug('Reindex batch tokens', {
         entityType,
@@ -264,55 +438,58 @@ export async function upsertIndexBatch(
         scopeTenant: scope.tenantId ?? null,
       })
     }
-    return
-  } catch {
-    await db.transaction().execute(async (trx) => {
-      for (const payload of basePayloads) {
-        let updateQuery = trx
-          .updateTable('entity_indexes' as any)
-          .set({
+    return buildResult(basePayloads.length)
+  }
+
+  // Deliberately NOT wrapped in a transaction. In Postgres a single failing statement
+  // aborts the surrounding transaction, and COMMIT on an aborted transaction answers
+  // with a ROLLBACK tag instead of raising — so one bad row used to discard the entire
+  // batch while this function reported success (issue GSM-266). Each row here is an
+  // independent, idempotent upsert; they share no invariant worth a transaction.
+  let written = 0
+  for (const payload of basePayloads) {
+    try {
+      if (await updateIndexRow(db, payload) > 0) {
+        written += 1
+        continue
+      }
+      try {
+        await db
+          .insertInto('entity_indexes' as any)
+          .values({
+            entity_type: payload.entity_type,
+            entity_id: payload.entity_id,
+            organization_id: payload.organization_id,
+            tenant_id: payload.tenant_id,
             doc: sql`${JSON.stringify(payload.doc)}::jsonb`,
             index_version: payload.index_version,
-            organization_id: payload.organization_id ?? null,
-            tenant_id: payload.tenant_id ?? null,
+            created_at: sql`now()`,
             updated_at: sql`now()`,
             deleted_at: null,
           } as any)
-          .where('entity_type' as any, '=', payload.entity_type)
-          .where('entity_id' as any, '=', payload.entity_id)
-        updateQuery = payload.organization_id == null
-          ? updateQuery.where('organization_id' as any, 'is', null as any)
-          : updateQuery.where('organization_id' as any, '=', payload.organization_id)
-        const result = await updateQuery.executeTakeFirst() as { numUpdatedRows?: bigint | number } | undefined
-        if (result && Number(result.numUpdatedRows ?? 0) > 0) continue
-        try {
-          await trx
-            .insertInto('entity_indexes' as any)
-            .values({
-              entity_type: payload.entity_type,
-              entity_id: payload.entity_id,
-              organization_id: payload.organization_id,
-              tenant_id: payload.tenant_id,
-              doc: sql`${JSON.stringify(payload.doc)}::jsonb`,
-              index_version: payload.index_version,
-              created_at: sql`now()`,
-              updated_at: sql`now()`,
-              deleted_at: null,
-            } as any)
-            .execute()
-        } catch {
-          // ignore duplicate insert race; another concurrent worker updated the row
+          .execute()
+        written += 1
+      } catch (insertError) {
+        // A concurrent worker may have inserted the row between our UPDATE and INSERT.
+        // Only a successful retry of the UPDATE proves the row is actually there.
+        if (isUniqueViolation(insertError) && await updateIndexRow(db, payload) > 0) {
+          written += 1
+          continue
         }
+        await recordRowFailure(payload.entity_id, insertError, 'query_index:reindex-batch:row', {
+          organizationId: payload.organization_id,
+          tenantId: payload.tenant_id,
+        })
       }
-    })
-  }
-  try {
-    await replaceSearchTokensForBatch(db, tokenPayloads)
-  } catch (searchTokenError) {
-    // Record instead of swallowing: a failed token write leaves fulltext search stale.
-    await recordIndexerError(
-      { db },
-      { source: 'fulltext', handler: 'query_index:reindex-batch', error: searchTokenError, entityType, tenantId: scope.tenantId ?? null, organizationId: scope.orgId ?? null },
-    ).catch(() => undefined)
+    } catch (rowError) {
+      await recordRowFailure(payload.entity_id, rowError, 'query_index:reindex-batch:row', {
+        organizationId: payload.organization_id,
+        tenantId: payload.tenant_id,
+      })
+    }
   }
+
+  const failedRecordIdSet = new Set(failedRecordIds)
+  await writeSearchTokens(tokenPayloads.filter((payload) => !failedRecordIdSet.has(payload.recordId)))
+  return buildResult(written)
 }

package/src/modules/query_index/lib/jobs.ts

@@ -135,14 +135,25 @@ export async function updateJobProgress(
   ).execute()
 }
 
+export type FinalizeJobOptions = {
+  /**
+   * Marks the finished job as failed. `finished_at` alone cannot express this — readers
+   * derive "completed" from its presence — so a run that lost records would otherwise
+   * still report success in the status API and UI.
+   */
+  status?: 'failed'
+}
+
 export async function finalizeJob(
   db: Kysely<any>,
   scope: JobScope,
+  options: FinalizeJobOptions = {},
 ): Promise<void> {
   await applyScopeWhere(
     db.updateTable('entity_index_jobs' as any).set({
       finished_at: sql`now()`,
       heartbeat_at: sql`now()`,
+      ...(options.status ? { status: options.status } : {}),
     } as any) as any,
     scope,
   ).execute()

package/src/modules/query_index/lib/reindexer.ts

@@ -3,7 +3,14 @@ import { type Kysely, sql } from 'kysely'
 import { resolveRegisteredEntityTableName } from '@open-mercato/shared/lib/query/engine'
 import { resolveTenantEncryptionService } from '@open-mercato/shared/lib/encryption/customFieldValues'
 import { decryptIndexDocForSearch, encryptIndexDocForStorage } from '@open-mercato/shared/lib/encryption/indexDoc'
-import { upsertIndexBatch, type AnyRow } from './batch'
+import {
+  upsertIndexBatch,
+  assertIndexBatchWritesLanded,
+  createEmptyUpsertIndexBatchResult,
+  mergeUpsertIndexBatchResults,
+  QueryIndexBatchWriteError,
+  type AnyRow,
+} from './batch'
 import { refreshCoverageSnapshot, writeCoverageCounts, applyCoverageAdjustments } from './coverage'
 import { prepareJob, updateJobProgress, finalizeJob, type JobScope } from './jobs'
 import { purgeOrphans } from './stale'
@@ -39,6 +46,12 @@ export type ReindexJobResult = {
 
 export const DEFAULT_REINDEX_PARTITIONS = 5
 const DEFAULT_BATCH_SIZE = 500
+/**
+ * Above this many failed records the purge exclusion list stops being a sane query, so
+ * the purge is skipped entirely instead. Failing closed keeps stale rows; the alternative
+ * deletes index entries the run failed to rebuild.
+ */
+const MAX_PURGE_EXCLUSIONS = 1000
 const deriveOrgFromId = new Set<string>(['directory:organization'])
 const COVERAGE_REFRESH_THROTTLE_MS = 5 * 60 * 1000
 const lastCoverageReset = new Map<string, number>()
@@ -305,6 +318,8 @@ export async function reindexEntity(
 
   let processed = 0
   let lastId: string | null = null
+  let jobFailed = false
+  const writeTotals = createEmptyUpsertIndexBatchResult()
 
   options?.onProgress?.({ processed, total, chunkSize: 0 })
 
@@ -422,10 +437,20 @@ export async function reindexEntity(
         return result
       }
 
-      await upsertIndexBatch(db, entityType, rows, scopeOverrides, { deriveOrganizationId: deriveOrg, encryptDoc, decryptDoc })
+      const batchResult = await upsertIndexBatch(db, entityType, rows, scopeOverrides, { deriveOrganizationId: deriveOrg, encryptDoc, decryptDoc })
+      mergeUpsertIndexBatchResults(writeTotals, batchResult)
+
+      // A whole batch failing is infrastructural (pool exhausted, disk full, KMS down),
+      // not a poison record. Abort now instead of grinding through the rest of the table.
+      if (batchResult.written === 0 && batchResult.attempted > 0) {
+        throw new QueryIndexBatchWriteError(entityType, writeTotals)
+      }
+
+      const failedInBatch = new Set(batchResult.failedRecordIds)
+      const writtenRows = failedInBatch.size ? rows.filter((row) => !failedInBatch.has(String(row.id))) : rows
 
       const coverageDeltas = new Map<string, { tenantId: string | null; organizationId: string | null; delta: number }>()
-      for (const row of rows) {
+      for (const row of writtenRows) {
         const scopeTenant = tenantId !== undefined
           ? tenantId ?? null
           : (hasTenantCol ? ((row as AnyRow).tenant_id ?? null) : null)
@@ -457,7 +482,7 @@ export async function reindexEntity(
 
       if (emitVectorize && eventBus) {
         await Promise.all(
-          rows.map((row) => {
+          writtenRows.map((row) => {
             const scopeOrg = organizationId !== undefined
               ? organizationId ?? null
               : hasOrgCol
@@ -478,20 +503,32 @@ export async function reindexEntity(
         )
       }
 
-      processed += rows.length
+      processed += batchResult.written
       lastId = String(rows[rows.length - 1]!.id)
-      options?.onProgress?.({ processed, total, chunkSize: rows.length })
-      await updateJobProgress(db, jobScope, rows.length)
+      options?.onProgress?.({ processed, total, chunkSize: batchResult.written })
+      await updateJobProgress(db, jobScope, batchResult.written)
     }
 
-    await purgeOrphans(db, {
-      entityType,
-      tenantId,
-      organizationId,
-      partitionIndex: usingPartitions ? partitionIndex : null,
-      partitionCount: usingPartitions ? partitionCountRaw : null,
-      startedAt: jobStartedAt,
-    })
+    // Records this run failed to write still look untouched to the purge predicate, so
+    // without the exclusion the purge would delete the index rows it just failed to
+    // rebuild — turning a stale entry into a missing one.
+    const purgeExclusions = writeTotals.failedRecordIds
+    if (purgeExclusions.length > MAX_PURGE_EXCLUSIONS) {
+      logger.warn('Skipping orphan purge after widespread write failures', {
+        entityType,
+        failedRecords: purgeExclusions.length,
+      })
+    } else {
+      await purgeOrphans(db, {
+        entityType,
+        tenantId,
+        organizationId,
+        partitionIndex: usingPartitions ? partitionIndex : null,
+        partitionCount: usingPartitions ? partitionCountRaw : null,
+        startedAt: jobStartedAt,
+        excludeRecordIds: purgeExclusions,
+      })
+    }
 
     if (force && vectorService && (!usingPartitions || partitionIndex === null)) {
       try {
@@ -522,8 +559,24 @@ export async function reindexEntity(
         },
       )
     }
+
+    // Deliberately after the coverage refresh: the authoritative recount is what keeps
+    // indexed_count truthful, and it is most worth having when a run has just failed.
+    // Throwing here fails the queue job so the loss is visible instead of silent.
+    if (writeTotals.searchTokenFailures > 0) {
+      logger.warn('Search token writes failed during reindex', {
+        entityType,
+        batches: writeTotals.searchTokenFailures,
+      })
+    }
+    assertIndexBatchWritesLanded(entityType, writeTotals)
+  } catch (error) {
+    jobFailed = true
+    throw error
   } finally {
-    await finalizeJob(db, jobScope)
+    // Still finalized on failure: the scope stays wedged behind the active-job guard
+    // while finished_at is null. The status carries the outcome instead.
+    await finalizeJob(db, jobScope, jobFailed ? { status: 'failed' } : {})
   }
 
   return {

package/src/modules/query_index/lib/stale.ts

@@ -7,6 +7,13 @@ type PurgeOrphansOptions = {
   partitionIndex: number | null
   partitionCount: number | null
   startedAt: Date
+  /**
+   * Records whose index rows must be preserved even though this run did not refresh
+   * them. A row that failed to be written still looks untouched to the `updated_at`
+   * predicate below, so without this the purge would delete the very entries the run
+   * failed to rebuild.
+   */
+  excludeRecordIds?: string[]
 }
 
 export async function purgeOrphans(
@@ -25,5 +32,8 @@ export async function purgeOrphans(
     q = q.where(sql<boolean>`mod(abs(hashtext(entity_id::text)), ${partitionCount}) = ${partitionIndex}`)
   }
   q = q.where('updated_at' as any, '<', startedAt as any)
+  if (options.excludeRecordIds?.length) {
+    q = q.where('entity_id' as any, 'not in', options.excludeRecordIds)
+  }
   await q.execute()
 }