@open-mercato/core 0.6.7-develop.6842.1.8cd8a8883c → 0.6.7-develop.6859.1.e4727b568a

package/src/modules/query_index/cli.ts

@@ -11,7 +11,7 @@ type ProgressBarHandle = {
   update(completed: number): void
   complete(): void
 }
-import { resolveEntityTableName } from '@open-mercato/shared/lib/query/engine'
+import { resolveEntityTableName, resolveRegisteredEntityTableName } from '@open-mercato/shared/lib/query/engine'
 import { recordIndexerError } from '@open-mercato/shared/lib/indexers/error-log'
 import { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'
 import {
@@ -23,6 +23,7 @@ import {
 } from './lib/batch'
 import { reindexEntity, DEFAULT_REINDEX_PARTITIONS } from './lib/reindexer'
 import { purgeIndexScope } from './lib/purge'
+import { refreshCoverageSnapshot } from './lib/coverage'
 import { flattenSystemEntityIds } from '@open-mercato/shared/lib/entities/system-entities'
 import type { VectorIndexService } from '@open-mercato/search/vector'
 
@@ -340,6 +341,168 @@ function describeScope(scope: ScopeDescriptor): string {
   return parts.length ? ` (${parts.join(' ')})` : ''
 }
 
+/**
+ * Above this many missing records the anti-join stops being the cheaper option and the
+ * repair falls back to a full single-partition rebuild of the entity.
+ */
+const MAX_TARGETED_REPAIR_RECORDS = 5000
+
+/**
+ * List the base records in scope that have no live index row, so a repair rebuilds only
+ * what is actually missing instead of re-upserting the whole entity for a one-row gap.
+ */
+async function findUnindexedRecordIds(
+  db: Kysely<any>,
+  options: {
+    entityType: string
+    tableName: string
+    tenantId?: string | null
+    organizationId?: string | null
+    limit: number
+  },
+): Promise<string[]> {
+  const columns = await getColumnSet(db, options.tableName)
+  let query = db
+    .selectFrom(`${options.tableName} as b` as any)
+    .select('b.id' as any)
+    .where(({ not, exists, selectFrom }: any) =>
+      not(exists(
+        selectFrom('entity_indexes as ei' as any)
+          .select(sql`1`.as('one'))
+          .whereRef('ei.entity_id' as any, '=', sql`b.id::text`)
+          .where('ei.entity_type' as any, '=', options.entityType)
+          .where('ei.deleted_at' as any, 'is', null as any),
+      )),
+    )
+    .limit(options.limit)
+  if (columns.has('deleted_at')) query = query.where('b.deleted_at' as any, 'is', null as any)
+  if (options.tenantId != null && columns.has('tenant_id')) {
+    query = query.where('b.tenant_id' as any, '=', options.tenantId)
+  }
+  if (options.organizationId != null && columns.has('organization_id')) {
+    query = query.where('b.organization_id' as any, '=', options.organizationId)
+  }
+  const rows = await query.execute() as Array<{ id: unknown }>
+  return rows.map((row) => String(row.id))
+}
+
+/**
+ * Recount the scope from the database after a reindex and rebuild whatever is still
+ * missing. Runs unconditionally so `mercato init` (and any scripted reindex) cannot finish
+ * leaving a gap that a human has to notice and repair by hand from the Query Indexes
+ * screen. When counts already agree this is two COUNT queries; when they do not, the
+ * repair is scoped to the records the anti-join says are actually missing.
+ */
+async function verifyAndRepairIndexCoverage(
+  em: EntityManager,
+  options: {
+    entityType: string
+    tenantId?: string | null
+    organizationId?: string | null
+    batchSize?: number
+  },
+): Promise<void> {
+  const scope = {
+    entityType: options.entityType,
+    tenantId: options.tenantId ?? null,
+    organizationId: options.organizationId ?? null,
+    withDeleted: false,
+  }
+
+  let counts: { baseCount: number; indexedCount: number } | null = null
+  try {
+    counts = await refreshCoverageSnapshot(em, scope)
+  } catch (error) {
+    console.warn(
+      `  -> ${options.entityType}: coverage verification skipped (${error instanceof Error ? error.message : String(error)})`,
+    )
+    return
+  }
+  if (!counts) return
+  if (counts.indexedCount > counts.baseCount) {
+    // Legitimate for entities whose base table cannot express the scope, since a per-tenant
+    // reindex stamps its own scope onto every row. Surfaced rather than repaired: a rebuild
+    // cannot remove the surplus, and the over-count would otherwise mask a real shortfall.
+    console.warn(
+      `  -> ${options.entityType}: ${counts.indexedCount} index rows for ${counts.baseCount} records — not repairable by reindexing`,
+    )
+    return
+  }
+  if (counts.indexedCount === counts.baseCount) return
+
+  const db = (em as any).getKysely() as Kysely<any>
+  let tableName: string | null = null
+  try {
+    tableName = resolveRegisteredEntityTableName(em as any, options.entityType)
+  } catch {
+    tableName = null
+  }
+
+  const missingIds = tableName
+    ? await findUnindexedRecordIds(db, {
+      entityType: options.entityType,
+      tableName,
+      tenantId: options.tenantId ?? null,
+      organizationId: options.organizationId ?? null,
+      limit: MAX_TARGETED_REPAIR_RECORDS + 1,
+    }).catch(() => null)
+    : null
+
+  const targeted = missingIds && missingIds.length > 0 && missingIds.length <= MAX_TARGETED_REPAIR_RECORDS
+  console.log(
+    `  -> ${options.entityType}: ${counts.indexedCount}/${counts.baseCount} indexed after reindex — repairing`
+    + `${targeted ? ` ${missingIds!.length} missing record(s)` : ' the whole entity'}...`,
+  )
+
+  try {
+    if (targeted) {
+      const columns = await getColumnSet(db, tableName!)
+      for (const recordId of missingIds!) {
+        await rebuildEntityIndexes({
+          em,
+          db,
+          entityType: options.entityType,
+          tableName: tableName!,
+          orgOverride: options.organizationId ?? undefined,
+          tenantOverride: options.tenantId ?? undefined,
+          global: false,
+          includeDeleted: false,
+          offset: 0,
+          recordId,
+          batchSize: 1,
+          supportsOrgFilter: columns.has('organization_id'),
+          supportsTenantFilter: columns.has('tenant_id'),
+          supportsDeletedFilter: columns.has('deleted_at'),
+        })
+      }
+    } else {
+      await reindexEntity(em, {
+        entityType: options.entityType,
+        tenantId: options.tenantId,
+        organizationId: options.organizationId,
+        force: false,
+        batchSize: options.batchSize,
+        emitVectorizeEvents: false,
+        resetCoverage: false,
+      })
+    }
+  } catch (error) {
+    console.warn(
+      `  -> ${options.entityType}: repair pass failed (${error instanceof Error ? error.message : String(error)})`,
+    )
+    return
+  }
+
+  const after = await refreshCoverageSnapshot(em, scope).catch(() => null)
+  if (after && after.indexedCount < after.baseCount) {
+    console.warn(
+      `  -> ${options.entityType}: still ${after.indexedCount}/${after.baseCount} indexed after repair`,
+    )
+  } else {
+    console.log(`  -> ${options.entityType}: coverage repaired`)
+  }
+}
+
 const rebuild: ModuleCli = {
   command: 'rebuild',
   async run(rest) {
@@ -710,6 +873,14 @@ const reindex: ModuleCli = {
         groupedProgress?.complete()
         const totalProcessed = stats.reduce((acc, value) => acc + value, 0)
         console.log(`Finished ${entity}: processed ${totalProcessed} row(s) across ${partitionTargets.length} partition(s)`)
+        if (partitionIndexOption === undefined) {
+          await verifyAndRepairIndexCoverage(baseEm, {
+            entityType: entity,
+            tenantId,
+            organizationId: orgId,
+            batchSize,
+          })
+        }
         await recordIndexerLog(
           { em: baseEm },
           {
@@ -848,6 +1019,14 @@ const reindex: ModuleCli = {
         groupedProgress?.complete()
         const totalProcessed = partitionResults.reduce((acc, value) => acc + value, 0)
         console.log(`  -> ${id} complete: processed ${totalProcessed} row(s) across ${partitionTargets.length} partition(s)`)
+        if (partitionIndexOption === undefined) {
+          await verifyAndRepairIndexCoverage(baseEm, {
+            entityType: id,
+            tenantId,
+            organizationId: orgId,
+            batchSize,
+          })
+        }
         await recordIndexerLog(
           { em: baseEm },
           {

package/src/modules/query_index/lib/coverage.ts

@@ -294,6 +294,19 @@ export async function deleteCoverageForEntity(db: Kysely<any>, entityType: strin
     .execute()
 }
 
+async function deleteCoverageScope(db: Kysely<any>, scope: CoverageScope): Promise<void> {
+  const { entityType, tenantId, organizationId, withDeleted } = scope
+  if (!entityType) return
+  let query = db
+    .deleteFrom('entity_index_coverage' as any)
+    .where('entity_type' as any, '=', entityType)
+    .where('with_deleted' as any, '=', withDeleted === true)
+  query = tenantId == null
+    ? query.where('tenant_id' as any, 'is', null as any)
+    : query.where('tenant_id' as any, '=', tenantId)
+  await applyOrganizationCondition(query as any, 'organization_id', organizationId ?? null).execute()
+}
+
 async function tableHasColumn(db: Kysely<any>, table: string, column: string): Promise<boolean> {
   const key = `${table}.${column}`
   if (COLUMN_CACHE.has(key)) return COLUMN_CACHE.get(key)!
@@ -381,9 +394,9 @@ export async function primeColumnCache(db: Kysely<any>, checks: ColumnCheck[]):
 export async function refreshCoverageSnapshot(
   em: EntityManager,
   scope: CoverageScope,
-): Promise<void> {
+): Promise<{ baseCount: number; indexedCount: number } | null> {
   const entityType = String(scope.entityType || '')
-  if (!entityType) return
+  if (!entityType) return null
   const tenantId = scope.tenantId ?? null
   const organizationId = scope.organizationId ?? null
   const withDeleted = scope.withDeleted === true
@@ -395,22 +408,37 @@ export async function refreshCoverageSnapshot(
   const hasTenant = await tableHasColumn(db, baseTable, 'tenant_id')
   const hasDeleted = await tableHasColumn(db, baseTable, 'deleted_at')
 
-  if (organizationId !== null && !hasOrg) return
-  if (tenantId !== null && !hasTenant) return
+  // A scope the base table cannot express must not narrow the index side either. Index rows
+  // can carry an organization the base table has no column for — `organizations` has no
+  // `organization_id` yet its index rows derive one from the record id. Filtering only the
+  // index side compares two different populations and reports a gap no reindex can close;
+  // returning early instead of recounting left the previous snapshot frozen, which is what
+  // surfaced as a permanent "out of sync" row for `directory:organization`.
+  //
+  // Tenant is different: a base table without `tenant_id` (`user_roles`) cannot be counted
+  // per tenant at all, and writing the cross-tenant total into one tenant's row would leak
+  // another tenant's volume into it. Drop the unusable scoped row instead, leaving only the
+  // global row — which is the one that can be true — rather than freezing a stale count.
+  const scopeOrg = organizationId !== null && hasOrg
+  if (tenantId !== null && !hasTenant) {
+    await deleteCoverageScope(db, { entityType, tenantId, organizationId, withDeleted })
+    return null
+  }
+  const scopeTenant = tenantId !== null && hasTenant
 
   let baseQuery = db
     .selectFrom(`${baseTable} as b` as any)
     .select(sql`count(*)`.as('count'))
-  if (organizationId !== null && hasOrg) baseQuery = baseQuery.where('b.organization_id' as any, '=', organizationId)
-  if (tenantId !== null && hasTenant) baseQuery = baseQuery.where('b.tenant_id' as any, '=', tenantId)
+  if (scopeOrg) baseQuery = baseQuery.where('b.organization_id' as any, '=', organizationId)
+  if (scopeTenant) baseQuery = baseQuery.where('b.tenant_id' as any, '=', tenantId)
   if (!withDeleted && hasDeleted) baseQuery = baseQuery.where('b.deleted_at' as any, 'is', null as any)
 
   let indexQuery = db
     .selectFrom('entity_indexes as ei' as any)
     .select(sql`count(*)`.as('count'))
     .where('ei.entity_type' as any, '=', entityType)
-  if (organizationId !== null) indexQuery = indexQuery.where('ei.organization_id' as any, '=', organizationId)
-  if (tenantId !== null) indexQuery = indexQuery.where('ei.tenant_id' as any, '=', tenantId)
+  if (scopeOrg) indexQuery = indexQuery.where('ei.organization_id' as any, '=', organizationId)
+  if (scopeTenant) indexQuery = indexQuery.where('ei.tenant_id' as any, '=', tenantId)
   if (!withDeleted) indexQuery = indexQuery.where('ei.deleted_at' as any, 'is', null as any)
 
   const vectorCountPromise = (async (): Promise<number | undefined> => {
@@ -453,6 +481,8 @@ export async function refreshCoverageSnapshot(
     indexedCount: indexCount,
     vectorCount,
   })
+
+  return { baseCount, indexedCount: indexCount }
 }
 
 export async function writeCoverageCounts(

package/src/modules/query_index/lib/reindexer.ts

@@ -323,55 +323,77 @@ export async function reindexEntity(
 
   options?.onProgress?.({ processed, total, chunkSize: 0 })
 
-  if (resetCoverage) {
-    if (force) {
+  // Partitions run concurrently, so a scope-wide purge here would delete rows a sibling
+  // partition has already written and never rebuild them (the run still reports success
+  // because progress counts attempted writes). Each partition therefore purges only the
+  // slice it is about to rebuild, and every partition purges — restricting the purge to
+  // the coverage-resetting partition would otherwise leave the other slices' stale rows
+  // behind on a forced rebuild.
+  if (force && (resetCoverage || usingPartitions)) {
+    try {
+      let purgeQuery = db
+        .deleteFrom('entity_indexes' as any)
+        .where('entity_type' as any, '=', entityType)
+      if (tenantId !== undefined) {
+        purgeQuery = purgeQuery.where(sql<boolean>`tenant_id is not distinct from ${tenantId ?? null}`)
+      }
+      if (organizationId !== undefined) {
+        purgeQuery = purgeQuery.where(sql<boolean>`organization_id is not distinct from ${organizationId ?? null}`)
+      }
+      if (usingPartitions && partitionIndex !== null) {
+        purgeQuery = purgeQuery.where(
+          sql<boolean>`mod(abs(hashtext(entity_id::text)), ${partitionCountRaw}) = ${partitionIndex}`,
+        )
+      }
+      await purgeQuery.execute()
+    } catch (error) {
+      logger.warn('Failed to purge index rows before force reindex', {
+        entityType,
+        tenantId: tenantId ?? null,
+        organizationId: organizationId ?? null,
+        partitionIndex: usingPartitions ? partitionIndex : null,
+        error: error instanceof Error ? error.message : error,
+      })
+    }
+  }
+
+  // The vector purge is scope-wide and cannot be partitioned — the queued job deletes every
+  // vector for the scope. Emitting it from one partition while siblings are already queueing
+  // per-record vectorize jobs is the same race the row purge just had, so a partitioned run
+  // leaves it to the caller that owns the fan-out (`api/reindex.ts` queues it once, before
+  // dispatching any partition).
+  if (force && resetCoverage && !usingPartitions && emitVectorize && eventBus) {
+    if (tenantId !== undefined) {
+      const payload: Record<string, unknown> = {
+        entityType,
+        tenantId: tenantId ?? null,
+      }
+      if (organizationId !== undefined) payload.organizationId = organizationId ?? null
       try {
-        let purgeQuery = db
-          .deleteFrom('entity_indexes' as any)
-          .where('entity_type' as any, '=', entityType)
-        if (tenantId !== undefined) {
-          purgeQuery = purgeQuery.where(sql<boolean>`tenant_id is not distinct from ${tenantId ?? null}`)
-        }
-        if (organizationId !== undefined) {
-          purgeQuery = purgeQuery.where(sql<boolean>`organization_id is not distinct from ${organizationId ?? null}`)
-        }
-        await purgeQuery.execute()
-      } catch (error) {
-        logger.warn('Failed to purge index rows before force reindex', {
+        await eventBus.emitEvent('query_index.vectorize_purge', payload)
+      } catch (err) {
+        logger.warn('Failed to queue vector purge before force reindex', {
           entityType,
           tenantId: tenantId ?? null,
           organizationId: organizationId ?? null,
-          error: error instanceof Error ? error.message : error,
+          error: err instanceof Error ? err.message : err,
         })
       }
-
-      if (emitVectorize && eventBus) {
-        if (tenantId !== undefined) {
-          const payload: Record<string, unknown> = {
-            entityType,
-            tenantId: tenantId ?? null,
-          }
-          if (organizationId !== undefined) payload.organizationId = organizationId ?? null
-          try {
-            await eventBus.emitEvent('query_index.vectorize_purge', payload)
-          } catch (err) {
-            logger.warn('Failed to queue vector purge before force reindex', {
-              entityType,
-              tenantId: tenantId ?? null,
-              organizationId: organizationId ?? null,
-              error: err instanceof Error ? err.message : err,
-            })
-          }
-        } else {
-          logger.warn('Skipping vector purge for force reindex without tenant scope', {
-            entityType,
-          })
-        }
-      }
+    } else {
+      logger.warn('Skipping vector purge for force reindex without tenant scope', {
+        entityType,
+      })
     }
+  }
 
+  if (resetCoverage) {
+    // Only meaningful for an unpartitioned run: `baseCounts` holds this partition's slice,
+    // so writing it as the scope's base count under-reports by the partition factor, and
+    // zeroing indexed_count scope-wide discards deltas siblings have already applied. The
+    // authoritative `refreshCoverageSnapshot` at the end of every partition owns the counts
+    // for partitioned runs.
     const nowTs = Date.now()
-    for (const scope of baseCounts.values()) {
+    for (const scope of usingPartitions ? [] : baseCounts.values()) {
       const key = `${entityType}|${scopeKey(scope.tenantId, scope.organizationId)}`
       const last = lastCoverageReset.get(key) ?? 0
       if (force || nowTs - last >= COVERAGE_REFRESH_THROTTLE_MS) {

package/src/modules/query_index/lib/subscriber-scope.ts

@@ -1,6 +1,7 @@
 import type { EntityManager } from '@mikro-orm/postgresql'
 import { getEntityIds } from '@open-mercato/shared/lib/encryption/entityIds'
 import { resolveRegisteredEntityTableName } from '@open-mercato/shared/lib/query/engine'
+import { listEntityMetadata } from '@open-mercato/shared/lib/db/entityMetadata'
 
 export type QueryIndexScope = {
   tenantId: string | null
@@ -58,13 +59,7 @@ export function resolveQueryIndexSourceMetadata(
     throw new QueryIndexScopeError(`Query index entity type has no registered ORM table: ${entityType}`)
   }
 
-  const registry = em.getMetadata?.()
-  const allMetadataRaw = typeof registry?.getAll === 'function' ? registry.getAll() : null
-  const allMetadata: QueryIndexEntityMetadata[] = Array.isArray(allMetadataRaw)
-    ? allMetadataRaw as QueryIndexEntityMetadata[]
-    : allMetadataRaw instanceof Map
-      ? Array.from(allMetadataRaw.values()) as QueryIndexEntityMetadata[]
-      : Object.values(allMetadataRaw ?? {}) as QueryIndexEntityMetadata[]
+  const allMetadata = listEntityMetadata(em) as unknown as QueryIndexEntityMetadata[]
   const metadata = allMetadata.find((candidate) => String(candidate.tableName ?? '') === table)
   if (!metadata) {
     throw new QueryIndexScopeError(`Query index entity metadata was not found for table: ${table}`)