@open-mercato/search 0.4.2-canary-c02407ff85

package/src/modules/search/api/embeddings/reindex/route.ts

@@ -0,0 +1,197 @@
+import { NextResponse } from 'next/server'
+import { createRequestContainer } from '@open-mercato/shared/lib/di/container'
+import { getAuthFromRequest } from '@open-mercato/shared/lib/auth/server'
+import type { SearchIndexer } from '../../../../../indexer/search-indexer'
+import type { EmbeddingService } from '../../../../../vector'
+import type { Knex } from 'knex'
+import type { EntityManager } from '@mikro-orm/postgresql'
+import { recordIndexerLog } from '@open-mercato/shared/lib/indexers/status-log'
+import { resolveTranslations } from '@open-mercato/shared/lib/i18n/server'
+import { resolveEmbeddingConfig } from '../../../lib/embedding-config'
+import type { EntityId } from '@open-mercato/shared/modules/entities'
+import { searchDebug, searchDebugWarn, searchError } from '../../../../../lib/debug'
+import { acquireReindexLock, clearReindexLock, getReindexLockStatus } from '../../../lib/reindex-lock'
+
+export const metadata = {
+  POST: { requireAuth: true, requireFeatures: ['search.embeddings.manage'] },
+}
+
+export async function POST(req: Request) {
+  const { t } = await resolveTranslations()
+  const auth = await getAuthFromRequest(req)
+  if (!auth?.tenantId) {
+    return NextResponse.json({ error: t('api.errors.unauthorized', 'Unauthorized') }, { status: 401 })
+  }
+
+  let payload: { entityId?: string; purgeFirst?: boolean } = {}
+  try {
+    payload = await req.json()
+  } catch {
+    // Default values
+  }
+
+  const entityId = typeof payload?.entityId === 'string' ? payload.entityId : undefined
+  const purgeFirst = payload?.purgeFirst === true
+
+  const container = await createRequestContainer()
+  const em = container.resolve('em') as EntityManager
+  const knex = (em.getConnection() as unknown as { getKnex: () => Knex }).getKnex()
+
+  // Check if another vector reindex operation is already in progress
+  const existingLock = await getReindexLockStatus(knex, auth.tenantId, { type: 'vector' })
+  if (existingLock) {
+    const startedAt = new Date(existingLock.startedAt)
+    return NextResponse.json(
+      {
+        error: t('search.api.errors.reindexInProgress', 'A reindex operation is already in progress'),
+        lock: {
+          type: existingLock.type,
+          action: existingLock.action,
+          startedAt: existingLock.startedAt,
+          elapsedMinutes: Math.round((Date.now() - startedAt.getTime()) / 60000),
+          processedCount: existingLock.processedCount,
+          totalCount: existingLock.totalCount,
+        },
+      },
+      { status: 409 }
+    )
+  }
+
+  // Acquire lock before starting the operation
+  const { acquired: lockAcquired } = await acquireReindexLock(knex, {
+    type: 'vector',
+    action: entityId ? `reindex:${entityId}` : 'reindex:all',
+    tenantId: auth.tenantId,
+    organizationId: auth.orgId ?? null,
+  })
+
+  if (!lockAcquired) {
+    return NextResponse.json(
+      { error: t('search.api.errors.lockFailed', 'Failed to acquire reindex lock') },
+      { status: 409 }
+    )
+  }
+
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    let em: any = null
+    try {
+      em = container.resolve('em')
+    } catch {
+      // em not available
+    }
+
+    let searchIndexer: SearchIndexer
+    try {
+      searchIndexer = container.resolve('searchIndexer') as SearchIndexer
+    } catch {
+      return NextResponse.json(
+        { error: t('search.api.errors.indexUnavailable', 'Search indexer unavailable') },
+        { status: 503 }
+      )
+    }
+
+    // Load saved embedding config and update the embedding service
+    try {
+      const embeddingConfig = await resolveEmbeddingConfig(container, { defaultValue: null })
+      if (embeddingConfig) {
+        const embeddingService = container.resolve<EmbeddingService>('vectorEmbeddingService')
+        embeddingService.updateConfig(embeddingConfig)
+        searchDebug('search.embeddings.reindex', 'using embedding config', {
+          providerId: embeddingConfig.providerId,
+          model: embeddingConfig.model,
+          dimension: embeddingConfig.dimension,
+        })
+      }
+    } catch (err) {
+      searchDebugWarn('search.embeddings.reindex', 'failed to load embedding config, using defaults', {
+        error: err instanceof Error ? err.message : err,
+      })
+    }
+
+    await recordIndexerLog(
+      { em: em ?? undefined },
+      {
+        source: 'vector',
+        handler: 'api:search.embeddings.reindex',
+        message: entityId
+          ? `Vector reindex requested for ${entityId}`
+          : 'Vector reindex requested for all entities',
+        entityType: entityId ?? null,
+        tenantId: auth.tenantId ?? null,
+        organizationId: auth.orgId ?? null,
+        details: { purgeFirst },
+      },
+    ).catch(() => undefined)
+
+    // Use queue-based vector reindexing (similar to fulltext)
+    // This enqueues batches for background processing by workers
+    let result
+    if (entityId) {
+      result = await searchIndexer.reindexEntityToVector({
+        entityId: entityId as EntityId,
+        tenantId: auth.tenantId,
+        organizationId: auth.orgId ?? null,
+        purgeFirst,
+        useQueue: true,
+      })
+    } else {
+      result = await searchIndexer.reindexAllToVector({
+        tenantId: auth.tenantId,
+        organizationId: auth.orgId ?? null,
+        purgeFirst,
+        useQueue: true,
+      })
+    }
+
+    await recordIndexerLog(
+      { em: em ?? undefined },
+      {
+        source: 'vector',
+        handler: 'api:search.embeddings.reindex',
+        message: result.jobsEnqueued
+          ? `Vector reindex enqueued ${result.jobsEnqueued} jobs for ${entityId ?? 'all entities'}`
+          : `Vector reindex completed for ${entityId ?? 'all entities'}`,
+        entityType: entityId ?? null,
+        tenantId: auth.tenantId ?? null,
+        organizationId: auth.orgId ?? null,
+        details: {
+          purgeFirst,
+          recordsIndexed: result.recordsIndexed,
+          jobsEnqueued: result.jobsEnqueued,
+          success: result.success,
+        },
+      },
+    ).catch(() => undefined)
+
+    return NextResponse.json({
+      ok: result.success,
+      recordsIndexed: result.recordsIndexed,
+      jobsEnqueued: result.jobsEnqueued,
+      entitiesProcessed: result.entitiesProcessed,
+      errors: result.errors.length > 0 ? result.errors : undefined,
+    })
+  } catch (error: unknown) {
+    const err = error as { message?: string; status?: number; statusCode?: number }
+    const status = typeof err?.status === 'number'
+      ? err.status
+      : (typeof err?.statusCode === 'number' ? err.statusCode : 500)
+    searchError('search.embeddings.reindex', 'failed', {
+      error: error instanceof Error ? error.message : error,
+      stack: error instanceof Error ? error.stack : undefined,
+      status,
+    })
+    return NextResponse.json(
+      { error: t('search.api.errors.reindexFailed', 'Vector reindex failed. Please try again or contact support.') },
+      { status: status >= 400 ? status : 500 }
+    )
+  } finally {
+    // Do NOT clear lock here - vector reindex always uses queue mode
+    // Workers update heartbeat and stale detection handles cleanup when done
+
+    const disposable = container as unknown as { dispose?: () => Promise<void> }
+    if (typeof disposable.dispose === 'function') {
+      await disposable.dispose()
+    }
+  }
+}

package/src/modules/search/api/embeddings/route.ts

@@ -0,0 +1,304 @@
+import { NextResponse } from 'next/server'
+import { z } from 'zod'
+import { createRequestContainer } from '@open-mercato/shared/lib/di/container'
+import { getAuthFromRequest } from '@open-mercato/shared/lib/auth/server'
+import type { ModuleConfigService } from '@open-mercato/core/modules/configs/lib/module-config-service'
+import { envDisablesAutoIndexing, resolveAutoIndexingEnabled, SEARCH_AUTO_INDEX_CONFIG_KEY } from '../../lib/auto-indexing'
+import {
+  resolveEmbeddingConfig,
+  saveEmbeddingConfig,
+  getConfiguredProviders,
+  detectConfigChange,
+  getEffectiveDimension,
+} from '../../lib/embedding-config'
+import { resolveTranslations } from '@open-mercato/shared/lib/i18n/server'
+import type { EmbeddingProviderConfig, EmbeddingProviderId, VectorDriver } from '../../../../vector'
+import { EMBEDDING_PROVIDERS, DEFAULT_EMBEDDING_CONFIG, EmbeddingService } from '../../../../vector'
+import { searchDebug, searchDebugWarn, searchError } from '../../../../lib/debug'
+
+const embeddingConfigSchema = z.object({
+  providerId: z.enum(['openai', 'google', 'mistral', 'cohere', 'bedrock', 'ollama']),
+  model: z.string(),
+  dimension: z.number(),
+  outputDimensionality: z.number().optional(),
+  baseUrl: z.string().optional(),
+})
+
+const updateSchema = z.object({
+  autoIndexingEnabled: z.boolean().optional(),
+  embeddingConfig: embeddingConfigSchema.optional(),
+})
+
+export const metadata = {
+  GET: { requireAuth: true, requireFeatures: ['search.embeddings.view'] },
+  POST: { requireAuth: true, requireFeatures: ['search.embeddings.manage'] },
+}
+
+type SettingsResponse = {
+  settings: {
+    openaiConfigured: boolean
+    autoIndexingEnabled: boolean
+    autoIndexingLocked: boolean
+    lockReason: string | null
+    embeddingConfig: EmbeddingProviderConfig | null
+    configuredProviders: EmbeddingProviderId[]
+    indexedDimension: number | null
+    reindexRequired: boolean
+    documentCount: number | null
+  }
+}
+
+const openAiConfigured = () => Boolean(process.env.OPENAI_API_KEY && process.env.OPENAI_API_KEY.trim().length > 0)
+
+const toJson = (payload: SettingsResponse, init?: ResponseInit) => NextResponse.json(payload, init)
+
+const unauthorized = async () => {
+  const { t } = await resolveTranslations()
+  return NextResponse.json({ error: t('api.errors.unauthorized', 'Unauthorized') }, { status: 401 })
+}
+
+const configUnavailable = async () => {
+  const { t } = await resolveTranslations()
+  return NextResponse.json({ error: t('search.api.errors.configUnavailable', 'Configuration service unavailable') }, { status: 503 })
+}
+
+async function getIndexedDimension(container: { resolve: <T = unknown>(name: string) => T }): Promise<number | null> {
+  try {
+    const drivers = container.resolve<VectorDriver[]>('vectorDrivers')
+    const pgvectorDriver = drivers.find((d) => d.id === 'pgvector')
+    if (pgvectorDriver?.getTableDimension) {
+      return await pgvectorDriver.getTableDimension()
+    }
+    return null
+  } catch {
+    return null
+  }
+}
+
+async function getVectorDocumentCount(
+  container: { resolve: <T = unknown>(name: string) => T },
+  tenantId: string,
+  organizationId?: string | null,
+): Promise<number | null> {
+  try {
+    const drivers = container.resolve<VectorDriver[]>('vectorDrivers')
+    const pgvectorDriver = drivers.find((d) => d.id === 'pgvector')
+    if (pgvectorDriver?.count) {
+      return await pgvectorDriver.count({ tenantId, organizationId: organizationId ?? undefined })
+    }
+    return null
+  } catch {
+    return null
+  }
+}
+
+export async function GET(req: Request) {
+  const auth = await getAuthFromRequest(req)
+  if (!auth?.sub) return await unauthorized()
+
+  const container = await createRequestContainer()
+  try {
+    const lockedByEnv = envDisablesAutoIndexing()
+    let autoIndexingEnabled = !lockedByEnv
+    if (!lockedByEnv) {
+      try {
+        autoIndexingEnabled = await resolveAutoIndexingEnabled(container, { defaultValue: true })
+      } catch {
+        autoIndexingEnabled = true
+      }
+    }
+
+    const embeddingConfig = await resolveEmbeddingConfig(container, { defaultValue: null })
+    const configuredProviders = getConfiguredProviders()
+    const indexedDimension = await getIndexedDimension(container)
+
+    const effectiveDimension = embeddingConfig
+      ? getEffectiveDimension(embeddingConfig)
+      : DEFAULT_EMBEDDING_CONFIG.dimension
+
+    const reindexRequired = Boolean(
+      indexedDimension &&
+      embeddingConfig &&
+      indexedDimension !== effectiveDimension
+    )
+
+    // Get document count for vector index
+    const documentCount = auth.tenantId
+      ? await getVectorDocumentCount(container, auth.tenantId, auth.orgId)
+      : null
+
+    return toJson({
+      settings: {
+        openaiConfigured: openAiConfigured(),
+        autoIndexingEnabled: lockedByEnv ? false : autoIndexingEnabled,
+        autoIndexingLocked: lockedByEnv,
+        lockReason: lockedByEnv ? 'env' : null,
+        embeddingConfig,
+        configuredProviders,
+        indexedDimension,
+        reindexRequired,
+        documentCount,
+      },
+    })
+  } finally {
+    const disposable = container as unknown as { dispose?: () => Promise<void> }
+    if (typeof disposable.dispose === 'function') {
+      await disposable.dispose()
+    }
+  }
+}
+
+export async function POST(req: Request) {
+  const { t } = await resolveTranslations()
+  const auth = await getAuthFromRequest(req)
+  if (!auth?.sub) return await unauthorized()
+
+  let body: unknown
+  try {
+    body = await req.json()
+  } catch {
+    return NextResponse.json({ error: t('api.errors.invalidJson', 'Invalid JSON payload.') }, { status: 400 })
+  }
+  const parsed = updateSchema.safeParse(body)
+  if (!parsed.success) {
+    return NextResponse.json({ error: t('api.errors.invalidPayload', 'Invalid payload.') }, { status: 400 })
+  }
+
+  const container = await createRequestContainer()
+  try {
+    let service: ModuleConfigService
+    try {
+      service = (container.resolve('moduleConfigService') as ModuleConfigService)
+    } catch {
+      return await configUnavailable()
+    }
+
+    if (parsed.data.autoIndexingEnabled !== undefined) {
+      if (envDisablesAutoIndexing()) {
+        return NextResponse.json(
+          { error: t('search.api.errors.autoIndexingDisabled', 'Auto-indexing is disabled via DISABLE_VECTOR_SEARCH_AUTOINDEXING.') },
+          { status: 409 },
+        )
+      }
+      await service.setValue('vector', SEARCH_AUTO_INDEX_CONFIG_KEY, parsed.data.autoIndexingEnabled)
+    }
+
+    let embeddingConfig = await resolveEmbeddingConfig(container, { defaultValue: null })
+    let reindexRequired = false
+    let indexedDimension = await getIndexedDimension(container)
+
+    if (parsed.data.embeddingConfig) {
+      const newConfig = parsed.data.embeddingConfig
+      const providerInfo = EMBEDDING_PROVIDERS[newConfig.providerId]
+
+      if (!providerInfo) {
+        return NextResponse.json(
+          { error: t('search.api.errors.invalidProvider', 'Invalid embedding provider.') },
+          { status: 400 },
+        )
+      }
+
+      const configuredProviders = getConfiguredProviders()
+      if (!configuredProviders.includes(newConfig.providerId)) {
+        return NextResponse.json(
+          { error: t('search.api.errors.providerNotConfigured', `Provider ${providerInfo.name} is not configured. Set ${providerInfo.envKeyRequired} environment variable.`) },
+          { status: 400 },
+        )
+      }
+
+      const change = detectConfigChange(
+        embeddingConfig,
+        {
+          ...newConfig,
+          updatedAt: new Date().toISOString(),
+        },
+        indexedDimension
+      )
+
+      if (change.requiresReindex) {
+        const newDimension = getEffectiveDimension(change.newConfig)
+        searchDebug('search.embeddings.update', 'config change detected, recreating table', {
+          requiresReindex: change.requiresReindex,
+          reason: change.reason,
+          oldDimension: indexedDimension,
+          newDimension,
+        })
+        try {
+          const drivers = container.resolve<VectorDriver[]>('vectorDrivers')
+          const pgvectorDriver = drivers.find((d) => d.id === 'pgvector')
+          if (pgvectorDriver?.recreateWithDimension) {
+            await pgvectorDriver.recreateWithDimension(newDimension)
+            // Query the actual dimension from the database to confirm
+            if (pgvectorDriver.getTableDimension) {
+              indexedDimension = await pgvectorDriver.getTableDimension()
+            } else {
+              indexedDimension = newDimension
+            }
+            searchDebug('search.embeddings.update', 'table recreated successfully', { indexedDimension })
+          } else {
+            searchDebugWarn('search.embeddings.update', 'pgvector driver does not have recreateWithDimension method')
+          }
+        } catch (error) {
+          searchError('search.embeddings.update', 'failed to recreate table', {
+            error: error instanceof Error ? error.message : error,
+          })
+          return NextResponse.json(
+            { error: t('search.api.errors.recreateFailed', 'Failed to recreate vector table with new dimension.') },
+            { status: 500 },
+          )
+        }
+      }
+
+      await saveEmbeddingConfig(container, change.newConfig)
+      embeddingConfig = change.newConfig
+
+      try {
+        const embeddingService = container.resolve<EmbeddingService>('vectorEmbeddingService')
+        embeddingService.updateConfig(embeddingConfig)
+      } catch {
+        // Embedding service may not be available in all contexts
+      }
+
+      reindexRequired = change.requiresReindex
+    }
+
+    const lockedByEnv = envDisablesAutoIndexing()
+    let autoIndexingEnabled = !lockedByEnv
+    if (!lockedByEnv) {
+      try {
+        autoIndexingEnabled = await resolveAutoIndexingEnabled(container, { defaultValue: true })
+      } catch {
+        autoIndexingEnabled = true
+      }
+    }
+
+    // Get updated document count
+    const updatedDocumentCount = auth.tenantId
+      ? await getVectorDocumentCount(container, auth.tenantId, auth.orgId)
+      : null
+
+    return toJson({
+      settings: {
+        openaiConfigured: openAiConfigured(),
+        autoIndexingEnabled: lockedByEnv ? false : autoIndexingEnabled,
+        autoIndexingLocked: lockedByEnv,
+        lockReason: lockedByEnv ? 'env' : null,
+        embeddingConfig,
+        configuredProviders: getConfiguredProviders(),
+        indexedDimension,
+        reindexRequired,
+        documentCount: updatedDocumentCount,
+      },
+    })
+  } catch (error) {
+    searchError('search.embeddings.update', 'failed', {
+      error: error instanceof Error ? error.message : error,
+    })
+    return NextResponse.json({ error: t('search.api.errors.updateFailed', 'Failed to update embedding settings.') }, { status: 500 })
+  } finally {
+    const disposable = container as unknown as { dispose?: () => Promise<void> }
+    if (typeof disposable.dispose === 'function') {
+      await disposable.dispose()
+    }
+  }
+}