@open-mercato/core 0.6.5-develop.4384.1.ce2ec6eaaa → 0.6.5-develop.4397.1.9a65481757

package/src/modules/communication_channels/workers/channel-import-history.ts

@@ -0,0 +1,252 @@
+import type { EntityManager } from '@mikro-orm/postgresql'
+import type { JobContext, QueuedJob, WorkerMeta } from '@open-mercato/queue'
+import type { CommandBus } from '@open-mercato/shared/lib/commands'
+import { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'
+import { CommunicationChannel } from '../data/entities'
+import {
+  COMMUNICATION_CHANNELS_INGEST_INBOUND_COMMAND_ID,
+  type IngestInboundMessageInput,
+} from '../commands/ingest-inbound-message'
+import { COMMUNICATION_CHANNELS_QUEUES } from '../lib/queue'
+import { refreshCredentialsIfNeeded } from '../lib/credential-refresh'
+import { classifyOutboundError } from '../lib/error-classification'
+import type { ChannelAdapterRegistry } from '../lib/registry'
+import type {
+  ChannelImportHistoryJobPayload,
+} from '../commands/queue-import-history'
+import type {
+  ProgressService,
+  ProgressServiceContext,
+} from '../../progress/lib/progressService'
+
+/**
+ * Spec B § Phase B6 — operator-triggered backlog import worker.
+ *
+ * Distinct from `poll-channel` (which runs every scheduler tick and ingests
+ * *new* mail since the channel cursor). This worker reaches backward in time
+ * by calling `adapter.importHistory` with explicit `sinceDays` / `contactEmails`
+ * filters, paginating until the adapter signals `hasMore: false` or the
+ * `maxMessages` cap is reached.
+ *
+ * Runs with `concurrency: 1` to avoid hammering the provider with multiple
+ * historical sweeps in parallel; per-channel concurrency is additionally
+ * enforced at enqueue time by `queueImportHistory`.
+ */
+export const metadata: WorkerMeta = {
+  queue: COMMUNICATION_CHANNELS_QUEUES.importHistory,
+  id: 'communication_channels:channel-import-history',
+  concurrency: 1,
+}
+
+type HandlerContext = JobContext & {
+  resolve: <T = unknown>(name: string) => T
+}
+
+type CredentialsServiceLike = {
+  resolve: (
+    integrationId: string,
+    scope: { organizationId: string; tenantId: string; userId?: string | null },
+  ) => Promise<Record<string, unknown> | null>
+  save?: (
+    integrationId: string,
+    credentials: Record<string, unknown>,
+    scope: { organizationId: string; tenantId: string; userId?: string | null },
+  ) => Promise<void>
+}
+
+export default async function handle(
+  job: QueuedJob<ChannelImportHistoryJobPayload>,
+  ctx: HandlerContext,
+): Promise<void> {
+  const payload = job.payload
+  const { progressJobId, channelId, sinceDays, contactEmails, maxMessages, scope } = payload
+
+  const progressService = ctx.resolve<ProgressService>('progressService')
+  const progressContext: ProgressServiceContext = {
+    tenantId: scope.tenantId,
+    organizationId: scope.organizationId,
+  }
+
+  const em = (ctx.resolve('em') as EntityManager).fork()
+  const adapterRegistry = ctx.resolve<ChannelAdapterRegistry>('channelAdapterRegistry')
+
+  try {
+    await progressService.startJob(progressJobId, progressContext)
+
+    const channel = await findOneWithDecryption(
+      em,
+      CommunicationChannel,
+      {
+        id: channelId,
+        tenantId: scope.tenantId,
+        organizationId: scope.organizationId,
+        deletedAt: null,
+      },
+      undefined,
+      scope,
+    )
+    if (!channel) {
+      await progressService.failJob(
+        progressJobId,
+        { errorMessage: 'Channel not found' },
+        progressContext,
+      )
+      return
+    }
+    if (!channel.isActive || channel.status !== 'connected') {
+      await progressService.failJob(
+        progressJobId,
+        { errorMessage: `Channel is not connected (status=${channel.status})` },
+        progressContext,
+      )
+      return
+    }
+
+    const adapter = adapterRegistry?.get(channel.providerKey)
+    if (!adapter || typeof adapter.importHistory !== 'function') {
+      await progressService.failJob(
+        progressJobId,
+        {
+          errorMessage: `Provider "${channel.providerKey}" does not support history import`,
+        },
+        progressContext,
+      )
+      return
+    }
+
+    // Resolve + refresh credentials. Same flow as poll-channel.
+    let credentialsService: CredentialsServiceLike | null = null
+    try {
+      credentialsService = ctx.resolve<CredentialsServiceLike>('integrationCredentialsService')
+    } catch {
+      credentialsService = null
+    }
+    const credentialsScope = {
+      tenantId: scope.tenantId,
+      organizationId: scope.organizationId,
+      userId: channel.userId ?? null,
+    }
+    let credentials: Record<string, unknown> = {}
+    if (channel.credentialsRef && credentialsService) {
+      try {
+        credentials =
+          (await credentialsService.resolve(`channel_${channel.providerKey}`, credentialsScope)) ?? {}
+      } catch {
+        credentials = {}
+      }
+    }
+    const refreshed = await refreshCredentialsIfNeeded(
+      { adapter, channelId: channel.id, credentials, scope: credentialsScope },
+      { credentialsService },
+    )
+    credentials = refreshed.credentials
+
+    const commandBus = ctx.resolve<CommandBus>('commandBus')
+    const containerProxy = { resolve: ctx.resolve.bind(ctx) }
+    const commandCtx = {
+      container: containerProxy as never,
+      auth: null,
+      organizationScope: null,
+      selectedOrganizationId: scope.organizationId,
+      organizationIds: [scope.organizationId],
+    }
+
+    let cursor: string | undefined
+    let processedCount = 0
+    let totalCount = maxMessages
+    let firstPage = true
+
+    // Bounded loop guards against a misbehaving adapter (infinite hasMore).
+    // 100 pages * HARD_CAP (default 200) = 20k messages — far above the 5k cap
+    // enforced by the schema, so this only trips on adapter bugs.
+    const MAX_PAGES = 100
+    for (let pageIndex = 0; pageIndex < MAX_PAGES; pageIndex += 1) {
+      if (await progressService.isCancellationRequested(progressJobId, scope.tenantId)) {
+        await progressService.markCancelled(progressJobId, progressContext)
+        return
+      }
+
+      const page = await adapter.importHistory!({
+        credentials,
+        scope: { tenantId: scope.tenantId, organizationId: scope.organizationId },
+        sinceDays,
+        contactEmails,
+        maxMessages,
+        cursor,
+      })
+
+      if (firstPage && typeof page.totalCandidates === 'number') {
+        totalCount = Math.min(maxMessages, page.totalCandidates)
+        await progressService.updateProgress(
+          progressJobId,
+          { totalCount, processedCount: 0 },
+          progressContext,
+        )
+        firstPage = false
+      }
+
+      for (const message of page.messages) {
+        if (processedCount >= maxMessages) break
+        try {
+          const input: IngestInboundMessageInput = {
+            channelId: channel.id,
+            providerKey: channel.providerKey,
+            channelType: channel.channelType,
+            scope: { tenantId: scope.tenantId, organizationId: scope.organizationId },
+            message,
+          }
+          await commandBus.execute(COMMUNICATION_CHANNELS_INGEST_INBOUND_COMMAND_ID, {
+            input,
+            ctx: commandCtx as never,
+          })
+        } catch (err) {
+          // Import-history is best-effort per-message. Same classification
+          // logic as poll-channel: a transient failure aborts the whole job
+          // (the operator can retry), permanent failures are logged and
+          // skipped (the bad message would loop forever otherwise).
+          const classification = classifyOutboundError(err)
+          if (classification.transient) {
+            throw err
+          }
+          console.warn(
+            `[communication_channels:channel-import-history] permanent ingest failure on channel ${channel.id}; skipping message ${message.externalMessageId}. Reason: ${classification.message}`,
+          )
+        }
+        processedCount += 1
+      }
+
+      await progressService.updateProgress(
+        progressJobId,
+        { processedCount, totalCount },
+        progressContext,
+      )
+
+      if (processedCount >= maxMessages) break
+      if (!page.hasMore || !page.nextCursor) break
+      cursor = page.nextCursor
+    }
+
+    await progressService.completeJob(
+      progressJobId,
+      { resultSummary: { importedCount: processedCount, channelId } },
+      progressContext,
+    )
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'Import history job failed'
+    const stack = err instanceof Error ? err.stack : undefined
+    try {
+      await progressService.failJob(
+        progressJobId,
+        { errorMessage: message.slice(0, 2000), errorStack: stack?.slice(0, 10000) },
+        progressContext,
+      )
+    } catch (failErr) {
+      console.error(
+        `[communication_channels:channel-import-history] failed to mark progress job ${progressJobId} as failed: ${
+          failErr instanceof Error ? failErr.message : String(failErr)
+        }`,
+      )
+    }
+    throw err
+  }
+}

package/src/modules/communication_channels/workers/gmail-history-sync.ts

@@ -0,0 +1,223 @@
+import type { EntityManager } from '@mikro-orm/postgresql'
+import type { JobContext, QueuedJob, WorkerMeta } from '@open-mercato/queue'
+import type { CommandBus } from '@open-mercato/shared/lib/commands'
+import { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'
+import { CommunicationChannel } from '../data/entities'
+import {
+  COMMUNICATION_CHANNELS_INGEST_INBOUND_COMMAND_ID,
+  type IngestInboundMessageInput,
+} from '../commands/ingest-inbound-message'
+import { COMMUNICATION_CHANNELS_QUEUES } from '../lib/queue'
+import { preservePushState } from '../lib/push-state'
+import { refreshCredentialsIfNeeded } from '../lib/credential-refresh'
+import { classifyOutboundError } from '../lib/error-classification'
+import { writeIngestDeadLetter } from '../lib/dead-letter'
+import type { ChannelAdapterRegistry } from '../lib/registry'
+
+/**
+ * Spec C § Phase C2 — Gmail Pub/Sub push delivery worker.
+ *
+ * One job per verified webhook notification. Calls
+ * `adapter.applyPushNotification(...)` to walk `users.history.list` from the
+ * channel's stored `historyId`, then dispatches each message through
+ * `ingest-inbound-message` (same path as the polling worker).
+ *
+ * Concurrency: bounded by the queue's default (10). Per-channel ordering is
+ * NOT guaranteed across notifications — but the ingest command is idempotent
+ * on `(channel_id, external_message_id)`, and Gmail's history cursor advances
+ * monotonically per channel, so out-of-order replay is safe.
+ */
+export type GmailHistorySyncJobPayload = {
+  channelId: string
+  scope: { tenantId: string; organizationId: string | null }
+  notification: { emailAddress: string; historyId: string }
+  /** Self-re-enqueue drain counter (bounds the multi-page drain loop). */
+  drainPage?: number
+}
+
+/** Hard cap on self-re-enqueue drain pages — guards against an adapter that
+ *  returns `hasMore` with a non-advancing cursor (a tight, unbounded loop). */
+const MAX_DRAIN_PAGES = 100
+
+export const metadata: WorkerMeta = {
+  queue: COMMUNICATION_CHANNELS_QUEUES.gmailHistorySync,
+  id: 'communication_channels:gmail-history-sync',
+  concurrency: 5,
+}
+
+type HandlerContext = JobContext & {
+  resolve: <T = unknown>(name: string) => T
+}
+
+type CredentialsServiceLike = {
+  resolve: (
+    integrationId: string,
+    scope: { organizationId: string; tenantId: string; userId?: string | null },
+  ) => Promise<Record<string, unknown> | null>
+  save?: (
+    integrationId: string,
+    credentials: Record<string, unknown>,
+    scope: { organizationId: string; tenantId: string; userId?: string | null },
+  ) => Promise<void>
+}
+
+export default async function handle(
+  job: QueuedJob<GmailHistorySyncJobPayload>,
+  ctx: HandlerContext,
+): Promise<void> {
+  const { channelId, scope } = job.payload
+  const em = (ctx.resolve('em') as EntityManager).fork()
+  const adapterRegistry = ctx.resolve<ChannelAdapterRegistry>('channelAdapterRegistry')
+
+  const channel = await findOneWithDecryption(
+    em,
+    CommunicationChannel,
+    {
+      id: channelId,
+      tenantId: scope.tenantId,
+      organizationId: scope.organizationId ?? null,
+      deletedAt: null,
+    },
+    undefined,
+    scope,
+  )
+  if (!channel) {
+    console.warn(`[gmail-history-sync] channel ${channelId} not found (skip)`)
+    return
+  }
+  if (!channel.isActive || channel.status !== 'connected') return
+
+  const adapter = adapterRegistry?.get(channel.providerKey)
+  if (!adapter || typeof adapter.applyPushNotification !== 'function') {
+    console.warn(
+      `[gmail-history-sync] adapter for '${channel.providerKey}' does not support applyPushNotification`,
+    )
+    return
+  }
+
+  let credentialsService: CredentialsServiceLike | null = null
+  try {
+    credentialsService = ctx.resolve<CredentialsServiceLike>('integrationCredentialsService')
+  } catch {
+    credentialsService = null
+  }
+  const credentialsScope = {
+    tenantId: scope.tenantId,
+    organizationId: scope.organizationId ?? scope.tenantId,
+    userId: channel.userId ?? null,
+  }
+  let credentials: Record<string, unknown> = {}
+  if (channel.credentialsRef && credentialsService) {
+    try {
+      credentials =
+        (await credentialsService.resolve(`channel_${channel.providerKey}`, credentialsScope)) ?? {}
+    } catch {
+      credentials = {}
+    }
+  }
+  const refreshed = await refreshCredentialsIfNeeded(
+    { adapter, channelId: channel.id, credentials, scope: credentialsScope },
+    { credentialsService },
+  )
+  credentials = refreshed.credentials
+
+  let page
+  try {
+    page = await adapter.applyPushNotification({
+      credentials,
+      scope: { tenantId: scope.tenantId, organizationId: scope.organizationId ?? scope.tenantId },
+      channelState: (channel.channelState as Record<string, unknown> | null) ?? {},
+      notification: job.payload.notification as unknown as Record<string, unknown>,
+    })
+  } catch (err) {
+    const classification = classifyOutboundError(err)
+    if (classification.transient) throw err
+    console.warn(
+      `[gmail-history-sync] permanent failure applying push for channel ${channel.id}: ${classification.message}`,
+    )
+    return
+  }
+
+  const messages = Array.isArray(page?.messages) ? page.messages : []
+  if (messages.length > 0) {
+    const commandBus = ctx.resolve<CommandBus>('commandBus')
+    const containerProxy = { resolve: ctx.resolve.bind(ctx) }
+    const commandCtx = {
+      container: containerProxy as never,
+      auth: null,
+      organizationScope: null,
+      selectedOrganizationId: scope.organizationId ?? null,
+      organizationIds: scope.organizationId ? [scope.organizationId] : null,
+    }
+    for (const message of messages) {
+      try {
+        const input: IngestInboundMessageInput = {
+          channelId: channel.id,
+          providerKey: channel.providerKey,
+          channelType: channel.channelType,
+          scope: { tenantId: scope.tenantId, organizationId: scope.organizationId ?? null },
+          message,
+        }
+        await commandBus.execute(COMMUNICATION_CHANNELS_INGEST_INBOUND_COMMAND_ID, {
+          input,
+          ctx: commandCtx as never,
+        })
+      } catch (err) {
+        const classification = classifyOutboundError(err)
+        if (classification.transient) {
+          // Per-message commit semantics from Spec B: re-throw so queue
+          // retry re-runs from a safe point.
+          throw err
+        }
+        console.warn(
+          `[gmail-history-sync] permanent ingest failure for channel ${channel.id}: ${classification.message}`,
+        )
+        await writeIngestDeadLetter({ em, scope, channel, message, err, errorMessage: classification.message })
+      }
+    }
+  }
+
+  // Persist the adapter's advanced cursor (encoded in nextCursor) back to
+  // channel.channelState. fetchHistory's nextCursor is a base64-JSON blob
+  // mirroring the new GmailChannelState — same as the polling worker writes.
+  if (page?.nextCursor) {
+    try {
+      const decoded = JSON.parse(Buffer.from(page.nextCursor, 'base64').toString('utf-8')) as Record<string, unknown>
+      // Carry the push keys forward via preservePushState rather than spreading
+      // the decoded cursor wholesale: a blind spread would retain a stale
+      // `pendingHistoryPageToken` after a completed drain and mis-route the next
+      // push notification.
+      channel.channelState = preservePushState(channel.channelState, decoded)
+      channel.lastPolledAt = new Date()
+      await em.flush()
+    } catch (err) {
+      console.warn(
+        `[gmail-history-sync] failed to persist next cursor for channel ${channel.id}: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      )
+    }
+  }
+
+  if (page?.hasMore && page?.nextCursor) {
+    // Re-enqueue self so the drain continues. The Pub/Sub notification fired
+    // once, but `history.list` may need multiple pages on a busy mailbox.
+    // Bound the drain and add a small delay so an adapter that returns
+    // `hasMore: true` with a non-advancing cursor cannot spin a tight,
+    // unthrottled re-enqueue loop against the provider/queue.
+    const drainPage = job.payload.drainPage ?? 0
+    if (drainPage < MAX_DRAIN_PAGES) {
+      const queue = (await import('../lib/queue')).getCommunicationChannelsQueue(
+        COMMUNICATION_CHANNELS_QUEUES.gmailHistorySync,
+      )
+      await queue.enqueue(
+        { ...job.payload, drainPage: drainPage + 1 } as unknown as Record<string, unknown>,
+        { delayMs: 250 },
+      )
+    } else {
+      console.warn(
+        `[gmail-history-sync] drain page cap (${MAX_DRAIN_PAGES}) reached for channel ${channelId}; stopping re-enqueue`,
+      )
+    }
+  }
+}

package/src/modules/communication_channels/workers/gmail-renew-watch.ts

@@ -0,0 +1,141 @@
+import type { EntityManager } from '@mikro-orm/postgresql'
+import type { JobContext, QueuedJob, WorkerMeta } from '@open-mercato/queue'
+import { findWithDecryption } from '@open-mercato/shared/lib/encryption/find'
+import { CommunicationChannel } from '../data/entities'
+import { COMMUNICATION_CHANNELS_QUEUES } from '../lib/queue'
+import { pushRenew } from '../commands/push-renew'
+import { emitCommunicationChannelsEvent } from '../events'
+
+/**
+ * Spec C § Phase C4 — Daily cron that re-issues `gmail.users.watch` for
+ * channels whose `watchExpirationMs` is within the renewal lead window.
+ *
+ * Gmail watch expires after ~7 days. We renew with a lead time of
+ * `OM_PUSH_RENEWAL_GMAIL_LEAD_HOURS` (default 24h) so a missed cron tick
+ * doesn't drop us into the polling fallback.
+ *
+ * The actual renewal is delegated to `pushRenew` (which in turn calls
+ * `pushRegister`) — Gmail's `users.watch` is idempotent, so re-calling it
+ * just returns a fresh `historyId` + `expiration` and persists them.
+ *
+ * Scoped to `job.payload.scope` (the cron is registered per-org in
+ * `setup.ts`); each per-org cron tick only walks channels for its own
+ * `(tenantId, organizationId)`. A missing scope means the legacy global
+ * sweep variant — supported for backwards compatibility, but new
+ * registrations always carry a scope.
+ */
+export type GmailRenewWatchPayload = {
+  scope?: { tenantId: string; organizationId: string | null }
+}
+
+export const metadata: WorkerMeta = {
+  queue: COMMUNICATION_CHANNELS_QUEUES.gmailRenewWatch,
+  id: 'communication_channels:gmail-renew-watch',
+  concurrency: 1,
+}
+
+type HandlerContext = JobContext & { resolve: <T = unknown>(name: string) => T }
+
+const LEAD_HOURS = Math.max(
+  1,
+  Number.parseInt(process.env.OM_PUSH_RENEWAL_GMAIL_LEAD_HOURS ?? '24', 10) || 24,
+)
+
+export default async function handle(
+  job: QueuedJob<GmailRenewWatchPayload>,
+  ctx: HandlerContext,
+): Promise<void> {
+  const em = (ctx.resolve('em') as EntityManager).fork()
+  const cutoffMs = Date.now() + LEAD_HOURS * 60 * 60 * 1000
+  const scope = job?.payload?.scope ?? null
+
+  // Per-org cron: filter by the scope embedded in the payload. The unscoped
+  // fallback is preserved so a deploy that hand-enqueues a renewal sweep
+  // (e.g. an operator-triggered backfill) still works.
+  const where: Record<string, unknown> = {
+    providerKey: 'gmail',
+    isActive: true,
+    deletedAt: null,
+  }
+  if (scope?.tenantId) where.tenantId = scope.tenantId
+  if (scope?.organizationId) where.organizationId = scope.organizationId
+  const channels = await findWithDecryption(
+    em,
+    CommunicationChannel,
+    where,
+    undefined,
+    scope ? { tenantId: scope.tenantId, organizationId: scope.organizationId ?? null } : undefined,
+  )
+
+  const containerProxy = (ctx as unknown as { container?: unknown }).container
+  let renewed = 0
+  let failed = 0
+  for (const channel of channels) {
+    const state =
+      (channel.channelState as { watchExpirationMs?: number; pushStatus?: string } | null) ?? null
+    if (!state || state.pushStatus !== 'active') continue
+    if (typeof state.watchExpirationMs !== 'number') continue
+    if (state.watchExpirationMs > cutoffMs) continue
+
+    const organizationId = channel.organizationId
+    if (!organizationId) {
+      console.warn(
+        `[gmail-renew-watch] skipping channel ${channel.id} — no organizationId on row`,
+      )
+      continue
+    }
+
+    try {
+      const container = resolveContainer(containerProxy, ctx)
+      const result = await pushRenew({
+        container,
+        scope: {
+          tenantId: channel.tenantId,
+          organizationId,
+          userId: channel.userId ?? null,
+        },
+        input: { channelId: channel.id },
+      })
+      if (result.pushStatus === 'active') {
+        renewed += 1
+        await emitCommunicationChannelsEvent(
+          'communication_channels.push.renewed',
+          {
+            channelId: channel.id,
+            providerKey: channel.providerKey,
+            tenantId: channel.tenantId,
+            organizationId,
+          },
+          { persistent: false },
+        )
+      } else {
+        failed += 1
+      }
+    } catch (err) {
+      failed += 1
+      console.warn(
+        `[gmail-renew-watch] failed to renew channel ${channel.id}: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      )
+    }
+  }
+  if (renewed > 0 || failed > 0) {
+    console.info(`[gmail-renew-watch] renewed=${renewed} failed=${failed}`)
+  }
+}
+
+function resolveContainer(
+  containerProxy: unknown,
+  ctx: { resolve: <T = unknown>(name: string) => T },
+): import('awilix').AwilixContainer {
+  // The queue runtime exposes container access via `ctx.container` in
+  // dedicated worker harnesses, but the bare-worker contract only provides
+  // `ctx.resolve(name)`. Both shapes appear in production (test harness,
+  // AUTO_SPAWN_WORKERS) so we accept either.
+  if (containerProxy && typeof containerProxy === 'object' && 'resolve' in containerProxy) {
+    return containerProxy as import('awilix').AwilixContainer
+  }
+  // Synthesize a minimal container shape from `resolve`.
+  return { resolve: ctx.resolve.bind(ctx) } as unknown as import('awilix').AwilixContainer
+}