@open-mercato/onboarding 0.6.6-develop.5431.1.384a97c7a2 → 0.6.6-develop.5483.1.a1129165ea

package/generated/entities/onboarding_request/index.ts

@@ -16,6 +16,7 @@ export const tenant_id = "tenant_id";
 export const organization_id = "organization_id";
 export const user_id = "user_id";
 export const last_email_sent_at = "last_email_sent_at";
+export const preparation_started_at = "preparation_started_at";
 export const preparation_completed_at = "preparation_completed_at";
 export const ready_email_sent_at = "ready_email_sent_at";
 export const created_at = "created_at";

package/generated/entity-fields-registry.ts

@@ -20,6 +20,7 @@ export const entityFieldsRegistry: Record<string, Record<string, string>> = {
       "organization_id": "organization_id",
       "user_id": "user_id",
       "last_email_sent_at": "last_email_sent_at",
+      "preparation_started_at": "preparation_started_at",
       "preparation_completed_at": "preparation_completed_at",
       "ready_email_sent_at": "ready_email_sent_at",
       "created_at": "created_at",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@open-mercato/onboarding",
-  "version": "0.6.6-develop.5431.1.384a97c7a2",
+  "version": "0.6.6-develop.5483.1.a1129165ea",
   "type": "module",
   "main": "./dist/index.js",
   "scripts": {
@@ -69,10 +69,10 @@
     }
   },
   "peerDependencies": {
-    "@open-mercato/shared": "0.6.6-develop.5431.1.384a97c7a2"
+    "@open-mercato/shared": "0.6.6-develop.5483.1.a1129165ea"
   },
   "devDependencies": {
-    "@open-mercato/shared": "0.6.6-develop.5431.1.384a97c7a2",
+    "@open-mercato/shared": "0.6.6-develop.5483.1.a1129165ea",
     "@types/jest": "^30.0.0",
     "jest": "^30.4.2",
     "ts-jest": "^29.4.11"

package/src/__tests__/deferred-provisioning.test.ts

@@ -0,0 +1,210 @@
+const seedExamples = jest.fn(async () => {
+  await new Promise((resolve) => setImmediate(resolve))
+})
+const purgeIndexScope = jest.fn(async () => {})
+const reindexEntity = jest.fn(async () => {})
+const refreshCoverageSnapshot = jest.fn(async () => {})
+const sendWorkspaceReadyEmail = jest.fn(async () => true)
+
+type ClaimState = {
+  status: string
+  preparationStartedAt: Date | null
+  preparationCompletedAt: Date | null
+}
+
+const claimState: ClaimState = {
+  status: 'completed',
+  preparationStartedAt: null,
+  preparationCompletedAt: null,
+}
+
+const claimPreparation = jest.fn(async (_requestId: string, claimedAt: Date, staleBefore: Date) => {
+  if (claimState.status !== 'completed') return false
+  if (claimState.preparationCompletedAt) return false
+  if (claimState.preparationStartedAt && claimState.preparationStartedAt.getTime() >= staleBefore.getTime()) {
+    return false
+  }
+  claimState.preparationStartedAt = claimedAt
+  return true
+})
+
+const renewPreparation = jest.fn(async (_requestId: string, renewedAt: Date) => {
+  if (claimState.status !== 'completed') return false
+  if (claimState.preparationCompletedAt) return false
+  if (!claimState.preparationStartedAt) return false
+  claimState.preparationStartedAt = renewedAt
+  return true
+})
+
+const findById = jest.fn(async (id: string) => ({
+  id,
+  status: claimState.status,
+  preparationCompletedAt: claimState.preparationCompletedAt,
+}))
+
+const markPreparationCompleted = jest.fn(async (_request: unknown, completedAt: Date) => {
+  claimState.preparationCompletedAt = completedAt
+  claimState.preparationStartedAt = null
+})
+
+jest.mock('@open-mercato/shared/lib/di/container', () => ({
+  createRequestContainer: jest.fn(async () => ({
+    resolve: (name: string) => {
+      if (name === 'em') return {}
+      throw new Error(`unexpected resolve(${name})`)
+    },
+  })),
+}))
+
+jest.mock('@open-mercato/shared/lib/modules/registry', () => ({
+  getModules: () => [{ id: 'catalog', setup: { seedExamples } }],
+}))
+
+jest.mock('@open-mercato/shared/lib/entities/system-entities', () => ({
+  flattenSystemEntityIds: () => ['catalog:product'],
+}))
+
+jest.mock('@open-mercato/shared/lib/encryption/entityIds', () => ({
+  getEntityIds: () => ({}),
+}))
+
+jest.mock('@open-mercato/core/modules/query_index/lib/reindexer', () => ({
+  reindexEntity: (...args: unknown[]) => reindexEntity(...(args as [])),
+}))
+
+jest.mock('@open-mercato/core/modules/query_index/lib/purge', () => ({
+  purgeIndexScope: (...args: unknown[]) => purgeIndexScope(...(args as [])),
+}))
+
+jest.mock('@open-mercato/core/modules/query_index/lib/coverage', () => ({
+  refreshCoverageSnapshot: (...args: unknown[]) => refreshCoverageSnapshot(...(args as [])),
+}))
+
+jest.mock('@open-mercato/onboarding/modules/onboarding/lib/ready-email', () => ({
+  sendWorkspaceReadyEmail: (...args: unknown[]) => sendWorkspaceReadyEmail(...(args as [])),
+}))
+
+jest.mock('@open-mercato/onboarding/modules/onboarding/lib/service', () => ({
+  OnboardingService: jest.fn().mockImplementation(() => ({
+    claimPreparation,
+    renewPreparation,
+    findById,
+    markPreparationCompleted,
+  })),
+}))
+
+import { runDeferredProvisioning } from '@open-mercato/onboarding/modules/onboarding/lib/deferred-provisioning'
+
+const RUN_ARGS = {
+  requestId: 'req-1',
+  tenantId: '11111111-1111-4111-8111-111111111111',
+  organizationId: '33333333-3333-4333-8333-333333333333',
+}
+
+describe('runDeferredProvisioning single-flight claim', () => {
+  beforeEach(() => {
+    jest.useFakeTimers({ doNotFake: ['setImmediate'] })
+    jest.clearAllMocks()
+    claimState.status = 'completed'
+    claimState.preparationStartedAt = null
+    claimState.preparationCompletedAt = null
+    sendWorkspaceReadyEmail.mockResolvedValue(true)
+  })
+
+  afterEach(() => {
+    jest.useRealTimers()
+  })
+
+  it('runs the provisioning chain only once when triggered concurrently by status polls (demo pool-exhaustion repro)', async () => {
+    // Repro for the 2026-06-11 demo outage: the preparing page polls
+    // /onboarding/status every ~1s and each poll scheduled a FULL
+    // runDeferredProvisioning chain (seedExamples for every module + a forced
+    // purge/reindex of every system entity) until preparationCompletedAt was
+    // flushed. Dozens of concurrent chains exhausted the 20-connection PG pool,
+    // the completion flag write itself timed out, and the loop never
+    // terminated. Concurrent triggers must collapse into exactly one run.
+    await Promise.all([
+      runDeferredProvisioning(RUN_ARGS),
+      runDeferredProvisioning(RUN_ARGS),
+      runDeferredProvisioning(RUN_ARGS),
+    ])
+
+    expect(seedExamples).toHaveBeenCalledTimes(1)
+    expect(markPreparationCompleted).toHaveBeenCalledTimes(1)
+    expect(purgeIndexScope).toHaveBeenCalledTimes(1)
+    expect(reindexEntity).toHaveBeenCalledTimes(1)
+    expect(sendWorkspaceReadyEmail).toHaveBeenCalledTimes(1)
+  })
+
+  it('does not run any heavy work when preparation already completed', async () => {
+    claimState.preparationCompletedAt = new Date()
+
+    await runDeferredProvisioning(RUN_ARGS)
+
+    expect(seedExamples).not.toHaveBeenCalled()
+    expect(purgeIndexScope).not.toHaveBeenCalled()
+    expect(reindexEntity).not.toHaveBeenCalled()
+    expect(sendWorkspaceReadyEmail).not.toHaveBeenCalled()
+  })
+
+  it('re-claims and recovers when a previous claim went stale', async () => {
+    claimState.preparationStartedAt = new Date(Date.now() - 20 * 60 * 1000)
+
+    await runDeferredProvisioning(RUN_ARGS)
+
+    expect(seedExamples).toHaveBeenCalledTimes(1)
+    expect(markPreparationCompleted).toHaveBeenCalledTimes(1)
+  })
+
+  it('renews the lease while working so a slow run never looks stale', async () => {
+    await runDeferredProvisioning(RUN_ARGS)
+
+    expect(renewPreparation).toHaveBeenCalled()
+    expect(renewPreparation.mock.invocationCallOrder[0]).toBeLessThan(
+      markPreparationCompleted.mock.invocationCallOrder[0],
+    )
+  })
+
+  it('does not complete a request that was re-submitted (reset to pending) mid-chain', async () => {
+    // A user can re-onboard the same email while an old chain is still running:
+    // createOrUpdateRequest resets the request to pending. The stale chain must
+    // not mark THAT request prepared — the new flow owns deferred provisioning.
+    findById.mockImplementationOnce(async (id: string) => ({
+      id,
+      status: 'pending',
+      preparationCompletedAt: null,
+    }))
+
+    await runDeferredProvisioning(RUN_ARGS)
+
+    expect(markPreparationCompleted).not.toHaveBeenCalled()
+  })
+
+  it('marks preparation completed only after the query-index rebuild (death mid-rebuild stays recoverable)', async () => {
+    // preparationCompletedAt is the terminal gate for both the status-route
+    // scheduling and claimPreparation. If it were written before the rebuild,
+    // a runner dying mid-rebuild would leave the tenant permanently without
+    // index rows — nothing would ever reclaim the run.
+    await runDeferredProvisioning(RUN_ARGS)
+
+    expect(reindexEntity).toHaveBeenCalled()
+    expect(markPreparationCompleted).toHaveBeenCalled()
+    expect(reindexEntity.mock.invocationCallOrder[0]).toBeLessThan(
+      markPreparationCompleted.mock.invocationCallOrder[0],
+    )
+  })
+
+  it('still rebuilds query indexes when the ready email fails', async () => {
+    // #2954's contract: post-provisioning steps are non-fatal. A transient
+    // SMTP failure must not abort the chain before the query-index rebuild,
+    // otherwise the tenant is left permanently without index rows (the
+    // completion flag is already set, so nothing ever retries the rebuild).
+    sendWorkspaceReadyEmail.mockRejectedValue(new Error('smtp down'))
+
+    await expect(runDeferredProvisioning(RUN_ARGS)).resolves.toBeUndefined()
+
+    expect(markPreparationCompleted).toHaveBeenCalledTimes(1)
+    expect(purgeIndexScope).toHaveBeenCalledTimes(1)
+    expect(reindexEntity).toHaveBeenCalledTimes(1)
+  })
+})

package/src/__tests__/status-endpoint-auth.test.ts

@@ -170,6 +170,44 @@ describe('onboarding status endpoint authorization', () => {
     expect(res.status).toBe(404)
   })
 
+  it('does not schedule deferred provisioning while a fresh preparation claim is active', async () => {
+    // Regression for the 2026-06-11 demo outage: every preparing-page poll
+    // scheduled another full deferred-provisioning chain until the completion
+    // flag landed, exhausting the PG connection pool. While a fresh claim is
+    // recorded the poll must not schedule another run.
+    findLatestByTenantId.mockResolvedValue(
+      makeRequest({
+        userId: '55555555-5555-4555-8555-555555555555',
+        preparationCompletedAt: null,
+        preparationStartedAt: new Date(),
+      }),
+    )
+
+    const res = await GET(
+      buildRequest({ tenantId: TENANT_ID, cookie: `om_login_tenant=${TENANT_ID}` }),
+    )
+
+    expect(res.status).toBe(200)
+    expect(runDeferredProvisioning).not.toHaveBeenCalled()
+  })
+
+  it('schedules deferred provisioning again once the preparation claim is stale', async () => {
+    findLatestByTenantId.mockResolvedValue(
+      makeRequest({
+        userId: '55555555-5555-4555-8555-555555555555',
+        preparationCompletedAt: null,
+        preparationStartedAt: new Date(Date.now() - 20 * 60 * 1000),
+      }),
+    )
+
+    const res = await GET(
+      buildRequest({ tenantId: TENANT_ID, cookie: `om_login_tenant=${TENANT_ID}` }),
+    )
+
+    expect(res.status).toBe(200)
+    expect(runDeferredProvisioning).toHaveBeenCalledTimes(1)
+  })
+
   it('recovers an interrupted processing request that already has provisioned ids', async () => {
     const request = makeRequest({
       status: 'processing',

package/src/modules/onboarding/__integration__/TC-ONB-002-multi-tenant-parallel-login.spec.ts

@@ -0,0 +1,198 @@
+// Adapted from PR #3007 (pkarw) — multi-tenant parallel onboarding repro for the
+// 2026-06-11 demo pool-exhaustion outage, ported to the preparation_started_at
+// lease column introduced by the single-flight deferred-provisioning fix.
+import { createHash, randomUUID } from 'node:crypto';
+import { expect, test, type APIRequestContext, type Browser, type BrowserContext, type Page } from '@playwright/test';
+import { withClient } from '@open-mercato/core/helpers/integration/dbFixtures';
+
+export const integrationMeta = {
+  dependsOnModules: ['onboarding'],
+  requiredEnvVars: ['SELF_SERVICE_ONBOARDING_ENABLED'],
+  requiredAnyEnvVars: ['CONSENT_INTEGRITY_SECRET', 'AUTH_SECRET', 'NEXTAUTH_SECRET', 'JWT_SECRET'],
+};
+
+type OnboardingTenant = {
+  email: string;
+  organizationName: string;
+  token: string;
+  requestId?: string;
+  tenantId?: string;
+};
+
+type BrowserTenantSession = {
+  context: BrowserContext;
+  page: Page;
+  refreshRequests: string[];
+};
+
+const BASE_URL = process.env.BASE_URL?.trim() || 'http://localhost:3000';
+const ONBOARDING_PASSWORD = 'ParallelPass123!';
+
+function hashToken(token: string): string {
+  return createHash('sha256').update(token).digest('hex');
+}
+
+async function replaceVerificationToken(email: string, token: string): Promise<string> {
+  return withClient(async (client) => {
+    const result = await client.query<{ id: string }>(
+      `update onboarding_requests
+         set token_hash = $2,
+             expires_at = now() + interval '24 hours',
+             updated_at = now()
+       where email = $1
+       returning id`,
+      [email, hashToken(token)],
+    );
+    expect(result.rowCount, `onboarding request should exist for ${email}`).toBe(1);
+    return result.rows[0].id;
+  });
+}
+
+async function readPreparationState(requestId: string): Promise<{
+  preparation_completed_at: Date | null;
+  preparation_started_at: Date | null;
+}> {
+  return withClient(async (client) => {
+    const result = await client.query<{
+      preparation_completed_at: Date | null;
+      preparation_started_at: Date | null;
+    }>(
+      `select preparation_completed_at, preparation_started_at
+         from onboarding_requests
+        where id = $1`,
+      [requestId],
+    );
+    expect(result.rowCount, `onboarding request ${requestId} should remain queryable`).toBe(1);
+    return result.rows[0];
+  });
+}
+
+async function waitForPreparationComplete(requestId: string): Promise<void> {
+  const deadline = Date.now() + 60_000;
+  let lastState: Awaited<ReturnType<typeof readPreparationState>> | null = null;
+  while (Date.now() < deadline) {
+    lastState = await readPreparationState(requestId);
+    if (lastState.preparation_completed_at && !lastState.preparation_started_at) return;
+    await new Promise((resolve) => setTimeout(resolve, 500));
+  }
+  expect(lastState?.preparation_started_at, 'deferred preparation lease should be cleared after completion').toBeNull();
+  expect(lastState?.preparation_completed_at, 'workspace preparation should complete').toBeTruthy();
+}
+
+async function submitOnboarding(request: APIRequestContext, tenant: OnboardingTenant): Promise<void> {
+  const response = await request.post(`${BASE_URL}/api/onboarding/onboarding`, {
+    data: {
+      email: tenant.email,
+      firstName: 'Parallel',
+      lastName: 'Login',
+      organizationName: tenant.organizationName,
+      password: ONBOARDING_PASSWORD,
+      confirmPassword: ONBOARDING_PASSWORD,
+      termsAccepted: true,
+      marketingConsent: true,
+      locale: 'en',
+    },
+  });
+  expect(response.status(), `onboarding start should succeed for ${tenant.email}`).toBe(200);
+}
+
+async function verifyTenantInBrowser(browser: Browser, token: string): Promise<BrowserTenantSession & { tenantId: string }> {
+  const context = await browser.newContext({ baseURL: BASE_URL });
+  const page = await context.newPage();
+  const refreshRequests: string[] = [];
+  page.on('request', (browserRequest) => {
+    const url = browserRequest.url();
+    if (url.includes('/api/auth/session/refresh')) refreshRequests.push(url);
+  });
+
+  await page.goto(`/api/onboarding/onboarding/verify?token=${encodeURIComponent(token)}`);
+  await expect(page).toHaveURL(/\/onboarding\/preparing\?tenant=[0-9a-f-]+/);
+  await expect(page.getByText('We are preparing your workspace')).toBeVisible();
+  const tenantId = new URL(page.url()).searchParams.get('tenant');
+  expect(tenantId, 'verify redirect should include tenant id').toMatch(
+    /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,
+  );
+  return { context, page, refreshRequests, tenantId: tenantId! };
+}
+
+async function pollOnboardingStatus(request: APIRequestContext, tenantId: string) {
+  return request.get(`${BASE_URL}/api/onboarding/onboarding/status?tenantId=${encodeURIComponent(tenantId)}`, {
+    headers: {
+      Cookie: `om_login_tenant=${encodeURIComponent(tenantId)}`,
+    },
+  });
+}
+
+async function loginAndAssertBackend(session: BrowserTenantSession, tenant: OnboardingTenant): Promise<void> {
+  expect(tenant.tenantId, `tenant id should exist for ${tenant.email}`).toBeTruthy();
+  await expect(session.page).toHaveURL(new RegExp(`/login\\?tenant=${tenant.tenantId}`));
+  await expect(session.page.locator('form[data-auth-ready="1"]')).toBeVisible();
+  await session.page.getByLabel('Email').fill(tenant.email);
+  await session.page.getByLabel('Password', { exact: true }).fill(ONBOARDING_PASSWORD);
+  await session.page.getByRole('button', { name: 'Sign in' }).click();
+  await expect(session.page, `browser login should land ${tenant.email} in the backend`).toHaveURL(/\/backend(?:\/.*)?$/);
+  await expect(session.page.getByText(/Session expired/i)).toHaveCount(0);
+
+  const profileResult = await session.page.evaluate(async () => {
+    const response = await fetch('/api/auth/profile', { credentials: 'include' });
+    const body = await response.json().catch(() => null);
+    return { status: response.status, body };
+  });
+  expect(profileResult.status, `browser page should keep ${tenant.email} authenticated`).toBe(200);
+  const profile = profileResult.body;
+  expect(profile.email).toBe(tenant.email);
+  expect(profile.roles).toContain('admin');
+
+  await session.page.goto('/backend');
+  await expect(session.page).toHaveURL(/\/backend(?:\/.*)?$/);
+  expect(session.refreshRequests, `backend should not bounce ${tenant.email} through session refresh`).toHaveLength(0);
+}
+
+test.describe('TC-ONB-002: multi-tenant onboarding parallel login', () => {
+  test('creates two self-service tenants, handles repeated status polling, and keeps both logins authenticated', async ({
+    browser,
+    request,
+  }) => {
+    // Two full onboardings + parallel browser logins + DB-polled preparation
+    // far exceed the 20s suite default (house pattern: TC-AI-AGENT-SETTINGS-005).
+    test.setTimeout(120_000);
+    const unique = randomUUID().slice(0, 8);
+    const tenants: OnboardingTenant[] = [1, 2].map((index) => ({
+      email: `qa-onboarding-parallel-${unique}-${index}@example.test`,
+      organizationName: `QA Onboarding Parallel ${unique} ${index}`,
+      token: `integration-parallel-${index}-${randomUUID().replace(/-/g, '')}`,
+    }));
+
+    await Promise.all(tenants.map((tenant) => submitOnboarding(request, tenant)));
+
+    for (const tenant of tenants) {
+      tenant.requestId = await replaceVerificationToken(tenant.email, tenant.token);
+    }
+
+    const browserSessions = await Promise.all(
+      tenants.map(async (tenant) => {
+        const session = await verifyTenantInBrowser(browser, tenant.token);
+        tenant.tenantId = session.tenantId;
+        return session;
+      }),
+    );
+
+    try {
+      const statusResponses = await Promise.all(
+        tenants.flatMap((tenant) =>
+          Array.from({ length: 6 }, () => pollOnboardingStatus(request, tenant.tenantId!)),
+        ),
+      );
+      for (const response of statusResponses) {
+        expect(response.status(), 'status polling should not exhaust the app DB pool').toBe(200);
+      }
+
+      await Promise.all(tenants.map((tenant) => waitForPreparationComplete(tenant.requestId!)));
+      await Promise.all(
+        tenants.map((tenant, index) => loginAndAssertBackend(browserSessions[index], tenant)),
+      );
+    } finally {
+      await Promise.all(browserSessions.map((session) => session.context.close()));
+    }
+  });
+});

package/src/modules/onboarding/api/get/onboarding/status.ts

@@ -9,6 +9,7 @@ import {
   resolveProvisioningIds,
   runDeferredProvisioning,
 } from '@open-mercato/onboarding/modules/onboarding/lib/deferred-provisioning'
+import { isPreparationClaimActive } from '@open-mercato/onboarding/modules/onboarding/lib/preparation-claim'
 import type { OpenApiMethodDoc, OpenApiRouteDoc } from '@open-mercato/shared/lib/openapi'
 
 export const metadata = {
@@ -78,7 +79,16 @@ export async function GET(req: Request) {
   if (provisioningIds && request.status === 'processing') {
     await service.markCompleted(request, provisioningIds)
   }
-  if (provisioningIds && request.status === 'completed' && !request.preparationCompletedAt) {
+  // Schedule deferred provisioning only while no runner holds a fresh claim —
+  // otherwise every ~1s poll piles another full seed + reindex chain onto the
+  // connection pool. The atomic claim inside runDeferredProvisioning remains
+  // the authoritative gate; this check just keeps polls cheap.
+  if (
+    provisioningIds &&
+    request.status === 'completed' &&
+    !request.preparationCompletedAt &&
+    !isPreparationClaimActive(request.preparationStartedAt)
+  ) {
     after(async () => {
       await runDeferredProvisioning({
         requestId: request.id,

package/src/modules/onboarding/data/entities.ts

@@ -60,6 +60,9 @@ export class OnboardingRequest {
   @Property({ name: 'last_email_sent_at', type: Date, nullable: true })
   lastEmailSentAt?: Date | null
 
+  @Property({ name: 'preparation_started_at', type: Date, nullable: true })
+  preparationStartedAt?: Date | null
+
   @Property({ name: 'preparation_completed_at', type: Date, nullable: true })
   preparationCompletedAt?: Date | null
 

package/src/modules/onboarding/lib/deferred-provisioning.ts

@@ -11,6 +11,7 @@ import { reindexEntity } from '@open-mercato/core/modules/query_index/lib/reinde
 import { purgeIndexScope } from '@open-mercato/core/modules/query_index/lib/purge'
 import { refreshCoverageSnapshot } from '@open-mercato/core/modules/query_index/lib/coverage'
 import { isUniqueViolation } from '@open-mercato/shared/lib/crud/errors'
+import { PREPARATION_CLAIM_STALE_MS } from '@open-mercato/onboarding/modules/onboarding/lib/preparation-claim'
 
 const VECTOR_REINDEX_ENQUEUE_TIMEOUT_MS = 5_000
 const SEED_EXAMPLES_TIMEOUT_MS = 15_000
@@ -80,13 +81,14 @@ async function runModuleSetupHook(args: {
 
 async function markWorkspaceReady(args: {
   requestId: string
+  service: OnboardingService
 }) {
-  const container = await createRequestContainer()
-  const em = container.resolve('em') as EntityManager
-  const service = new OnboardingService(em)
-  const request = await service.findById(args.requestId)
-  if (!request || request.preparationCompletedAt) return
-  await service.markPreparationCompleted(request, new Date())
+  const request = await args.service.findById(args.requestId)
+  // The status guard protects a request that was re-submitted (and reset to
+  // pending) while this chain was still running — completing it would let the
+  // new flow skip its own deferred provisioning.
+  if (!request || request.preparationCompletedAt || request.status !== 'completed') return
+  await args.service.markPreparationCompleted(request, new Date())
 }
 
 async function enqueueVectorReindex(args: {
@@ -187,10 +189,36 @@ export async function runDeferredProvisioning(args: {
 }) {
   const container = await createRequestContainer()
   const em = container.resolve('em') as EntityManager
+  const service = new OnboardingService(em)
+
+  // Single-flight guard: the preparing page polls the status endpoint every
+  // second and each poll (plus the verify handler) schedules this chain. The
+  // atomic claim collapses those triggers into one run per request — without
+  // it, dozens of concurrent seed + full-reindex chains exhaust the PG
+  // connection pool (2026-06-11 demo outage). A stale claim (crashed runner)
+  // becomes reclaimable after PREPARATION_CLAIM_STALE_MS.
+  const claimedAt = new Date()
+  const claimed = await service.claimPreparation(
+    args.requestId,
+    claimedAt,
+    new Date(claimedAt.getTime() - PREPARATION_CLAIM_STALE_MS),
+  )
+  if (!claimed) {
+    console.info('[onboarding.verify] deferred provisioning skipped (already claimed or completed)', {
+      requestId: args.requestId,
+      tenantId: args.tenantId,
+    })
+    return
+  }
+
   const modules = getModules()
 
   for (const mod of modules) {
     if (!mod.setup?.seedExamples) continue
+    // Heartbeat: keep the lease fresh while legitimately working so a slow run
+    // (many modules × 15s seed timeout + rebuild) can never look stale and get
+    // double-claimed by a later poll.
+    await service.renewPreparation(args.requestId, new Date()).catch(() => {})
     try {
       await runModuleSetupHook({
         moduleId: mod.id,
@@ -221,10 +249,26 @@ export async function runDeferredProvisioning(args: {
     }
   }
 
+  // The rebuild runs BEFORE the completion flag: preparationCompletedAt is the
+  // terminal gate for both the status-route scheduling and claimPreparation,
+  // so a runner that dies mid-rebuild must leave the flag unset — the stale
+  // claim then makes the whole chain reclaimable and the rebuild self-heals.
+  // rebuildTenantQueryIndexes never throws (it logs per-entity failures).
+  await service.renewPreparation(args.requestId, new Date()).catch(() => {})
+  await rebuildTenantQueryIndexes({
+    em,
+    tenantId: args.tenantId,
+    organizationId: args.organizationId,
+  })
+
   await markWorkspaceReady({
     requestId: args.requestId,
+    service,
   })
 
+  // Non-fatal (#2954 contract): an email failure must not abort the chain.
+  // The status endpoint retries the ready email on later polls while
+  // readyEmailSentAt is unset.
   await sendWorkspaceReadyEmail({
     requestId: args.requestId,
     tenantId: args.tenantId,
@@ -235,13 +279,6 @@ export async function runDeferredProvisioning(args: {
       organizationId: args.organizationId,
       error,
     })
-    throw error
-  })
-
-  await rebuildTenantQueryIndexes({
-    em,
-    tenantId: args.tenantId,
-    organizationId: args.organizationId,
   })
 
   await enqueueVectorReindex({

package/src/modules/onboarding/lib/preparation-claim.ts

@@ -0,0 +1,6 @@
+export const PREPARATION_CLAIM_STALE_MS = 10 * 60 * 1000
+
+export function isPreparationClaimActive(startedAt: Date | null | undefined, now: Date = new Date()): boolean {
+  if (!startedAt) return false
+  return startedAt.getTime() > now.getTime() - PREPARATION_CLAIM_STALE_MS
+}