@shipfox/api-runners 12.3.0 → 12.4.1

package/src/db/runner-instances.ts

@@ -17,15 +17,20 @@ import {
   sql,
 } from 'drizzle-orm';
 import {alias} from 'drizzle-orm/pg-core';
+import {config} from '#config.js';
 import type {RunnerInstance, RunnerInstanceState} from '#core/entities/runner-instance.js';
 import {sanitizeRunnerLabels} from '#core/runner-labels.js';
+import {recordRunnerReservationCapacityFailure} from '#metrics/index.js';
 import type {Tx} from './db.js';
 import {db} from './db.js';
 import {
   listRunningJobExecutionsByRunnerInstanceTx,
   type RunnerInstanceBoundJobExecution,
 } from './job-executions.js';
+import {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';
 import {releaseReservationUnits} from './reservations.js';
+import {validateRunnerReservationCapacityTx} from './runner-assignments.js';
+import {terminalStates} from './runner-states.js';
 import {ephemeralRegistrationTokens} from './schema/ephemeral-registration-tokens.js';
 import {provisionerTokens} from './schema/provisioner-tokens.js';
 import {reservations} from './schema/reservations.js';
@@ -34,11 +39,6 @@ import {providerRunners, toRunnerInstance} from './schema/runner-instances.js';
 import {runnerSessions} from './schema/runner-sessions.js';
 import {runningJobExecutions} from './schema/running-job-executions.js';
 
-export const terminalStates = [
-  'stopped',
-  'failed',
-  'terminated',
-] as const satisfies readonly RunnerInstanceState[];
 export const activeStates = [
   'starting',
   'running',
@@ -49,11 +49,12 @@ export const divergenceCountStates = ['starting', 'running'] as const satisfies
   'starting' | 'running'
 >[];
 
-export type RunnerInstanceTerminateIntentReason = 'job-cancelled';
+export type RunnerInstanceTerminateIntentReason = 'activation-timeout' | 'job-cancelled';
 
 export interface RunnerInstanceTerminateIntent {
   providerRunnerId: string;
   reason: RunnerInstanceTerminateIntentReason;
+  activationTimeoutRetry?: boolean;
 }
 
 export interface ActiveRunnerInstanceTemplateCount {
@@ -155,13 +156,14 @@ export async function reportRunnerInstances(params: ReportRunnerInstancesParams)
     }
 
     const events = await hydrateRunnerSessionIdsFromConsumedTokens(tx, params, aggregatedEvents);
+    const reservationSafeEvents = await guardReportedReservationIdsTx(tx, params, events);
     const terminateIntentsHonored = await listTerminateIntentsHonoredByTerminatedReportsTx(
       tx,
       params,
-      events,
+      reservationSafeEvents,
     );
 
-    const values = events.map((event) => ({
+    const values = reservationSafeEvents.map((event) => ({
       workspaceId: params.workspaceId,
       provisionerId: params.provisionerId,
       providerRunnerId: event.providerRunnerId,
@@ -213,10 +215,105 @@ export async function reportRunnerInstances(params: ReportRunnerInstancesParams)
       });
 
     const reservationsReleased = hasTerminalEvent
-      ? await releaseTerminalRunnerInstanceReservations(tx, params, events)
+      ? await releaseTerminalRunnerInstanceReservations(tx, params, reservationSafeEvents)
       : 0;
 
-    return {accepted: events.length, reservationsReleased, terminateIntentsHonored};
+    return {
+      accepted: reservationSafeEvents.length,
+      reservationsReleased,
+      terminateIntentsHonored,
+    };
+  });
+}
+
+async function guardReportedReservationIdsTx(
+  tx: Tx,
+  params: ReportRunnerInstancesParams,
+  events: RunnerInstanceReportRow[],
+): Promise<RunnerInstanceReportRow[]> {
+  const reservationIds = [
+    ...new Set(events.flatMap((event) => (event.reservationId ? [event.reservationId] : []))),
+  ].sort();
+  if (reservationIds.length === 0) return events;
+
+  await lockRunnerReservationAdvisoryKeysTx(tx, {
+    provisionerId: params.provisionerId,
+    reservationIds,
+  });
+  const providerRunnerIds = events.map((event) => event.providerRunnerId);
+  const existingRows = await tx
+    .select({
+      providerRunnerId: providerRunners.providerRunnerId,
+      reservationId: providerRunners.reservationId,
+      intendedReservationId: providerRunners.intendedReservationId,
+      reservationReleasedAt: providerRunners.reservationReleasedAt,
+    })
+    .from(providerRunners)
+    .where(
+      and(
+        eq(providerRunners.provisionerId, params.provisionerId),
+        inArray(providerRunners.providerRunnerId, providerRunnerIds),
+      ),
+    )
+    .for('update');
+  const existingByProviderRunnerId = new Map(
+    existingRows.flatMap((row) =>
+      row.providerRunnerId ? [[row.providerRunnerId, row] as const] : [],
+    ),
+  );
+  const candidateReservationByProviderRunnerId = new Map<string, string>();
+  for (const event of events) {
+    if (!event.reservationId) continue;
+    if (isTerminalState(event.state)) continue;
+    const existing = existingByProviderRunnerId.get(event.providerRunnerId);
+    if (existing?.reservationId || existing?.intendedReservationId === event.reservationId)
+      continue;
+    if (existing?.intendedReservationId || existing?.reservationReleasedAt) continue;
+    candidateReservationByProviderRunnerId.set(event.providerRunnerId, event.reservationId);
+  }
+
+  const validation = await validateRunnerReservationCapacityTx(
+    tx,
+    {
+      provisionerId: params.provisionerId,
+      requests: [...candidateReservationByProviderRunnerId.values()].map((reservationId) => ({
+        reservationId,
+        count: 1,
+      })),
+    },
+    {advisoryLocksHeld: true},
+  );
+  for (const {reason, count} of validation.unavailableByReservation.values())
+    recordRunnerReservationCapacityFailure(reason, count);
+
+  const remainingAcceptedByReservation = new Map(validation.acceptedByReservation);
+  return events.map((event) => {
+    const existing = existingByProviderRunnerId.get(event.providerRunnerId);
+    const reservationId = candidateReservationByProviderRunnerId.get(event.providerRunnerId);
+    if (!reservationId) {
+      if (event.reservationId && isTerminalState(event.state)) {
+        // Terminal cleanup releases the IDs stored on the projection row. An arbitrary
+        // terminal report must not consume reservation capacity during that cleanup.
+        const matchesExistingReservation =
+          existing !== undefined &&
+          (existing.reservationId === event.reservationId ||
+            existing.intendedReservationId === event.reservationId);
+        if (!matchesExistingReservation) return {...event, reservationId: null};
+      }
+      if (
+        event.reservationId &&
+        existing?.intendedReservationId &&
+        existing.intendedReservationId !== event.reservationId
+      )
+        return {...event, reservationId: null};
+      if (event.reservationId && existing?.reservationReleasedAt)
+        return {...event, reservationId: null};
+      return event;
+    }
+    const remaining = remainingAcceptedByReservation.get(reservationId) ?? 0;
+    if (remaining === 0) return {...event, reservationId: null};
+    remainingAcceptedByReservation.set(reservationId, remaining - 1);
+    return event;
   });
 }
 
@@ -278,6 +375,80 @@ export async function countStaleEnrolledRunnerInstances(params: {
   return row?.count ?? 0;
 }
 
+export type ProviderRunnerPhase =
+  | 'control_session'
+  | 'enrollment'
+  | 'assignment'
+  | 'activation'
+  | 'idle';
+
+export interface ProviderRunnerPhaseMetric {
+  phase: ProviderRunnerPhase;
+  provider: string;
+  launchKind: 'demand' | 'warm' | 'manual';
+  count: number;
+  oldestAgeMilliseconds: number;
+}
+
+export async function listProviderRunnerByPhaseMetrics(): Promise<ProviderRunnerPhaseMetric[]> {
+  const phase = sql<ProviderRunnerPhase>`case
+    when ${runnerControlSessions.id} is null then 'control_session'
+    when ${providerRunners.state} <> 'running' then 'enrollment'
+    when ${providerRunners.intendedReservationId} is not null
+      and ${providerRunners.workspaceId} is null then 'assignment'
+    when ${providerRunners.workspaceId} is not null then 'activation'
+    else 'idle'
+  end`;
+  const startedAt = sql<Date | null>`case
+    when ${runnerControlSessions.id} is null then ${providerRunners.createdAt}
+    when ${providerRunners.state} <> 'running' then ${runnerControlSessions.createdAt}
+    when ${providerRunners.intendedReservationId} is not null
+      and ${providerRunners.workspaceId} is null then ${runnerControlSessions.createdAt}
+    when ${providerRunners.workspaceId} is not null
+      then coalesce(${providerRunners.assignedAt}, ${runnerControlSessions.createdAt})
+    else coalesce(
+      ${runnerControlSessions.createdAt},
+      ${providerRunners.assignedAt},
+      ${providerRunners.createdAt}
+    )
+  end`;
+  const rows = await db()
+    .select({
+      phase,
+      provider: sql<string>`coalesce(${providerRunners.providerKind}, 'unknown')`,
+      launchKind: providerRunners.launchKind,
+      count: sql<number>`count(*)::int`,
+      oldestAgeMilliseconds: sql<number>`coalesce(
+        max(extract(epoch from (now() - (${startedAt}))) * 1000),
+        0
+      )::double precision`,
+    })
+    .from(providerRunners)
+    .leftJoin(
+      runnerControlSessions,
+      and(
+        eq(runnerControlSessions.runnerInstanceId, providerRunners.id),
+        isNull(runnerControlSessions.closedAt),
+        gt(runnerControlSessions.expiresAt, sql`now()`),
+      ),
+    )
+    .where(
+      and(
+        inArray(providerRunners.state, ['starting', 'running']),
+        isNull(providerRunners.runnerSessionId),
+      ),
+    )
+    .groupBy(phase, providerRunners.providerKind, providerRunners.launchKind);
+
+  return rows.map((row) => ({
+    phase: row.phase,
+    provider: row.provider,
+    launchKind: row.launchKind,
+    count: row.count,
+    oldestAgeMilliseconds: Math.max(0, row.oldestAgeMilliseconds),
+  }));
+}
+
 export async function listActiveRunnerInstances(params: {
   workspaceId: string;
   windowSeconds: number;
@@ -336,9 +507,42 @@ export async function listProvisionerTerminateIntentRowsTx(
     );
   }
 
-  return returnedRows.flatMap((row) =>
-    row.providerRunnerId ? [{...row, providerRunnerId: row.providerRunnerId}] : [],
+  const activationTimeoutRunnerIds = returnedRows.flatMap((row) =>
+    row.reason === 'activation-timeout' && row.providerRunnerId ? [row.providerRunnerId] : [],
   );
+  if (activationTimeoutRunnerIds.length > 0) {
+    await tx
+      .update(providerRunners)
+      .set({reservationReleasedAt: sql`now()`, updatedAt: sql`now()`})
+      .where(
+        and(
+          eq(providerRunners.workspaceId, params.workspaceId),
+          eq(providerRunners.provisionerId, params.provisionerId),
+          inArray(providerRunners.providerRunnerId, activationTimeoutRunnerIds),
+          isNull(providerRunners.reservationReleasedAt),
+        ),
+      );
+  }
+
+  return returnedRows.flatMap((row) => toRunnerInstanceTerminateIntent(row));
+}
+
+function toRunnerInstanceTerminateIntent(row: {
+  providerRunnerId: string | null;
+  reason: RunnerInstanceTerminateIntentReason;
+  activationTimeoutRetry: boolean;
+}): RunnerInstanceTerminateIntent[] {
+  if (!row.providerRunnerId) return [];
+
+  return [
+    {
+      providerRunnerId: row.providerRunnerId,
+      reason: row.reason,
+      ...(row.reason === 'activation-timeout' && row.activationTimeoutRetry
+        ? {activationTimeoutRetry: true}
+        : {}),
+    },
+  ];
 }
 
 function provisionerTerminateIntentsQuery(
@@ -346,59 +550,91 @@ function provisionerTerminateIntentsQuery(
   params: {workspaceId: string; provisionerId: string; providerRunnerIds?: string[]},
 ) {
   const newerRunningJobExecutions = alias(runningJobExecutions, 'newer_running_jobs');
+  const latestCancelledJob = exists(
+    tx
+      .select({id: runningJobExecutions.id})
+      .from(runningJobExecutions)
+      .where(
+        and(
+          eq(runningJobExecutions.workspaceId, params.workspaceId),
+          eq(runningJobExecutions.provisionerId, params.provisionerId),
+          eq(runningJobExecutions.providerRunnerId, providerRunners.providerRunnerId),
+          isNotNull(runningJobExecutions.cancellationRequestedAt),
+          notExists(
+            tx
+              .select({id: newerRunningJobExecutions.id})
+              .from(newerRunningJobExecutions)
+              .where(
+                and(
+                  eq(newerRunningJobExecutions.workspaceId, runningJobExecutions.workspaceId),
+                  eq(newerRunningJobExecutions.provisionerId, runningJobExecutions.provisionerId),
+                  eq(
+                    newerRunningJobExecutions.providerRunnerId,
+                    runningJobExecutions.providerRunnerId,
+                  ),
+                  or(
+                    gt(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt),
+                    and(
+                      eq(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt),
+                      gt(
+                        newerRunningJobExecutions.jobExecutionId,
+                        runningJobExecutions.jobExecutionId,
+                      ),
+                    ),
+                  ),
+                ),
+              ),
+          ),
+        ),
+      ),
+  );
+  const activationTimeout = and(
+    eq(providerRunners.launchKind, 'demand'),
+    isNull(providerRunners.runnerSessionId),
+    lt(
+      providerRunners.createdAt,
+      sql`now() - (${config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS} || ' seconds')::interval`,
+    ),
+    notExists(
+      tx
+        .select({id: reservations.id})
+        .from(reservations)
+        .where(
+          and(
+            or(
+              eq(reservations.id, providerRunners.reservationId),
+              eq(reservations.id, providerRunners.intendedReservationId),
+            ),
+            eq(reservations.workspaceId, params.workspaceId),
+            eq(reservations.provisionerId, params.provisionerId),
+            gt(reservations.expiresAt, sql`now()`),
+          ),
+        ),
+    ),
+  );
 
   return tx
     .select({
       providerRunnerId: providerRunners.providerRunnerId,
-      reason: sql<RunnerInstanceTerminateIntentReason>`'job-cancelled'`,
+      activationTimeoutRetry: sql<boolean>`${providerRunners.reservationReleasedAt} is not null`,
+      reason: sql<RunnerInstanceTerminateIntentReason>`case
+        when ${activationTimeout} then 'activation-timeout'
+        else 'job-cancelled'
+      end`,
     })
-    .from(runningJobExecutions)
-    .innerJoin(
-      providerRunners,
-      and(
-        eq(providerRunners.workspaceId, runningJobExecutions.workspaceId),
-        eq(providerRunners.provisionerId, runningJobExecutions.provisionerId),
-        eq(providerRunners.providerRunnerId, runningJobExecutions.providerRunnerId),
-      ),
-    )
+    .from(providerRunners)
     .where(
       and(
-        eq(runningJobExecutions.workspaceId, params.workspaceId),
-        eq(runningJobExecutions.provisionerId, params.provisionerId),
+        eq(providerRunners.workspaceId, params.workspaceId),
+        eq(providerRunners.provisionerId, params.provisionerId),
         isNotNull(providerRunners.providerRunnerId),
-        isNotNull(runningJobExecutions.cancellationRequestedAt),
         inArray(providerRunners.state, activeStates),
         params.providerRunnerIds && params.providerRunnerIds.length > 0
           ? inArray(providerRunners.providerRunnerId, params.providerRunnerIds)
           : undefined,
-        notExists(
-          tx
-            .select({id: newerRunningJobExecutions.id})
-            .from(newerRunningJobExecutions)
-            .where(
-              and(
-                eq(newerRunningJobExecutions.workspaceId, runningJobExecutions.workspaceId),
-                eq(newerRunningJobExecutions.provisionerId, runningJobExecutions.provisionerId),
-                eq(
-                  newerRunningJobExecutions.providerRunnerId,
-                  runningJobExecutions.providerRunnerId,
-                ),
-                or(
-                  gt(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt),
-                  and(
-                    eq(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt),
-                    gt(
-                      newerRunningJobExecutions.jobExecutionId,
-                      runningJobExecutions.jobExecutionId,
-                    ),
-                  ),
-                ),
-              ),
-            ),
-        ),
+        or(latestCancelledJob, activationTimeout),
       ),
-    )
-    .groupBy(providerRunners.providerRunnerId);
+    );
 }
 
 async function listTerminateIntentsHonoredByTerminatedReportsTx(
@@ -419,9 +655,7 @@ async function listTerminateIntentsHonoredByTerminatedReportsTx(
     provisionerId: params.provisionerId,
     providerRunnerIds: terminatedRunnerInstanceIds,
   }).orderBy(asc(providerRunners.providerRunnerId));
-  return rows.flatMap((row) =>
-    row.providerRunnerId ? [{...row, providerRunnerId: row.providerRunnerId}] : [],
-  );
+  return rows.flatMap((row) => toRunnerInstanceTerminateIntent(row));
 }
 
 export async function reconcileRunnerInstances(

package/src/db/runner-sessions.ts

@@ -3,6 +3,10 @@ import {and, asc, eq, gt, inArray, isNull, lt, notExists, or, sql} from 'drizzle
 import type {RunnerSession} from '#core/entities/runner-session.js';
 import {EmptyRunnerLabelsError} from '#core/errors.js';
 import {sanitizeRunnerLabelsOrThrow} from '#core/runner-labels.js';
+import {
+  type ProviderRunnerLifecycleObservation,
+  recordProviderRunnerAssignmentToActivation,
+} from '#metrics/instance.js';
 import {db} from './db.js';
 import {provisionerTokens} from './schema/provisioner-tokens.js';
 import {runnerActivationTokens} from './schema/runner-activation-tokens.js';
@@ -58,7 +62,8 @@ export async function createRunnerSessionConsumingActivationToken(params: {
   labels: string[];
   toolCapabilities?: RunnerToolCapabilitiesDto | null;
 }) {
-  return await db().transaction(async (tx) => {
+  let assignmentToActivationObservation: ProviderRunnerLifecycleObservation | null = null;
+  const session = await db().transaction(async (tx) => {
     const [runner] = await tx
       .select({
         activationTokenId: runnerActivationTokens.id,
@@ -66,6 +71,9 @@ export async function createRunnerSessionConsumingActivationToken(params: {
         workspaceId: providerRunners.workspaceId,
         provisionerId: providerRunners.provisionerId,
         providerRunnerId: providerRunners.providerRunnerId,
+        assignedAt: providerRunners.assignedAt,
+        provider: providerRunners.providerKind,
+        launchKind: providerRunners.launchKind,
         runnerSessionId: providerRunners.runnerSessionId,
       })
       .from(runnerActivationTokens)
@@ -143,8 +151,18 @@ export async function createRunnerSessionConsumingActivationToken(params: {
           isNull(runnerControlSessions.closedAt),
         ),
       );
+    if (runner.assignedAt)
+      assignmentToActivationObservation = {
+        durationMilliseconds: session.createdAt.getTime() - runner.assignedAt.getTime(),
+        provider: runner.provider,
+        launchKind: runner.launchKind,
+        runnerInstanceId: runner.runnerInstanceId,
+      };
     return toRunnerSession(session);
   });
+  if (assignmentToActivationObservation)
+    recordProviderRunnerAssignmentToActivation(assignmentToActivationObservation);
+  return session;
 }
 
 export interface DeleteExpiredRunnerSessionsParams {

package/src/db/runner-states.ts

@@ -0,0 +1,7 @@
+import type {RunnerInstanceState} from '#core/entities/runner-instance.js';
+
+export const terminalStates = [
+  'stopped',
+  'failed',
+  'terminated',
+] as const satisfies readonly RunnerInstanceState[];

package/src/db/schema/reservations.ts

@@ -1,9 +1,11 @@
 import {uuidv7PrimaryKey} from '@shipfox/node-drizzle';
 import {sql} from 'drizzle-orm';
-import {check, index, integer, text, timestamp, uuid} from 'drizzle-orm/pg-core';
+import {check, index, integer, pgEnum, text, timestamp, uuid} from 'drizzle-orm/pg-core';
 import type {Reservation} from '#core/entities/reservation.js';
 import {pgTable} from './common.js';
 
+export const reservationKindEnum = pgEnum('runners_reservation_kind', ['bound', 'launch']);
+
 export const reservations = pgTable(
   'reservations',
   {
@@ -12,6 +14,7 @@ export const reservations = pgTable(
     provisionerId: uuid('provisioner_id').notNull(),
     requiredLabels: text('required_labels').array().notNull(),
     count: integer('count').notNull(),
+    kind: reservationKindEnum('kind').notNull().default('launch'),
     createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),
     expiresAt: timestamp('expires_at', {withTimezone: true}).notNull(),
   },
@@ -32,6 +35,7 @@ export function toReservation(row: ReservationDb): Reservation {
     provisionerId: row.provisionerId,
     requiredLabels: row.requiredLabels,
     count: row.count,
+    kind: row.kind,
     createdAt: row.createdAt,
     expiresAt: row.expiresAt,
   };

package/src/db/schema/runner-instances.ts

@@ -14,6 +14,12 @@ export const providerRunnerStateEnum = pgEnum('runners_provider_runner_state', [
   'terminated',
 ]);
 
+export const providerRunnerLaunchKindEnum = pgEnum('runners_provider_runner_launch_kind', [
+  'demand',
+  'warm',
+  'manual',
+]);
+
 export const providerRunners = pgTable(
   'runner_instances',
   {
@@ -25,6 +31,7 @@ export const providerRunners = pgTable(
     providerRunnerId: text('provider_runner_id'),
     intendedReservationId: uuid('intended_reservation_id'),
     reservationId: uuid('reservation_id'),
+    launchKind: providerRunnerLaunchKindEnum('launch_kind').notNull().default('manual'),
     assignedAt: timestamp('assigned_at', {withTimezone: true}),
     templateKey: text('template_key'),
     labels: text('labels').array().notNull().default([]),
@@ -82,7 +89,9 @@ export function toRunnerInstance(row: RunnerInstanceDb): RunnerInstance {
     workspaceId: row.workspaceId,
     provisionerId: row.provisionerId,
     providerRunnerId: row.providerRunnerId,
+    intendedReservationId: row.intendedReservationId,
     reservationId: row.reservationId,
+    launchKind: row.launchKind,
     assignedAt: row.assignedAt,
     templateKey: row.templateKey,
     labels: row.labels,

package/src/metrics/index.ts

@@ -1,13 +1,36 @@
-export type {RunnerReservationPromotionFailureReason} from './instance.js';
+export type {
+  RunnerActivationTokenNotIssuedReason,
+  RunnerActivationTokenNotIssuedSurface,
+  RunnerAssignmentSurface,
+  RunnerLaunchKind,
+  RunnerReservationCapacityFailureReason,
+  RunnerReservationPromotionFailureReason,
+} from './instance.js';
 export {
   jobExecutionClaimedCount,
   jobExecutionEnqueuedCount,
   jobExecutionLeaseExpiredCount,
+  jobExecutionQueueTimeDuration,
   providerRunnerAbsentTerminatedCount,
+  providerRunnerActivationOutcomeCount,
+  providerRunnerActivationToFirstClaimDuration,
+  providerRunnerAssignmentRejectedCount,
+  providerRunnerAssignmentToActivationDuration,
+  providerRunnerControlSessionToAssignmentDuration,
   providerRunnerCountDivergenceCount,
+  providerRunnerCreatedToControlSessionDuration,
   providerRunnerReconcileCallCount,
   providerRunnerTerminateIntentHonoredCount,
   providerRunnerTerminateIntentIssuedCount,
+  recordJobExecutionQueueTime,
+  recordProviderRunnerActivationOutcome,
+  recordProviderRunnerActivationToFirstClaim,
+  recordProviderRunnerAssignmentRejected,
+  recordProviderRunnerAssignmentToActivation,
+  recordProviderRunnerControlSessionToAssignment,
+  recordProviderRunnerCreatedToControlSession,
+  recordRunnerActivationTokenNotIssued,
+  recordRunnerReservationCapacityFailure,
   recordRunnerReservationPromotionFailure,
   recordRunnersRateLimitCheck,
   recordRunnersRateLimitPruneFailure,