@things-factory/operato-hub 4.3.822 → 4.3.824

package/server/graphql/resolvers/synchronization/powrup-get-airway-bill.ts

@@ -7,87 +7,140 @@ import { ORDER_STATUS, OrderPackage } from '@things-factory/sales-base'
 
 const { PerformanceObserver, performance } = require('node:perf_hooks')
 
+// Stable key for pg_try_advisory_lock — keeps two concurrent invocations of this
+// resolver from contending on the same order_packages rows.
+const ADVISORY_LOCK_KEY = 'powrup-get-airway-bill'
+
 export const powrupGetAirwayBillResolver = {
   async powrupGetAirwayBill(_: void, { }, context: any): Promise<void> {
+    // We deliberately do NOT wrap the resolver in a transaction. The previous
+    // implementation held a single long transaction across up to 100 external
+    // PowrupAPI HTTP calls, keeping row locks on every UPDATE'd order_packages
+    // row until the loop finished — minutes of contention against the packing
+    // UI, sellercraft updates, and any other path that touches those rows.
+    //
+    // Instead we use a single QueryRunner connection for the duration of the
+    // resolver, acquire a session-level advisory lock so concurrent runs skip
+    // early, and let each UPDATE auto-commit on its own (row lock held only for
+    // the duration of that single statement).
+    const queryRunner = getConnection().createQueryRunner()
+    let connected = false
+
     try {
-      await getConnection().transaction(async tx => {
-        try {
-          const powrup: Powrup = await tx
-            .getRepository(Powrup)
-            .findOne({ where: { platform: Platform.POWRUP, active: true } })
+      await queryRunner.connect()
+      connected = true
 
-          let orderPackageQb = await tx
-            .getRepository(OrderPackage)
-            .createQueryBuilder('opk')
-            .innerJoinAndSelect('opk.releaseGood', 'rg')
-            .innerJoinAndSelect('opk.domain', 'domain')
-            .innerJoinAndSelect('opk.bizplace', 'bizplace')
-            .innerJoinAndSelect('opk.creator', 'creator')
-            .where(`rg.source = :source`, { source: ApplicationType.POWRUP })
-            .andWhere(`rg.type = :rgType`, { rgType: 'b2c' })
-            .andWhere(`rg.ref_order_id NOTNULL`)
-            .andWhere(`(rg.lmd_option = false or rg.lmd_option ISNULL)`)
-            .andWhere(`rg.status NOT IN (:...status) `, {
-              status: [ORDER_STATUS.PACKING, ORDER_STATUS.CANCELLED, ORDER_STATUS.PENDING_CANCEL, ORDER_STATUS.DONE]
-            })
-            .andWhere(`opk.airway_bill ISNULL`)
-            .orderBy(`rg.created_at`)
-            .limit(100)
+      const lockResult: { acquired: boolean }[] = await queryRunner.query(
+        `SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS acquired`,
+        [ADVISORY_LOCK_KEY]
+      )
+      if (!lockResult?.[0]?.acquired) {
+        logger.info(`powrupGetAirwayBill: another instance is running, skipping`)
+        return
+      }
 
-          let orderPackagesWithoutAirwayBill: OrderPackage[] = await orderPackageQb.getMany()
+      try {
+        const manager = queryRunner.manager
 
-          if (orderPackagesWithoutAirwayBill?.length > 0) {
-            for (let i = 0; i < orderPackagesWithoutAirwayBill.length; i++) {
-              const orderPackage: OrderPackage = orderPackagesWithoutAirwayBill[i]
-              performance.mark(`start:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
-              try {
-                const {
-                  trackingNo,
-                  transporter,
-                  printLabels
-                }: { trackingNo: string; transporter: string; printLabels: any[] } = await PowrupAPI.fetchDocument(
-                  powrup,
-                  {
-                    context: { state: { domain: null, user: null } },
-                    refOrderId: orderPackage.releaseGood.refOrderId,
-                    packageId: orderPackage.packageId
-                  }
-                )
-                performance.mark(`end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
-                let updateInfo: any
-                if (printLabels?.length) {
-                  printLabels.map(pl => {
-                    updateInfo = { ...updateInfo, airwayBill: pl.awbUrl }
-                  })
+        const powrup: Powrup = await manager.findOne(Powrup, {
+          where: { platform: Platform.POWRUP, active: true }
+        })
 
-                  updateInfo = {
-                    ...updateInfo,
-                    trackingNo: trackingNo,
-                    transporter: transporter
-                  }
+        const orderPackagesWithoutAirwayBill: OrderPackage[] = await manager
+          .createQueryBuilder(OrderPackage, 'opk')
+          .innerJoinAndSelect('opk.releaseGood', 'rg')
+          .innerJoinAndSelect('opk.domain', 'domain')
+          .innerJoinAndSelect('opk.bizplace', 'bizplace')
+          .innerJoinAndSelect('opk.creator', 'creator')
+          .where(`rg.source = :source`, { source: ApplicationType.POWRUP })
+          .andWhere(`rg.type = :rgType`, { rgType: 'b2c' })
+          .andWhere(`rg.ref_order_id NOTNULL`)
+          .andWhere(`(rg.lmd_option = false or rg.lmd_option ISNULL)`)
+          .andWhere(`rg.status NOT IN (:...status) `, {
+            status: [ORDER_STATUS.PACKING, ORDER_STATUS.CANCELLED, ORDER_STATUS.PENDING_CANCEL, ORDER_STATUS.DONE]
+          })
+          .andWhere(`opk.airway_bill ISNULL`)
+          .orderBy(`rg.created_at`)
+          .limit(100)
+          .getMany()
 
-                  await tx.getRepository(OrderPackage).update({ id: orderPackage.id }, { ...updateInfo })
+        if (orderPackagesWithoutAirwayBill?.length > 0) {
+          for (let i = 0; i < orderPackagesWithoutAirwayBill.length; i++) {
+            const orderPackage: OrderPackage = orderPackagesWithoutAirwayBill[i]
+            performance.mark(`start:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
+            try {
+              const {
+                trackingNo,
+                transporter,
+                printLabels
+              }: { trackingNo: string; transporter: string; printLabels: any[] } = await PowrupAPI.fetchDocument(
+                powrup,
+                {
+                  context: { state: { domain: null, user: null } },
+                  refOrderId: orderPackage.releaseGood.refOrderId,
+                  packageId: orderPackage.packageId
                 }
-              } catch (e) {
-                performance.mark(`end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
-                logger.error(`powrupGetAirwayBill[GetAirwayBill][Order Package ID: ${orderPackage.id}]: ${e}`)
-                continue
-              } finally {
-                let perf = performance.measure(
-                  `powrupGetAirwayBill: ${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`,
-                  `start:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`,
-                  `end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`
-                )
-                console.info(perf.toJSON())
+              )
+              performance.mark(`end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
+
+              if (printLabels?.length) {
+                let updateInfo: any
+                printLabels.map(pl => {
+                  updateInfo = { ...updateInfo, airwayBill: pl.awbUrl }
+                })
+
+                updateInfo = {
+                  ...updateInfo,
+                  trackingNo: trackingNo,
+                  transporter: transporter
+                }
+
+                // Auto-commit single-statement UPDATE on the QueryRunner's connection.
+                // Row lock held only for this UPDATE, not across the API call above.
+                await manager.update(OrderPackage, { id: orderPackage.id }, updateInfo)
               }
+            } catch (e) {
+              performance.mark(`end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
+              logger.error(`powrupGetAirwayBill[GetAirwayBill][Order Package ID: ${orderPackage.id}]: ${e}`)
+              continue
+            } finally {
+              let perf = performance.measure(
+                `powrupGetAirwayBill: ${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`,
+                `start:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`,
+                `end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`
+              )
+              console.info(perf.toJSON())
             }
           }
-        } catch (e) {
-          logger.error(`powrupGetAirwayBill[GetAirwayBill]: ${e}`)
         }
-      })
+      } catch (e) {
+        logger.error(`powrupGetAirwayBill[GetAirwayBill]: ${e}`)
+      } finally {
+        // Release the advisory lock so the next scheduled invocation can proceed.
+        // Wrapped in try/catch because release must not mask the original error.
+        try {
+          await queryRunner.query(`SELECT pg_advisory_unlock(hashtextextended($1, 0))`, [ADVISORY_LOCK_KEY])
+        } catch (releaseErr) {
+          logger.error(`powrupGetAirwayBill[AdvisoryUnlock]: ${releaseErr}`)
+        }
+      }
     } catch (e) {
       logger.error(`powrupGetAirwayBill[MainError]: ${e}`)
+    } finally {
+      // Defensive: clear any session-level advisory locks still held on this
+      // connection before returning it to the pool. Guards against the lock
+      // leaking if pg_advisory_unlock above failed (DB blip, statement timeout)
+      // and the pool then hands this connection to a future invocation.
+      // Skip when connect() itself failed — querying an unconnected QueryRunner
+      // would just throw with nothing to actually clean up.
+      if (connected) {
+        try {
+          await queryRunner.query(`SELECT pg_advisory_unlock_all()`)
+        } catch (unlockAllErr) {
+          logger.error(`powrupGetAirwayBill[AdvisoryUnlockAll]: ${unlockAllErr}`)
+        }
+      }
+      await queryRunner.release()
     }
   }
 }

package/server/graphql/resolvers/synchronization/powrup-get-tracking-no.ts

@@ -5,60 +5,100 @@ import { logger } from '@things-factory/env'
 import { Platform, Powrup, PowrupAPI } from '@things-factory/integration-powrup'
 import { ORDER_STATUS, OrderPackage } from '@things-factory/sales-base'
 
+const ADVISORY_LOCK_KEY = 'powrup-get-tracking-no'
+
 export const powrupGetTrackingNoResolver = {
   async powrupGetTrackingNo(_: void, { }, context: any): Promise<void> {
+    // Same fix as powrupGetAirwayBill: avoid wrapping the resolver in a long
+    // transaction across an external API call. Use a single QueryRunner with a
+    // session-level advisory lock for concurrency safety, and let each UPDATE
+    // auto-commit on its own.
+    const queryRunner = getConnection().createQueryRunner()
+    let connected = false
+
     try {
-      await getConnection().transaction(async tx => {
-        try {
-          const powrup: Powrup = await tx
-            .getRepository(Powrup)
-            .findOne({ where: { platform: Platform.POWRUP, active: true } })
+      await queryRunner.connect()
+      connected = true
+
+      const lockResult: { acquired: boolean }[] = await queryRunner.query(
+        `SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS acquired`,
+        [ADVISORY_LOCK_KEY]
+      )
+      if (!lockResult?.[0]?.acquired) {
+        logger.info(`powrupGetTrackingNo: another instance is running, skipping`)
+        return
+      }
 
-          let orderPackageQb = await tx
-            .getRepository(OrderPackage)
-            .createQueryBuilder('opk')
-            .innerJoinAndSelect('opk.releaseGood', 'rg')
-            .innerJoinAndSelect('opk.domain', 'domain')
-            .innerJoinAndSelect('opk.bizplace', 'bizplace')
-            .innerJoinAndSelect('opk.creator', 'creator')
-            .where(`rg.source = :source`, { source: ApplicationType.POWRUP })
-            .andWhere(`rg.type = :rgType`, { rgType: 'b2c' })
-            .andWhere(`rg.ref_order_id NOTNULL`)
-            .andWhere(`rg.status NOT IN(:...status) `, {
-              status: [ORDER_STATUS.CANCELLED, ORDER_STATUS.PENDING_CANCEL]
-            })
-            .andWhere(`opk.tracking_no IS NULL`)
-            .andWhere(`opk.airway_bill NOTNULL`)
-            .andWhere(`opk.package_id NOTNULL`)
-            .limit(500)
+      try {
+        const manager = queryRunner.manager
 
-          let orderPackagesWithoutTrackingNo: OrderPackage[] = await orderPackageQb.getMany()
+        const powrup: Powrup = await manager.findOne(Powrup, {
+          where: { platform: Platform.POWRUP, active: true }
+        })
 
-          if (orderPackagesWithoutTrackingNo?.length > 0) {
-            // Combine all package id into packageIds
-            let orderPackageIds: any = orderPackagesWithoutTrackingNo.map(op => op.packageId)
+        const orderPackagesWithoutTrackingNo: OrderPackage[] = await manager
+          .createQueryBuilder(OrderPackage, 'opk')
+          .innerJoinAndSelect('opk.releaseGood', 'rg')
+          .innerJoinAndSelect('opk.domain', 'domain')
+          .innerJoinAndSelect('opk.bizplace', 'bizplace')
+          .innerJoinAndSelect('opk.creator', 'creator')
+          .where(`rg.source = :source`, { source: ApplicationType.POWRUP })
+          .andWhere(`rg.type = :rgType`, { rgType: 'b2c' })
+          .andWhere(`rg.ref_order_id NOTNULL`)
+          .andWhere(`rg.status NOT IN(:...status) `, {
+            status: [ORDER_STATUS.CANCELLED, ORDER_STATUS.PENDING_CANCEL]
+          })
+          .andWhere(`opk.tracking_no IS NULL`)
+          .andWhere(`opk.airway_bill NOTNULL`)
+          .andWhere(`opk.package_id NOTNULL`)
+          .limit(500)
+          .getMany()
 
-            // Call API with multiple package ids in one call
-            const trackingNos: any = await PowrupAPI.getTrackingNo(powrup, {
-              context: { state: { domain: null, user: null } },
-              packageIds: orderPackageIds
-            })
+        if (orderPackagesWithoutTrackingNo?.length > 0) {
+          // Batch the API call for all package IDs in one shot.
+          const orderPackageIds: any = orderPackagesWithoutTrackingNo.map(op => op.packageId)
+          const trackingNos: any = await PowrupAPI.getTrackingNo(powrup, {
+            context: { state: { domain: null, user: null } },
+            packageIds: orderPackageIds
+          })
 
-            if (trackingNos?.length) {
-              await Promise.all(
-                trackingNos.map(async tn => {
-                  const { packageId, trackingNo } = tn
-                  await tx.getRepository(OrderPackage).update({ packageId: packageId }, { trackingNo: trackingNo })
-                })
-              )
+          if (trackingNos?.length) {
+            // Run UPDATEs sequentially as auto-commits. Parallel saves on the same
+            // QueryRunner connection are not safe (single connection), and parallel
+            // saves across separate connections gain little because the rows are
+            // independent and each UPDATE is fast.
+            for (const tn of trackingNos) {
+              const { packageId, trackingNo } = tn
+              await manager.update(OrderPackage, { packageId }, { trackingNo })
             }
           }
-        } catch (e) {
-          logger.error(`powrupGetTrackingNo[GetTrackingNo]: ${e}`)
         }
-      })
+      } catch (e) {
+        logger.error(`powrupGetTrackingNo[GetTrackingNo]: ${e}`)
+      } finally {
+        try {
+          await queryRunner.query(`SELECT pg_advisory_unlock(hashtextextended($1, 0))`, [ADVISORY_LOCK_KEY])
+        } catch (releaseErr) {
+          logger.error(`powrupGetTrackingNo[AdvisoryUnlock]: ${releaseErr}`)
+        }
+      }
     } catch (e) {
       logger.error(`powrupGetTrackingNo[MainError]: ${e}`)
+    } finally {
+      // Defensive: clear any session-level advisory locks still held on this
+      // connection before returning it to the pool. Guards against the lock
+      // leaking if pg_advisory_unlock above failed (DB blip, statement timeout)
+      // and the pool then hands this connection to a future invocation.
+      // Skip when connect() itself failed — querying an unconnected QueryRunner
+      // would just throw with nothing to actually clean up.
+      if (connected) {
+        try {
+          await queryRunner.query(`SELECT pg_advisory_unlock_all()`)
+        } catch (unlockAllErr) {
+          logger.error(`powrupGetTrackingNo[AdvisoryUnlockAll]: ${unlockAllErr}`)
+        }
+      }
+      await queryRunner.release()
     }
   }
 }

package/server/graphql/resolvers/synchronization/powrup-initiate-shipment.ts

@@ -1,100 +1,138 @@
-import { getConnection, getRepository } from 'typeorm'
+import { getConnection } from 'typeorm'
 
 import { ApplicationType } from '@things-factory/auth-base'
 import { logger } from '@things-factory/env'
 import { Platform, Powrup, PowrupAPI } from '@things-factory/integration-powrup'
 import { ORDER_STATUS, OrderPackage } from '@things-factory/sales-base'
 
+const ADVISORY_LOCK_KEY = 'powrup-initiate-shipment'
+
 export const powrupInitiateShipmentResolver = {
   async powrupInitiateShipment(_: void, {}, context: any): Promise<void> {
     const maxRTSRetry: number = 10
+
+    // Same fix as powrupGetAirwayBill / powrupGetTrackingNo: avoid wrapping the
+    // resolver in a long transaction across external PowrupAPI HTTP calls. Use a
+    // single QueryRunner with a session-level advisory lock for concurrency safety,
+    // and let each UPDATE auto-commit so row locks are held only for the duration
+    // of the UPDATE statement, not across the API call.
+    const queryRunner = getConnection().createQueryRunner()
+    let connected = false
+
     try {
-      await getConnection().transaction(async tx => {
-        try {
-          const powrup: Powrup = await tx
-            .getRepository(Powrup)
-            .findOne({ where: { platform: Platform.POWRUP, active: true } })
-
-          let orderPackageQb = tx
-            .getRepository(OrderPackage)
-            .createQueryBuilder('opk')
-            .innerJoinAndSelect('opk.releaseGood', 'rg')
-            .innerJoinAndSelect('opk.domain', 'domain')
-            .innerJoinAndSelect('opk.bizplace', 'bizplace')
-            .leftJoinAndSelect('opk.creator', 'creator')
-            .leftJoinAndSelect('rg.lastMileDelivery', 'lmd')
-            .where(`rg.source = :source`, { source: ApplicationType.POWRUP })
-            .andWhere(`rg.type = :rgType`, { rgType: 'b2c' })
-            .andWhere(`rg.ref_order_id NOTNULL`)
-            .andWhere(`rg.status = :status`, {
-              status: ORDER_STATUS.DONE
-            })
-            .andWhere(`opk.market_rts_call_success ISNULL`)
-            .andWhere(`COALESCE(opk.market_rts_call_retry, 0) <= :maxRTSRetry`, { maxRTSRetry: maxRTSRetry })
-            .orderBy(`opk.created_at`)
-
-          let orderPackagesWithoutRTS: OrderPackage[] = await orderPackageQb.getMany()
-
-          if (orderPackagesWithoutRTS?.length > 0) {
-            for (let i = 0; i < orderPackagesWithoutRTS.length; i++) {
-              const orderPackage: OrderPackage = orderPackagesWithoutRTS[i]
-              const releaseGood = orderPackage.releaseGood
-
-              if (releaseGood.lmdOption == true && orderPackage?.trackingNo == null) {
-                continue
-              }
+      await queryRunner.connect()
+      connected = true
 
-              let orderInformation: any = {
-                refOrderId: orderPackage.releaseGood.refOrderId,
-                packageId: orderPackage.packageId,
-                platform: orderPackage.releaseGood.platformCode
-              }
-              if (orderPackage?.releaseGood?.lmdOption) {
-                orderInformation = {
-                  ...orderInformation,
-                  awb: orderPackage.airwayBill,
-                  trackingNo: orderPackage.trackingNo,
-                  lmdOption: orderPackage?.releaseGood?.lmdOption,
-                  shipperName: orderPackage?.releaseGood?.lastMileDelivery?.platform
-                }
-              }
+      const lockResult: { acquired: boolean }[] = await queryRunner.query(
+        `SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS acquired`,
+        [ADVISORY_LOCK_KEY]
+      )
+      if (!lockResult?.[0]?.acquired) {
+        logger.info(`powrupInitiateShipment: another instance is running, skipping`)
+        return
+      }
+
+      try {
+        const manager = queryRunner.manager
+
+        const powrup: Powrup = await manager.findOne(Powrup, {
+          where: { platform: Platform.POWRUP, active: true }
+        })
+
+        const orderPackagesWithoutRTS: OrderPackage[] = await manager
+          .createQueryBuilder(OrderPackage, 'opk')
+          .innerJoinAndSelect('opk.releaseGood', 'rg')
+          .innerJoinAndSelect('opk.domain', 'domain')
+          .innerJoinAndSelect('opk.bizplace', 'bizplace')
+          .leftJoinAndSelect('opk.creator', 'creator')
+          .leftJoinAndSelect('rg.lastMileDelivery', 'lmd')
+          .where(`rg.source = :source`, { source: ApplicationType.POWRUP })
+          .andWhere(`rg.type = :rgType`, { rgType: 'b2c' })
+          .andWhere(`rg.ref_order_id NOTNULL`)
+          .andWhere(`rg.status = :status`, { status: ORDER_STATUS.DONE })
+          .andWhere(`opk.market_rts_call_success ISNULL`)
+          .andWhere(`COALESCE(opk.market_rts_call_retry, 0) <= :maxRTSRetry`, { maxRTSRetry })
+          .orderBy(`opk.created_at`)
+          .getMany()
+
+        if (orderPackagesWithoutRTS?.length > 0) {
+          for (let i = 0; i < orderPackagesWithoutRTS.length; i++) {
+            const orderPackage: OrderPackage = orderPackagesWithoutRTS[i]
+            const releaseGood = orderPackage.releaseGood
+
+            if (releaseGood.lmdOption == true && orderPackage?.trackingNo == null) {
+              continue
+            }
 
-              performance.mark(`start:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
-              try {
-                await PowrupAPI.initiateShipment(powrup, {
-                  ...orderInformation,
-                  context: { state: { domain: null, user: null } }
-                })
-                await tx
-                  .getRepository(OrderPackage)
-                  .update({ id: orderPackage.id }, { marketRtsCallSuccess: new Date() })
-                performance.mark(`end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
-              } catch (error) {
-                performance.mark(`end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
-                logger.error(`powrupInitiateShipment[InitiateShipment][Order Package ID: ${orderPackage.id}]: ${error}`)
-                await getRepository(OrderPackage)
-                  .createQueryBuilder()
-                  .update(OrderPackage)
-                  .set({ marketRtsCallRetry: () => `coalesce("market_rts_call_retry", 0) + 1` })
-                  .where('id = :id', { id: orderPackage.id })
-                  .execute()
-                continue
-              } finally {
-                let perf = performance.measure(
-                  `powrupInitiateShipment: ${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`,
-                  `start:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`,
-                  `end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`
-                )
-                console.info(perf.toJSON())
+            let orderInformation: any = {
+              refOrderId: orderPackage.releaseGood.refOrderId,
+              packageId: orderPackage.packageId,
+              platform: orderPackage.releaseGood.platformCode
+            }
+            if (orderPackage?.releaseGood?.lmdOption) {
+              orderInformation = {
+                ...orderInformation,
+                awb: orderPackage.airwayBill,
+                trackingNo: orderPackage.trackingNo,
+                lmdOption: orderPackage?.releaseGood?.lmdOption,
+                shipperName: orderPackage?.releaseGood?.lastMileDelivery?.platform
               }
             }
+
+            performance.mark(`start:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
+            try {
+              await PowrupAPI.initiateShipment(powrup, {
+                ...orderInformation,
+                context: { state: { domain: null, user: null } }
+              })
+
+              // Auto-commit single-statement UPDATE — row lock held only for this UPDATE,
+              // not across the API call above.
+              await manager.update(OrderPackage, { id: orderPackage.id }, { marketRtsCallSuccess: new Date() })
+
+              performance.mark(`end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
+            } catch (error) {
+              performance.mark(`end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
+              logger.error(`powrupInitiateShipment[InitiateShipment][Order Package ID: ${orderPackage.id}]: ${error}`)
+
+              await manager
+                .createQueryBuilder()
+                .update(OrderPackage)
+                .set({ marketRtsCallRetry: () => `coalesce("market_rts_call_retry", 0) + 1` })
+                .where('id = :id', { id: orderPackage.id })
+                .execute()
+
+              continue
+            } finally {
+              let perf = performance.measure(
+                `powrupInitiateShipment: ${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`,
+                `start:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`,
+                `end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`
+              )
+              console.info(perf.toJSON())
+            }
           }
-        } catch (e) {
-          logger.error(`powrupInitiateShipment[InitiateShipment]: ${e}`)
         }
-      })
+      } catch (e) {
+        logger.error(`powrupInitiateShipment[InitiateShipment]: ${e}`)
+      } finally {
+        try {
+          await queryRunner.query(`SELECT pg_advisory_unlock(hashtextextended($1, 0))`, [ADVISORY_LOCK_KEY])
+        } catch (releaseErr) {
+          logger.error(`powrupInitiateShipment[AdvisoryUnlock]: ${releaseErr}`)
+        }
+      }
     } catch (e) {
       logger.error(`powrupInitiateShipment[MainError]: ${e}`)
+    } finally {
+      if (connected) {
+        try {
+          await queryRunner.query(`SELECT pg_advisory_unlock_all()`)
+        } catch (unlockAllErr) {
+          logger.error(`powrupInitiateShipment[AdvisoryUnlockAll]: ${unlockAllErr}`)
+        }
+      }
+      await queryRunner.release()
     }
   }
 }