@things-factory/operato-hub 4.3.823 → 4.3.824

package/server/graphql/resolvers/synchronization/powrup-initiate-shipment.ts

@@ -1,100 +1,138 @@
-import { getConnection, getRepository } from 'typeorm'
+import { getConnection } from 'typeorm'
 
 import { ApplicationType } from '@things-factory/auth-base'
 import { logger } from '@things-factory/env'
 import { Platform, Powrup, PowrupAPI } from '@things-factory/integration-powrup'
 import { ORDER_STATUS, OrderPackage } from '@things-factory/sales-base'
 
+const ADVISORY_LOCK_KEY = 'powrup-initiate-shipment'
+
 export const powrupInitiateShipmentResolver = {
   async powrupInitiateShipment(_: void, {}, context: any): Promise<void> {
     const maxRTSRetry: number = 10
+
+    // Same fix as powrupGetAirwayBill / powrupGetTrackingNo: avoid wrapping the
+    // resolver in a long transaction across external PowrupAPI HTTP calls. Use a
+    // single QueryRunner with a session-level advisory lock for concurrency safety,
+    // and let each UPDATE auto-commit so row locks are held only for the duration
+    // of the UPDATE statement, not across the API call.
+    const queryRunner = getConnection().createQueryRunner()
+    let connected = false
+
     try {
-      await getConnection().transaction(async tx => {
-        try {
-          const powrup: Powrup = await tx
-            .getRepository(Powrup)
-            .findOne({ where: { platform: Platform.POWRUP, active: true } })
-
-          let orderPackageQb = tx
-            .getRepository(OrderPackage)
-            .createQueryBuilder('opk')
-            .innerJoinAndSelect('opk.releaseGood', 'rg')
-            .innerJoinAndSelect('opk.domain', 'domain')
-            .innerJoinAndSelect('opk.bizplace', 'bizplace')
-            .leftJoinAndSelect('opk.creator', 'creator')
-            .leftJoinAndSelect('rg.lastMileDelivery', 'lmd')
-            .where(`rg.source = :source`, { source: ApplicationType.POWRUP })
-            .andWhere(`rg.type = :rgType`, { rgType: 'b2c' })
-            .andWhere(`rg.ref_order_id NOTNULL`)
-            .andWhere(`rg.status = :status`, {
-              status: ORDER_STATUS.DONE
-            })
-            .andWhere(`opk.market_rts_call_success ISNULL`)
-            .andWhere(`COALESCE(opk.market_rts_call_retry, 0) <= :maxRTSRetry`, { maxRTSRetry: maxRTSRetry })
-            .orderBy(`opk.created_at`)
-
-          let orderPackagesWithoutRTS: OrderPackage[] = await orderPackageQb.getMany()
-
-          if (orderPackagesWithoutRTS?.length > 0) {
-            for (let i = 0; i < orderPackagesWithoutRTS.length; i++) {
-              const orderPackage: OrderPackage = orderPackagesWithoutRTS[i]
-              const releaseGood = orderPackage.releaseGood
-
-              if (releaseGood.lmdOption == true && orderPackage?.trackingNo == null) {
-                continue
-              }
+      await queryRunner.connect()
+      connected = true
 
-              let orderInformation: any = {
-                refOrderId: orderPackage.releaseGood.refOrderId,
-                packageId: orderPackage.packageId,
-                platform: orderPackage.releaseGood.platformCode
-              }
-              if (orderPackage?.releaseGood?.lmdOption) {
-                orderInformation = {
-                  ...orderInformation,
-                  awb: orderPackage.airwayBill,
-                  trackingNo: orderPackage.trackingNo,
-                  lmdOption: orderPackage?.releaseGood?.lmdOption,
-                  shipperName: orderPackage?.releaseGood?.lastMileDelivery?.platform
-                }
-              }
+      const lockResult: { acquired: boolean }[] = await queryRunner.query(
+        `SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS acquired`,
+        [ADVISORY_LOCK_KEY]
+      )
+      if (!lockResult?.[0]?.acquired) {
+        logger.info(`powrupInitiateShipment: another instance is running, skipping`)
+        return
+      }
+
+      try {
+        const manager = queryRunner.manager
+
+        const powrup: Powrup = await manager.findOne(Powrup, {
+          where: { platform: Platform.POWRUP, active: true }
+        })
+
+        const orderPackagesWithoutRTS: OrderPackage[] = await manager
+          .createQueryBuilder(OrderPackage, 'opk')
+          .innerJoinAndSelect('opk.releaseGood', 'rg')
+          .innerJoinAndSelect('opk.domain', 'domain')
+          .innerJoinAndSelect('opk.bizplace', 'bizplace')
+          .leftJoinAndSelect('opk.creator', 'creator')
+          .leftJoinAndSelect('rg.lastMileDelivery', 'lmd')
+          .where(`rg.source = :source`, { source: ApplicationType.POWRUP })
+          .andWhere(`rg.type = :rgType`, { rgType: 'b2c' })
+          .andWhere(`rg.ref_order_id NOTNULL`)
+          .andWhere(`rg.status = :status`, { status: ORDER_STATUS.DONE })
+          .andWhere(`opk.market_rts_call_success ISNULL`)
+          .andWhere(`COALESCE(opk.market_rts_call_retry, 0) <= :maxRTSRetry`, { maxRTSRetry })
+          .orderBy(`opk.created_at`)
+          .getMany()
+
+        if (orderPackagesWithoutRTS?.length > 0) {
+          for (let i = 0; i < orderPackagesWithoutRTS.length; i++) {
+            const orderPackage: OrderPackage = orderPackagesWithoutRTS[i]
+            const releaseGood = orderPackage.releaseGood
+
+            if (releaseGood.lmdOption == true && orderPackage?.trackingNo == null) {
+              continue
+            }
 
-              performance.mark(`start:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
-              try {
-                await PowrupAPI.initiateShipment(powrup, {
-                  ...orderInformation,
-                  context: { state: { domain: null, user: null } }
-                })
-                await tx
-                  .getRepository(OrderPackage)
-                  .update({ id: orderPackage.id }, { marketRtsCallSuccess: new Date() })
-                performance.mark(`end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
-              } catch (error) {
-                performance.mark(`end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
-                logger.error(`powrupInitiateShipment[InitiateShipment][Order Package ID: ${orderPackage.id}]: ${error}`)
-                await getRepository(OrderPackage)
-                  .createQueryBuilder()
-                  .update(OrderPackage)
-                  .set({ marketRtsCallRetry: () => `coalesce("market_rts_call_retry", 0) + 1` })
-                  .where('id = :id', { id: orderPackage.id })
-                  .execute()
-                continue
-              } finally {
-                let perf = performance.measure(
-                  `powrupInitiateShipment: ${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`,
-                  `start:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`,
-                  `end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`
-                )
-                console.info(perf.toJSON())
+            let orderInformation: any = {
+              refOrderId: orderPackage.releaseGood.refOrderId,
+              packageId: orderPackage.packageId,
+              platform: orderPackage.releaseGood.platformCode
+            }
+            if (orderPackage?.releaseGood?.lmdOption) {
+              orderInformation = {
+                ...orderInformation,
+                awb: orderPackage.airwayBill,
+                trackingNo: orderPackage.trackingNo,
+                lmdOption: orderPackage?.releaseGood?.lmdOption,
+                shipperName: orderPackage?.releaseGood?.lastMileDelivery?.platform
               }
             }
+
+            performance.mark(`start:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
+            try {
+              await PowrupAPI.initiateShipment(powrup, {
+                ...orderInformation,
+                context: { state: { domain: null, user: null } }
+              })
+
+              // Auto-commit single-statement UPDATE — row lock held only for this UPDATE,
+              // not across the API call above.
+              await manager.update(OrderPackage, { id: orderPackage.id }, { marketRtsCallSuccess: new Date() })
+
+              performance.mark(`end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
+            } catch (error) {
+              performance.mark(`end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`)
+              logger.error(`powrupInitiateShipment[InitiateShipment][Order Package ID: ${orderPackage.id}]: ${error}`)
+
+              await manager
+                .createQueryBuilder()
+                .update(OrderPackage)
+                .set({ marketRtsCallRetry: () => `coalesce("market_rts_call_retry", 0) + 1` })
+                .where('id = :id', { id: orderPackage.id })
+                .execute()
+
+              continue
+            } finally {
+              let perf = performance.measure(
+                `powrupInitiateShipment: ${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`,
+                `start:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`,
+                `end:${orderPackage.releaseGood.refOrderId} - ${orderPackage.packageId}`
+              )
+              console.info(perf.toJSON())
+            }
           }
-        } catch (e) {
-          logger.error(`powrupInitiateShipment[InitiateShipment]: ${e}`)
         }
-      })
+      } catch (e) {
+        logger.error(`powrupInitiateShipment[InitiateShipment]: ${e}`)
+      } finally {
+        try {
+          await queryRunner.query(`SELECT pg_advisory_unlock(hashtextextended($1, 0))`, [ADVISORY_LOCK_KEY])
+        } catch (releaseErr) {
+          logger.error(`powrupInitiateShipment[AdvisoryUnlock]: ${releaseErr}`)
+        }
+      }
     } catch (e) {
       logger.error(`powrupInitiateShipment[MainError]: ${e}`)
+    } finally {
+      if (connected) {
+        try {
+          await queryRunner.query(`SELECT pg_advisory_unlock_all()`)
+        } catch (unlockAllErr) {
+          logger.error(`powrupInitiateShipment[AdvisoryUnlockAll]: ${unlockAllErr}`)
+        }
+      }
+      await queryRunner.release()
     }
   }
 }

package/server/graphql/resolvers/synchronization/sellercraft-initiate-shipment.ts

@@ -1,116 +1,159 @@
-import { getConnection, getRepository } from 'typeorm'
+import { getConnection } from 'typeorm'
 
 import { ApplicationType } from '@things-factory/auth-base'
 import { logger } from '@things-factory/env'
 import { Sellercraft, SellercraftAPI } from '@things-factory/integration-sellercraft'
 import { ORDER_STATUS, OrderPackage } from '@things-factory/sales-base'
 
+const ADVISORY_LOCK_KEY = 'sellercraft-initiate-shipment'
+
 export const sellercraftInitiateShipmentResolver = {
   async sellercraftInitiateShipment(_: void, {}, context: any): Promise<void> {
     const maxRTSRetry: number = 10
+
+    // Same fix as powrupGetAirwayBill / powrupGetTrackingNo: do not wrap the
+    // resolver in a single long transaction across external SellercraftAPI HTTP
+    // calls. The previous version held row locks on every UPDATE'd order_packages
+    // row until the loop finished, which was the direct cause of the order_packages
+    // lock chains observed in pg_stat_activity (sessions stuck `idle in transaction`
+    // for 10+ minutes after `UPDATE order_packages SET market_rts_call_success`).
+    //
+    // Use a single QueryRunner with a session-level advisory lock for concurrency
+    // safety, and let each UPDATE auto-commit on its own (row lock held only for
+    // the duration of that single statement).
+    const queryRunner = getConnection().createQueryRunner()
+    let connected = false
+
     try {
-      await getConnection().transaction(async tx => {
-        try {
-          let orderPackageQb = tx
-            .getRepository(OrderPackage)
-            .createQueryBuilder('opk')
-            .innerJoinAndSelect('opk.releaseGood', 'rg')
-            .innerJoinAndSelect('opk.domain', 'domain')
-            .innerJoinAndSelect('opk.bizplace', 'bizplace')
-            .innerJoinAndSelect('bizplace.domain', 'bdomain')
-            .leftJoinAndSelect('opk.creator', 'creator')
-            .leftJoinAndSelect('rg.lastMileDelivery', 'lmd')
-            .where(`rg.source = :source`, { source: ApplicationType.SELLERCRAFT })
-            .andWhere(`rg.type = :rgType`, { rgType: 'b2c' })
-            .andWhere(`rg.status = :status`, {
-              status: ORDER_STATUS.DONE
+      await queryRunner.connect()
+      connected = true
+
+      const lockResult: { acquired: boolean }[] = await queryRunner.query(
+        `SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS acquired`,
+        [ADVISORY_LOCK_KEY]
+      )
+      if (!lockResult?.[0]?.acquired) {
+        logger.info(`sellercraftInitiateShipment: another instance is running, skipping`)
+        return
+      }
+
+      try {
+        const manager = queryRunner.manager
+
+        const orderPackagesWithoutRTS: OrderPackage[] = await manager
+          .createQueryBuilder(OrderPackage, 'opk')
+          .innerJoinAndSelect('opk.releaseGood', 'rg')
+          .innerJoinAndSelect('opk.domain', 'domain')
+          .innerJoinAndSelect('opk.bizplace', 'bizplace')
+          .innerJoinAndSelect('bizplace.domain', 'bdomain')
+          .leftJoinAndSelect('opk.creator', 'creator')
+          .leftJoinAndSelect('rg.lastMileDelivery', 'lmd')
+          .where(`rg.source = :source`, { source: ApplicationType.SELLERCRAFT })
+          .andWhere(`rg.type = :rgType`, { rgType: 'b2c' })
+          .andWhere(`rg.status = :status`, { status: ORDER_STATUS.DONE })
+          .andWhere(`opk.market_rts_call_success ISNULL`)
+          .andWhere(`COALESCE(opk.market_rts_call_retry, 0) <= :maxRTSRetry`, { maxRTSRetry })
+          .orderBy(`opk.created_at`)
+          .getMany()
+
+        if (orderPackagesWithoutRTS?.length > 0) {
+          for (let i = 0; i < orderPackagesWithoutRTS.length; i++) {
+            const orderPackage: OrderPackage = orderPackagesWithoutRTS[i]
+            const releaseGood = orderPackage.releaseGood
+
+            const sellercraft: Sellercraft = await manager.findOne(Sellercraft, {
+              where: { domain: { id: orderPackage.bizplace.domain.id }, status: 'ACTIVE' }
             })
-            .andWhere(`opk.market_rts_call_success ISNULL`)
-            .andWhere(`COALESCE(opk.market_rts_call_retry, 0) <= :maxRTSRetry`, { maxRTSRetry: maxRTSRetry })
-            .orderBy(`opk.created_at`)
 
-          let orderPackagesWithoutRTS: OrderPackage[] = await orderPackageQb.getMany()
+            if (!sellercraft) {
+              continue
+            }
 
-          if (orderPackagesWithoutRTS?.length > 0) {
-            for (let i = 0; i < orderPackagesWithoutRTS.length; i++) {
-              const orderPackage: OrderPackage = orderPackagesWithoutRTS[i]
-              const releaseGood = orderPackage.releaseGood
+            let sellercraftSetting: any = sellercraft.sellercraftSetting
+            let disableInitiateOrderShipment: boolean =
+              sellercraftSetting?.disableInitiateOrderShipment &&
+              sellercraftSetting?.disableInitiateOrderShipment == 1
+                ? true
+                : false
 
-              const sellercraft: Sellercraft = await tx
-                .getRepository(Sellercraft)
-                .findOne({ where: { domain: { id: orderPackage.bizplace.domain.id }, status: 'ACTIVE' } })
+            if (
+              disableInitiateOrderShipment === true ||
+              (releaseGood.lmdOption == true && orderPackage?.trackingNo == null)
+            ) {
+              continue
+            }
 
-              if (!sellercraft) {
-                continue
+            let orderInformation: any = {
+              accountId: sellercraft.accountId,
+              orderId: releaseGood.refNo2,
+              packageId: orderPackage.packageId
+            }
+            if (orderPackage?.releaseGood?.lmdOption) {
+              orderInformation = {
+                ...orderInformation,
+                awb: orderPackage.airwayBill,
+                trackingNo: orderPackage.trackingNo,
+                lmdOption: orderPackage?.releaseGood?.lmdOption,
+                shipperName: orderPackage?.releaseGood?.lastMileDelivery?.platform
               }
+            }
 
-              let sellercraftSetting: any = sellercraft.sellercraftSetting
-              let disableInitiateOrderShipment: boolean =
-                sellercraftSetting?.disableInitiateOrderShipment &&
-                sellercraftSetting?.disableInitiateOrderShipment == 1
-                  ? true
-                  : false
-
-              if (
-                disableInitiateOrderShipment === true ||
-                (releaseGood.lmdOption == true && orderPackage?.trackingNo == null)
-              ) {
-                continue
-              }
+            performance.mark(`start:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`)
+            try {
+              await SellercraftAPI.initiateOrderShipment(sellercraft, {
+                ...orderInformation,
+                context: { state: { domain: null, user: null } }
+              })
 
-              let orderInformation: any = {
-                accountId: sellercraft.accountId,
-                orderId: releaseGood.refNo2,
-                packageId: orderPackage.packageId
-              }
-              if (orderPackage?.releaseGood?.lmdOption) {
-                orderInformation = {
-                  ...orderInformation,
-                  awb: orderPackage.airwayBill,
-                  trackingNo: orderPackage.trackingNo,
-                  lmdOption: orderPackage?.releaseGood?.lmdOption,
-                  shipperName: orderPackage?.releaseGood?.lastMileDelivery?.platform
-                }
-              }
+              // Auto-commit single-statement UPDATE — row lock held only for this UPDATE,
+              // not across the API call above.
+              await manager.update(OrderPackage, { id: orderPackage.id }, { marketRtsCallSuccess: new Date() })
 
-              performance.mark(`start:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`)
-              try {
-                await SellercraftAPI.initiateOrderShipment(sellercraft, {
-                  ...orderInformation,
-                  context: { state: { domain: null, user: null } }
-                })
-                await tx
-                  .getRepository(OrderPackage)
-                  .update({ id: orderPackage.id }, { marketRtsCallSuccess: new Date() })
-                performance.mark(`end:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`)
-              } catch (error) {
-                performance.mark(`end:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`)
-                logger.error(
-                  `sellercraftInitiateShipment[InitiateShipment][Order Package ID: ${orderPackage.id}]: ${error}`
-                )
-                await getRepository(OrderPackage)
-                  .createQueryBuilder()
-                  .update(OrderPackage)
-                  .set({ marketRtsCallRetry: () => `coalesce("market_rts_call_retry", 0) + 1` })
-                  .where('id = :id', { id: orderPackage.id })
-                  .execute()
-                continue
-              } finally {
-                let perf = performance.measure(
-                  `sellercraftInitiateShipment: ${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`,
-                  `start:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`,
-                  `end:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`
-                )
-                console.info(perf.toJSON())
-              }
+              performance.mark(`end:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`)
+            } catch (error) {
+              performance.mark(`end:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`)
+              logger.error(
+                `sellercraftInitiateShipment[InitiateShipment][Order Package ID: ${orderPackage.id}]: ${error}`
+              )
+
+              await manager
+                .createQueryBuilder()
+                .update(OrderPackage)
+                .set({ marketRtsCallRetry: () => `coalesce("market_rts_call_retry", 0) + 1` })
+                .where('id = :id', { id: orderPackage.id })
+                .execute()
+
+              continue
+            } finally {
+              let perf = performance.measure(
+                `sellercraftInitiateShipment: ${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`,
+                `start:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`,
+                `end:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`
+              )
+              console.info(perf.toJSON())
             }
           }
-        } catch (e) {
-          logger.error(`sellercraftInitiateShipment[InitiateShipment]: ${e}`)
         }
-      })
+      } catch (e) {
+        logger.error(`sellercraftInitiateShipment[InitiateShipment]: ${e}`)
+      } finally {
+        try {
+          await queryRunner.query(`SELECT pg_advisory_unlock(hashtextextended($1, 0))`, [ADVISORY_LOCK_KEY])
+        } catch (releaseErr) {
+          logger.error(`sellercraftInitiateShipment[AdvisoryUnlock]: ${releaseErr}`)
+        }
+      }
     } catch (e) {
       logger.error(`sellercraftInitiateShipment[MainError]: ${e}`)
+    } finally {
+      if (connected) {
+        try {
+          await queryRunner.query(`SELECT pg_advisory_unlock_all()`)
+        } catch (unlockAllErr) {
+          logger.error(`sellercraftInitiateShipment[AdvisoryUnlockAll]: ${unlockAllErr}`)
+        }
+      }
+      await queryRunner.release()
     }
   }
 }

package/server/graphql/types/synchronization/index.ts

@@ -19,14 +19,14 @@ export const Mutation = /* GraphQL */ `
   syncAllProductsToSellercraft: Boolean @transaction
   syncAllProductsToPowrup: Boolean @transaction
   sellercraftPackMarketplaceOrder: Boolean @transaction
-  sellercraftInitiateShipment: Boolean @transaction
   powrupPackOrder: Boolean @transaction
   # Intentionally NOT @transaction — these resolvers manage their own QueryRunner
-  # to avoid holding row locks on order_packages across external Powrup API calls.
-  # See powrup-get-airway-bill.ts and powrup-get-tracking-no.ts for the rationale.
+  # to avoid holding row locks on order_packages across external API calls.
+  # See the resolver files for the rationale.
   powrupGetTrackingNo: Boolean
   powrupGetAirwayBill: Boolean
-  powrupInitiateShipment: Boolean @transaction
+  powrupInitiateShipment: Boolean
+  sellercraftInitiateShipment: Boolean
   getLmdAwb: Boolean @transaction
   createDatShipments: Boolean @transaction
   autoUpdateAllMarketplaceProductVariationQuantity(companyDomainId: String, warehouseId: String): Boolean @transaction

package/server/routers/api/restful-apis/v1/warehouse/add-release-order.ts

@@ -929,7 +929,18 @@ async function massageOrderItems(
   inputOrderProducts = await inputOrderProducts.reduce(async (accPromise, itm, idx) => {
     const acc = await accPromise
     let { sku, refCode } = itm.product
-    let { packingType, packingSize, batchId, releaseQty, sellingPrice, paidAmount, uom, refItemId, warehouseCode } = itm
+    let {
+      packingType,
+      packingSize,
+      batchId,
+      releaseQty,
+      sellingPrice,
+      paidAmount,
+      uom,
+      refItemId,
+      warehouseCode,
+      remark
+    } = itm
 
     if (!sku && !refCode) {
       throw new ApiError('E01', 'sku or refCode not found')
@@ -965,7 +976,8 @@ async function massageOrderItems(
         uom: uom,
         batchId: batchId?.trim() || '',
         refItemId: refItemId ?? '',
-        warehouseCode: normalizedWarehouseCode
+        warehouseCode: normalizedWarehouseCode,
+        remark: remark || null
       })
     }