@things-factory/operato-hub 4.3.822 → 4.3.824

package/server/graphql/resolvers/synchronization/sellercraft-initiate-shipment.ts

@@ -1,116 +1,159 @@
-import { getConnection, getRepository } from 'typeorm'
+import { getConnection } from 'typeorm'
 
 import { ApplicationType } from '@things-factory/auth-base'
 import { logger } from '@things-factory/env'
 import { Sellercraft, SellercraftAPI } from '@things-factory/integration-sellercraft'
 import { ORDER_STATUS, OrderPackage } from '@things-factory/sales-base'
 
+const ADVISORY_LOCK_KEY = 'sellercraft-initiate-shipment'
+
 export const sellercraftInitiateShipmentResolver = {
   async sellercraftInitiateShipment(_: void, {}, context: any): Promise<void> {
     const maxRTSRetry: number = 10
+
+    // Same fix as powrupGetAirwayBill / powrupGetTrackingNo: do not wrap the
+    // resolver in a single long transaction across external SellercraftAPI HTTP
+    // calls. The previous version held row locks on every UPDATE'd order_packages
+    // row until the loop finished, which was the direct cause of the order_packages
+    // lock chains observed in pg_stat_activity (sessions stuck `idle in transaction`
+    // for 10+ minutes after `UPDATE order_packages SET market_rts_call_success`).
+    //
+    // Use a single QueryRunner with a session-level advisory lock for concurrency
+    // safety, and let each UPDATE auto-commit on its own (row lock held only for
+    // the duration of that single statement).
+    const queryRunner = getConnection().createQueryRunner()
+    let connected = false
+
     try {
-      await getConnection().transaction(async tx => {
-        try {
-          let orderPackageQb = tx
-            .getRepository(OrderPackage)
-            .createQueryBuilder('opk')
-            .innerJoinAndSelect('opk.releaseGood', 'rg')
-            .innerJoinAndSelect('opk.domain', 'domain')
-            .innerJoinAndSelect('opk.bizplace', 'bizplace')
-            .innerJoinAndSelect('bizplace.domain', 'bdomain')
-            .leftJoinAndSelect('opk.creator', 'creator')
-            .leftJoinAndSelect('rg.lastMileDelivery', 'lmd')
-            .where(`rg.source = :source`, { source: ApplicationType.SELLERCRAFT })
-            .andWhere(`rg.type = :rgType`, { rgType: 'b2c' })
-            .andWhere(`rg.status = :status`, {
-              status: ORDER_STATUS.DONE
+      await queryRunner.connect()
+      connected = true
+
+      const lockResult: { acquired: boolean }[] = await queryRunner.query(
+        `SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS acquired`,
+        [ADVISORY_LOCK_KEY]
+      )
+      if (!lockResult?.[0]?.acquired) {
+        logger.info(`sellercraftInitiateShipment: another instance is running, skipping`)
+        return
+      }
+
+      try {
+        const manager = queryRunner.manager
+
+        const orderPackagesWithoutRTS: OrderPackage[] = await manager
+          .createQueryBuilder(OrderPackage, 'opk')
+          .innerJoinAndSelect('opk.releaseGood', 'rg')
+          .innerJoinAndSelect('opk.domain', 'domain')
+          .innerJoinAndSelect('opk.bizplace', 'bizplace')
+          .innerJoinAndSelect('bizplace.domain', 'bdomain')
+          .leftJoinAndSelect('opk.creator', 'creator')
+          .leftJoinAndSelect('rg.lastMileDelivery', 'lmd')
+          .where(`rg.source = :source`, { source: ApplicationType.SELLERCRAFT })
+          .andWhere(`rg.type = :rgType`, { rgType: 'b2c' })
+          .andWhere(`rg.status = :status`, { status: ORDER_STATUS.DONE })
+          .andWhere(`opk.market_rts_call_success ISNULL`)
+          .andWhere(`COALESCE(opk.market_rts_call_retry, 0) <= :maxRTSRetry`, { maxRTSRetry })
+          .orderBy(`opk.created_at`)
+          .getMany()
+
+        if (orderPackagesWithoutRTS?.length > 0) {
+          for (let i = 0; i < orderPackagesWithoutRTS.length; i++) {
+            const orderPackage: OrderPackage = orderPackagesWithoutRTS[i]
+            const releaseGood = orderPackage.releaseGood
+
+            const sellercraft: Sellercraft = await manager.findOne(Sellercraft, {
+              where: { domain: { id: orderPackage.bizplace.domain.id }, status: 'ACTIVE' }
             })
-            .andWhere(`opk.market_rts_call_success ISNULL`)
-            .andWhere(`COALESCE(opk.market_rts_call_retry, 0) <= :maxRTSRetry`, { maxRTSRetry: maxRTSRetry })
-            .orderBy(`opk.created_at`)
 
-          let orderPackagesWithoutRTS: OrderPackage[] = await orderPackageQb.getMany()
+            if (!sellercraft) {
+              continue
+            }
 
-          if (orderPackagesWithoutRTS?.length > 0) {
-            for (let i = 0; i < orderPackagesWithoutRTS.length; i++) {
-              const orderPackage: OrderPackage = orderPackagesWithoutRTS[i]
-              const releaseGood = orderPackage.releaseGood
+            let sellercraftSetting: any = sellercraft.sellercraftSetting
+            let disableInitiateOrderShipment: boolean =
+              sellercraftSetting?.disableInitiateOrderShipment &&
+              sellercraftSetting?.disableInitiateOrderShipment == 1
+                ? true
+                : false
 
-              const sellercraft: Sellercraft = await tx
-                .getRepository(Sellercraft)
-                .findOne({ where: { domain: { id: orderPackage.bizplace.domain.id }, status: 'ACTIVE' } })
+            if (
+              disableInitiateOrderShipment === true ||
+              (releaseGood.lmdOption == true && orderPackage?.trackingNo == null)
+            ) {
+              continue
+            }
 
-              if (!sellercraft) {
-                continue
+            let orderInformation: any = {
+              accountId: sellercraft.accountId,
+              orderId: releaseGood.refNo2,
+              packageId: orderPackage.packageId
+            }
+            if (orderPackage?.releaseGood?.lmdOption) {
+              orderInformation = {
+                ...orderInformation,
+                awb: orderPackage.airwayBill,
+                trackingNo: orderPackage.trackingNo,
+                lmdOption: orderPackage?.releaseGood?.lmdOption,
+                shipperName: orderPackage?.releaseGood?.lastMileDelivery?.platform
               }
+            }
 
-              let sellercraftSetting: any = sellercraft.sellercraftSetting
-              let disableInitiateOrderShipment: boolean =
-                sellercraftSetting?.disableInitiateOrderShipment &&
-                sellercraftSetting?.disableInitiateOrderShipment == 1
-                  ? true
-                  : false
-
-              if (
-                disableInitiateOrderShipment === true ||
-                (releaseGood.lmdOption == true && orderPackage?.trackingNo == null)
-              ) {
-                continue
-              }
+            performance.mark(`start:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`)
+            try {
+              await SellercraftAPI.initiateOrderShipment(sellercraft, {
+                ...orderInformation,
+                context: { state: { domain: null, user: null } }
+              })
 
-              let orderInformation: any = {
-                accountId: sellercraft.accountId,
-                orderId: releaseGood.refNo2,
-                packageId: orderPackage.packageId
-              }
-              if (orderPackage?.releaseGood?.lmdOption) {
-                orderInformation = {
-                  ...orderInformation,
-                  awb: orderPackage.airwayBill,
-                  trackingNo: orderPackage.trackingNo,
-                  lmdOption: orderPackage?.releaseGood?.lmdOption,
-                  shipperName: orderPackage?.releaseGood?.lastMileDelivery?.platform
-                }
-              }
+              // Auto-commit single-statement UPDATE — row lock held only for this UPDATE,
+              // not across the API call above.
+              await manager.update(OrderPackage, { id: orderPackage.id }, { marketRtsCallSuccess: new Date() })
 
-              performance.mark(`start:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`)
-              try {
-                await SellercraftAPI.initiateOrderShipment(sellercraft, {
-                  ...orderInformation,
-                  context: { state: { domain: null, user: null } }
-                })
-                await tx
-                  .getRepository(OrderPackage)
-                  .update({ id: orderPackage.id }, { marketRtsCallSuccess: new Date() })
-                performance.mark(`end:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`)
-              } catch (error) {
-                performance.mark(`end:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`)
-                logger.error(
-                  `sellercraftInitiateShipment[InitiateShipment][Order Package ID: ${orderPackage.id}]: ${error}`
-                )
-                await getRepository(OrderPackage)
-                  .createQueryBuilder()
-                  .update(OrderPackage)
-                  .set({ marketRtsCallRetry: () => `coalesce("market_rts_call_retry", 0) + 1` })
-                  .where('id = :id', { id: orderPackage.id })
-                  .execute()
-                continue
-              } finally {
-                let perf = performance.measure(
-                  `sellercraftInitiateShipment: ${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`,
-                  `start:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`,
-                  `end:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`
-                )
-                console.info(perf.toJSON())
-              }
+              performance.mark(`end:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`)
+            } catch (error) {
+              performance.mark(`end:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`)
+              logger.error(
+                `sellercraftInitiateShipment[InitiateShipment][Order Package ID: ${orderPackage.id}]: ${error}`
+              )
+
+              await manager
+                .createQueryBuilder()
+                .update(OrderPackage)
+                .set({ marketRtsCallRetry: () => `coalesce("market_rts_call_retry", 0) + 1` })
+                .where('id = :id', { id: orderPackage.id })
+                .execute()
+
+              continue
+            } finally {
+              let perf = performance.measure(
+                `sellercraftInitiateShipment: ${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`,
+                `start:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`,
+                `end:${orderPackage.releaseGood.refNo2} - ${orderPackage.packageId}`
+              )
+              console.info(perf.toJSON())
             }
           }
-        } catch (e) {
-          logger.error(`sellercraftInitiateShipment[InitiateShipment]: ${e}`)
         }
-      })
+      } catch (e) {
+        logger.error(`sellercraftInitiateShipment[InitiateShipment]: ${e}`)
+      } finally {
+        try {
+          await queryRunner.query(`SELECT pg_advisory_unlock(hashtextextended($1, 0))`, [ADVISORY_LOCK_KEY])
+        } catch (releaseErr) {
+          logger.error(`sellercraftInitiateShipment[AdvisoryUnlock]: ${releaseErr}`)
+        }
+      }
     } catch (e) {
       logger.error(`sellercraftInitiateShipment[MainError]: ${e}`)
+    } finally {
+      if (connected) {
+        try {
+          await queryRunner.query(`SELECT pg_advisory_unlock_all()`)
+        } catch (unlockAllErr) {
+          logger.error(`sellercraftInitiateShipment[AdvisoryUnlockAll]: ${unlockAllErr}`)
+        }
+      }
+      await queryRunner.release()
     }
   }
 }

package/server/graphql/types/synchronization/index.ts

@@ -19,11 +19,14 @@ export const Mutation = /* GraphQL */ `
   syncAllProductsToSellercraft: Boolean @transaction
   syncAllProductsToPowrup: Boolean @transaction
   sellercraftPackMarketplaceOrder: Boolean @transaction
-  sellercraftInitiateShipment: Boolean @transaction
   powrupPackOrder: Boolean @transaction
-  powrupGetTrackingNo: Boolean @transaction
-  powrupGetAirwayBill: Boolean @transaction
-  powrupInitiateShipment: Boolean @transaction
+  # Intentionally NOT @transaction — these resolvers manage their own QueryRunner
+  # to avoid holding row locks on order_packages across external API calls.
+  # See the resolver files for the rationale.
+  powrupGetTrackingNo: Boolean
+  powrupGetAirwayBill: Boolean
+  powrupInitiateShipment: Boolean
+  sellercraftInitiateShipment: Boolean
   getLmdAwb: Boolean @transaction
   createDatShipments: Boolean @transaction
   autoUpdateAllMarketplaceProductVariationQuantity(companyDomainId: String, warehouseId: String): Boolean @transaction

package/server/routers/api/restful-apis/v1/utils/params.ts

@@ -513,6 +513,7 @@ export const params = {
     'orderInventories.batchId',
     'orderInventories.warehouseCode',
     'orderInventories.refItemId',
+    'orderInventories.remark',
     'remark',
     'billTo',
     'billTo.billingAddress',

package/server/routers/api/restful-apis/v1/warehouse/add-release-order.ts

@@ -929,7 +929,18 @@ async function massageOrderItems(
   inputOrderProducts = await inputOrderProducts.reduce(async (accPromise, itm, idx) => {
     const acc = await accPromise
     let { sku, refCode } = itm.product
-    let { packingType, packingSize, batchId, releaseQty, sellingPrice, paidAmount, uom, refItemId, warehouseCode } = itm
+    let {
+      packingType,
+      packingSize,
+      batchId,
+      releaseQty,
+      sellingPrice,
+      paidAmount,
+      uom,
+      refItemId,
+      warehouseCode,
+      remark
+    } = itm
 
     if (!sku && !refCode) {
       throw new ApiError('E01', 'sku or refCode not found')
@@ -965,7 +976,8 @@ async function massageOrderItems(
         uom: uom,
         batchId: batchId?.trim() || '',
         refItemId: refItemId ?? '',
-        warehouseCode: normalizedWarehouseCode
+        warehouseCode: normalizedWarehouseCode,
+        remark: remark || null
       })
     }
 

package/server/routers/xilnex-router.ts

@@ -12,6 +12,7 @@ import {
   AccountWebhookOrder,
   Xilnex
 } from '@things-factory/integration-accounting'
+import { webhookHandler, WebhookEventsEnum } from '@things-factory/integration-base'
 // import { createPayloadLog, PayloadType } from '@things-factory/integration-base'
 import { Product, ProductBundle, ProductDetail } from '@things-factory/product-base'
 import {
@@ -545,27 +546,41 @@ xilnexRouter.post('/webhook-xilnex-transfer-note', async (context, next) => {
               })
             )
 
+            let result
             if (errorList.length > 0) {
-              context.throw(404, errorList.join(', '))
+              // Insufficient stock: persist the order as a DraftReleaseGood so it is
+              // not lost. Once stock arrives, the draft can be released via the
+              // existing draft-to-release flow.
+              logger.warn(
+                `xilnex-router[${releaseGood.orderSourceType}][insufficient-stock]: refNo=${releaseGood.refNo} creating DraftReleaseGood; details=${errorList.join(', ')}`
+              )
+              result = await createDraftReleaseGood(
+                releaseGood,
+                massagedData.orderProducts,
+                customerBizplace,
+                warehouseDomain,
+                context,
+                tx
+              )
+            } else {
+              const assignedOrderProducts = await assignToInventory(
+                massagedData.orderProducts,
+                customerBizplace,
+                warehouseDomain,
+                context,
+                tx
+              )
+
+              result = await createReleaseGood(
+                releaseGood,
+                assignedOrderProducts,
+                customerBizplace,
+                warehouseDomain,
+                context,
+                tx
+              )
             }
 
-            let assignedOrderProducts = await assignToInventory(
-              massagedData.orderProducts,
-              customerBizplace,
-              warehouseDomain,
-              context,
-              tx
-            )
-
-            let result = await createReleaseGood(
-              releaseGood,
-              assignedOrderProducts,
-              customerBizplace,
-              warehouseDomain,
-              context,
-              tx
-            )
-
             let addReleaseGood = {
               id: result.id,
               name: result.name,
@@ -889,27 +904,41 @@ xilnexRouter.post('/webhook-xilnex-stock-request', async (context, next) => {
               })
             )
 
+            let result
             if (errorList.length > 0) {
-              context.throw(404, errorList.join(', '))
+              // Insufficient stock: persist the order as a DraftReleaseGood so it is
+              // not lost. Once stock arrives, the draft can be released via the
+              // existing draft-to-release flow.
+              logger.warn(
+                `xilnex-router[${releaseGood.orderSourceType}][insufficient-stock]: refNo=${releaseGood.refNo} creating DraftReleaseGood; details=${errorList.join(', ')}`
+              )
+              result = await createDraftReleaseGood(
+                releaseGood,
+                massagedData.orderProducts,
+                customerBizplace,
+                warehouseDomain,
+                context,
+                tx
+              )
+            } else {
+              const assignedOrderProducts = await assignToInventory(
+                massagedData.orderProducts,
+                customerBizplace,
+                warehouseDomain,
+                context,
+                tx
+              )
+
+              result = await createReleaseGood(
+                releaseGood,
+                assignedOrderProducts,
+                customerBizplace,
+                warehouseDomain,
+                context,
+                tx
+              )
             }
 
-            let assignedOrderProducts = await assignToInventory(
-              massagedData.orderProducts,
-              customerBizplace,
-              warehouseDomain,
-              context,
-              tx
-            )
-
-            let result = await createReleaseGood(
-              releaseGood,
-              assignedOrderProducts,
-              customerBizplace,
-              warehouseDomain,
-              context,
-              tx
-            )
-
             let addReleaseGood = {
               id: result.id,
               name: result.name,
@@ -1072,12 +1101,21 @@ xilnexRouter.post('/webhook-xilnex-stock-request', async (context, next) => {
 })
 
 async function checkExistingOrder(releaseGood, bizplace, checkDuplicationValidation = true, context, tx) {
-  const { domain }: { domain: Domain } = context.state
+  // Use the warehouse domain set on the releaseGood (caller passes warehouseDomain explicitly)
+  // so dedupe is consistent with the domain that ReleaseGood / DraftReleaseGood are saved under.
+  const domain: Domain = releaseGood.domain
   const refNo: string = releaseGood.refNo
   const refNo2: string = releaseGood.refNo2
   const refNo3: string = releaseGood.refNo3
 
   if (checkDuplicationValidation) {
+    // Acquire a transaction-scoped advisory lock keyed on the dedupe tuple. Serializes
+    // concurrent webhook deliveries for the same (domain, refNo, refNo2, refNo3) so the
+    // SELECT-then-INSERT pattern below can't race two duplicates through. The lock is
+    // automatically released when the transaction commits or rolls back.
+    const lockKey = `xilnex-dedupe:${domain.id}:${refNo || ''}:${refNo2 || ''}:${refNo3 || ''}`
+    await tx.query(`SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, [lockKey])
+
     const foundDraftReleaseGood: DraftReleaseGood = await tx.getRepository(DraftReleaseGood).findOne({
       where: {
         domain,
@@ -1770,3 +1808,93 @@ async function createReleaseGood(
     throw error
   }
 }
+
+// Inline draft creation that mirrors createReleaseGood's domain handling: takes warehouseDomain
+// explicitly instead of relying on context.state.domain. A simplified parallel of
+// createDraftReleaseGoodFunction in @things-factory/sales-base — kept tight (no LMD / shipping
+// order / attachments / country normalization) because none apply to the Xilnex webhook path.
+async function createDraftReleaseGood(
+  releaseGood: any,
+  orderProducts: OrderProduct[],
+  bizplace: Bizplace,
+  domain: Domain,
+  context: any,
+  tx: EntityManager
+): Promise<DraftReleaseGood> {
+  const { user }: { user: any } = context.state
+
+  if (orderProducts.length <= 0) {
+    throw new ValidationError({
+      errorCode: 'ITEM_NOT_FOUND',
+      message: 'no order items found',
+      detail: JSON.stringify(releaseGood)
+    })
+  }
+
+  // Resolve DRO number under the warehouse domain
+  const droNoSetting: Setting = await tx.getRepository(Setting).findOne({
+    where: { domain, name: ORDER_NUMBER_SETTING_KEY.DRO_NUMBER_RULE }
+  })
+
+  const orderNo: string = droNoSetting
+    ? await generateId({ domain, type: ORDER_NUMBER_RULE_TYPE.DRO_NUMBER, seed: {} })
+    : OrderNoGenerator.draftReleaseGood()
+
+  const newDraftReleaseGood = {
+    name: orderNo,
+    domain,
+    bizplace,
+    refNo: releaseGood.refNo,
+    refNo2: releaseGood.refNo2 || null,
+    refNo3: releaseGood.refNo3 || null,
+    collectionOrderNo: releaseGood.collectionOrderNo,
+    type: releaseGood?.type ? releaseGood.type : 'b2b',
+    courierOption: releaseGood.courierOption,
+    exportOption: releaseGood.exportOption,
+    ownTransport: releaseGood.ownTransport,
+    packingOption: releaseGood.packingOption,
+    marketplaceOrderStatus: releaseGood?.marketplaceOrderStatus || null,
+    releaseDate: releaseGood.releaseDate,
+    remark: releaseGood?.remark || null,
+    source: releaseGood?.source,
+    status: DRAFT_RELEASE_ORDER_STATUS.DRAFT,
+    creator: user,
+    updater: user
+  }
+
+  const resultDraftReleaseGood: DraftReleaseGood = await tx.getRepository(DraftReleaseGood).save(newDraftReleaseGood)
+
+  // Persist OrderProduct rows linked to the draft. Mirrors createReleaseGood's
+  // OrderProduct save loop, swapping releaseGood for draftReleaseGood and skipping
+  // OrderInventory creation (drafts have no assigned inventory yet).
+  const savedOrderProducts = await Promise.all(
+    orderProducts.map(async orderProduct => {
+      // status MUST be 'DRAFT' — draft-release-good-create.ts:130 filters
+      // orderProducts on `op.status == 'DRAFT'` during draft → release conversion.
+      // Setting any other ORDER_PRODUCT_STATUS value here would cause the converted
+      // ReleaseGood to come out with zero line items.
+      const draftOrderProduct = Object.assign(new OrderProduct(), {
+        ...orderProduct,
+        name: OrderNoGenerator.orderProduct(),
+        domain,
+        bizplace,
+        draftReleaseGood: resultDraftReleaseGood,
+        type: ORDER_TYPES.RELEASE_OF_GOODS,
+        status: DRAFT_RELEASE_ORDER_STATUS.DRAFT,
+        creator: user,
+        updater: user
+      })
+      return await tx.getRepository(OrderProduct).save(draftOrderProduct)
+    })
+  )
+
+  resultDraftReleaseGood.orderProducts = savedOrderProducts
+
+  // Notify external subscribers that a draft has been created. Fire-and-forget,
+  // matching the pattern used at add-release-order.ts:557 for ReleaseOrderCreated.
+  // Note: this fires inside the outer transaction; if the transaction rolls back
+  // the webhook will still have been queued. Consumers should be idempotent.
+  webhookHandler(resultDraftReleaseGood, bizplace, WebhookEventsEnum.DraftOrderCreated)
+
+  return resultDraftReleaseGood
+}