@develit-services/bank 5.0.1 → 5.1.0

package/dist/export/workflows.cjs

@@ -1,10 +1,10 @@
 'use strict';
 
 const backendSdk = require('@develit-io/backend-sdk');
-const ott_zod = require('../shared/bank.BSX82jhx.cjs');
+const bank = require('../shared/bank.CjEKRnVl.cjs');
 const batchLifecycle = require('../shared/bank.NF8bZBy0.cjs');
 const drizzleOrm = require('drizzle-orm');
-const credentialsResolver = require('../shared/bank.CibQRM2D.cjs');
+const credentialsResolver = require('../shared/bank.Dji7yhN_.cjs');
 const cloudflare_workers = require('cloudflare:workers');
 const cloudflare_workflows = require('cloudflare:workflows');
 const d1 = require('drizzle-orm/d1');
@@ -22,10 +22,10 @@ const updateAccountLastSyncCommand = (db, {
   accountId,
   lastSyncMetadata
 }) => {
-  const command = db.update(ott_zod.tables.account).set({
+  const command = db.update(bank.tables.account).set({
     lastSyncAt,
     lastSyncMetadata
-  }).where(drizzleOrm.eq(ott_zod.tables.account.id, accountId)).returning();
+  }).where(drizzleOrm.eq(bank.tables.account.id, accountId)).returning();
   return {
     command
   };
@@ -63,7 +63,7 @@ async function failBatchAndPayments(db, batch, paymentRequests, reason) {
 class BankProcessBatch extends cloudflare_workers.WorkflowEntrypoint {
   async run(event, step) {
     const { batchId } = event.payload;
-    const db = d1.drizzle(this.env.BANK_D1, { schema: ott_zod.tables, relations: ott_zod.relations });
+    const db = d1.drizzle(this.env.BANK_D1, { schema: bank.tables, relations: bank.relations });
     const batch = await step.do("load batch", async () => {
       const batch2 = await credentialsResolver.getBatchByIdQuery(db, { batchId });
       if (!batch2) {
@@ -109,14 +109,14 @@ class BankProcessBatch extends cloudflare_workers.WorkflowEntrypoint {
       },
       async () => {
         const batchedPayments = paymentRequests.map(
-          ott_zod.toBatchedPaymentFromPaymentRequest
+          bank.toBatchedPaymentFromPaymentRequest
         );
         const currentBatch = await credentialsResolver.getBatchByIdQuery(db, { batchId });
         if (currentBatch?.batchPaymentInitiatedAt) {
           return {
             authorizationUrls: currentBatch.authorizationUrls,
             preparedPayments: batchedPayments.map(
-              (p) => ott_zod.toPreparedPayment(
+              (p) => bank.toPreparedPayment(
                 p,
                 void 0,
                 currentBatch.batchPaymentInitiatedAt
@@ -134,7 +134,7 @@ class BankProcessBatch extends cloudflare_workers.WorkflowEntrypoint {
           throw new cloudflare_workflows.NonRetryableError(msg);
         }
         const resolveCredentials = await credentialsResolver.createCredentialsResolver(db, this.env);
-        const connector = await ott_zod.initiateConnector({
+        const connector = await bank.initiateConnector({
           env: this.env,
           bank: account.connectorKey,
           resolveCredentials,
@@ -306,6 +306,7 @@ function createWorkflowLogger(instanceId) {
 }
 
 function getStepCount(ctx) {
+  if (!ctx) return 0;
   return ctx.step?.count ?? 0;
 }
 
@@ -314,17 +315,21 @@ async function pushToQueue(queue, message) {
     await queue.send(message, { contentType: "v8" });
     return;
   }
-  await queue.sendBatch(
-    message.map((m) => ({
-      body: m,
-      contentType: "v8"
-    }))
-  );
+  const QUEUE_BATCH_SIZE = 100;
+  for (let i = 0; i < message.length; i += QUEUE_BATCH_SIZE) {
+    const chunk = message.slice(i, i + QUEUE_BATCH_SIZE);
+    await queue.sendBatch(
+      chunk.map((m) => ({
+        body: m,
+        contentType: "v8"
+      }))
+    );
+  }
 }
 class BankSyncAccountPayments extends cloudflare_workers.WorkflowEntrypoint {
   async run(event, step) {
     const { accountId } = event.payload;
-    const db = d1.drizzle(this.env.BANK_D1, { schema: ott_zod.tables, relations: ott_zod.relations });
+    const db = d1.drizzle(this.env.BANK_D1, { schema: bank.tables, relations: bank.relations });
     const logger = createWorkflowLogger(event.instanceId);
     if (!accountId) {
       throw new cloudflare_workflows.NonRetryableError(`Haven't obtained accountId to load.`);
@@ -345,13 +350,13 @@ class BankSyncAccountPayments extends cloudflare_workers.WorkflowEntrypoint {
       if (!account.lastSyncAt) {
         throw new Error(`lastSyncedAt is not set for account: ${accountId}`);
       }
-      const payments = await step.do(
-        "fetch bank payments",
+      await step.do(
+        "fetch and process payments",
         {
           retries: { limit: 5, delay: "2 minutes", backoff: "exponential" },
-          timeout: "2 minutes"
+          timeout: "5 minutes"
         },
-        async () => {
+        async (ctx) => {
           try {
             logger.info("payments.fetch.started", {
               accountId,
@@ -362,7 +367,7 @@ class BankSyncAccountPayments extends cloudflare_workers.WorkflowEntrypoint {
               db,
               this.env
             );
-            const connector = await ott_zod.initiateConnector({
+            const connector = await bank.initiateConnector({
               env: this.env,
               bank: account.connectorKey,
               resolveCredentials,
@@ -374,19 +379,192 @@ class BankSyncAccountPayments extends cloudflare_workers.WorkflowEntrypoint {
                 }
               ]
             });
-            const result = await connector.getAllAccountPayments({
+            const payments = await connector.getAllAccountPayments({
               account,
               filter: { dateFrom: account.lastSyncAt }
             });
             logger.info("payments.fetch.completed", {
               accountId,
               connectorKey: account.connectorKey,
-              paymentsCount: result.length
+              paymentsCount: payments.length
+            });
+            const paymentsToProcess = payments.filter(
+              (p) => bank.isPaymentCompleted(p.parsed)
+            );
+            logger.info("payments.filtered.toProcess", {
+              accountId,
+              totalFetched: payments.length,
+              paymentsToProcess: paymentsToProcess.length,
+              sampleStatuses: payments.slice(0, 5).map((p) => ({
+                bankRefId: p.parsed.bankRefId,
+                status: p.parsed.status,
+                isCompleted: bank.isPaymentCompleted(p.parsed)
+              }))
+            });
+            const lastSyncBankRefIds = account.lastSyncMetadata?.lastSyncBankRefIds || [];
+            const paymentsToInsert = paymentsToProcess.filter(
+              (p) => !lastSyncBankRefIds.includes(p.parsed.bankRefId)
+            );
+            logger.info("payments.filtered.toInsert", {
+              accountId,
+              paymentsToProcess: paymentsToProcess.length,
+              paymentsToInsert: paymentsToInsert.length,
+              lastSyncBankRefIdsCount: lastSyncBankRefIds.length,
+              sampleLastSyncBankRefIds: lastSyncBankRefIds.slice(0, 10),
+              sampleToInsert: paymentsToInsert.slice(0, 5).map((p) => p.parsed.bankRefId)
+            });
+            const eventsToEmit = [];
+            const bankRefIds = paymentsToInsert.map((p) => p.parsed.bankRefId).filter((id) => id != null);
+            const BANK_REF_ID_CHUNK_SIZE = 90;
+            const matchingRequests = [];
+            for (let i = 0; i < bankRefIds.length; i += BANK_REF_ID_CHUNK_SIZE) {
+              const chunkIds = bankRefIds.slice(i, i + BANK_REF_ID_CHUNK_SIZE);
+              const rows = await db.select().from(bank.tables.paymentRequest).where(
+                drizzleOrm.and(
+                  drizzleOrm.inArray(bank.tables.paymentRequest.bankRefId, chunkIds),
+                  drizzleOrm.eq(bank.tables.paymentRequest.accountId, account.id),
+                  drizzleOrm.eq(
+                    bank.tables.paymentRequest.connectorKey,
+                    account.connectorKey
+                  )
+                )
+              );
+              matchingRequests.push(...rows);
+            }
+            const requestByBankRefId = Object.fromEntries(
+              matchingRequests.map((r) => [r.bankRefId, r])
+            );
+            const enrichedPayments = paymentsToInsert.map((p) => {
+              const req = p.parsed.bankRefId && requestByBankRefId[p.parsed.bankRefId] || null;
+              if (!req) return p;
+              return {
+                ...p,
+                parsed: {
+                  ...p.parsed,
+                  // queue-bus: transaction matching (DBU doesn't echo these in statements)
+                  vs: p.parsed.vs ?? req.vs,
+                  ss: p.parsed.ss ?? req.ss,
+                  ks: p.parsed.ks ?? req.ks,
+                  message: p.parsed.message ?? req.message,
+                  // queue-bus: party creation
+                  creditor: {
+                    ...p.parsed.creditor,
+                    holderName: p.parsed.creditor?.holderName ?? req.creditor?.holderName
+                  },
+                  debtor: {
+                    ...p.parsed.debtor,
+                    holderName: p.parsed.debtor?.holderName ?? req.debtor?.holderName
+                  }
+                  // NOT enriched: chargeBearer, instructionPriority, refId, batchId,
+                  //               createdAt, address, swiftBic — no downstream consumer in payment table
+                }
+              };
+            });
+            const createCommands = enrichedPayments.map(
+              (p) => credentialsResolver.createPaymentCommand(db, { payment: p.parsed }).command
+            );
+            logger.info("payments.commands.created", {
+              accountId,
+              createCommandsCount: createCommands.length,
+              enrichedPaymentsCount: enrichedPayments.length
+            });
+            eventsToEmit.push(
+              ...enrichedPayments.map((p) => ({
+                eventType: "BANK_PAYMENT",
+                eventSignal: "paymentFetched",
+                bankPayment: p.parsed,
+                metadata: {
+                  correlationId: p.parsed.correlationId,
+                  entityId: p.parsed.id,
+                  timestamp: /* @__PURE__ */ new Date()
+                }
+              }))
+            );
+            const lastSyncMetadata = {
+              payments: payments.length,
+              paymentsToProcess: paymentsToProcess.length,
+              paymentsInserted: paymentsToInsert.length,
+              lastSyncBankRefIds: paymentsToProcess.filter((p) => p.parsed.status === "BOOKED").map((p) => p.parsed.bankRefId),
+              lastSyncPayments: lastSyncBankRefIds.length,
+              eventsEmitted: eventsToEmit.length,
+              iterationCount: getStepCount(ctx),
+              workflowStartedAt
+            };
+            const updateLastSyncCommand = updateAccountLastSyncCommand(db, {
+              accountId: account.id,
+              lastSyncAt: now,
+              lastSyncMetadata
+            }).command;
+            logger.info("payments.database.beforeBatch", {
+              accountId,
+              createCommandsCount: createCommands.length,
+              willUseBatch: createCommands.length > 0
             });
-            return result;
+            if (createCommands.length) {
+              logger.info("payments.database.batchStart", {
+                accountId,
+                totalCommands: createCommands.length + 1,
+                paymentsToInsert: createCommands.length
+              });
+              const BATCH_CHUNK_SIZE = 90;
+              let totalInserted = 0;
+              for (let i = 0; i < createCommands.length; i += BATCH_CHUNK_SIZE) {
+                const chunkCommands = createCommands.slice(
+                  i,
+                  i + BATCH_CHUNK_SIZE
+                );
+                const isLastChunk = i + BATCH_CHUNK_SIZE >= createCommands.length;
+                const batchCommands = isLastChunk ? backendSdk.asNonEmpty([updateLastSyncCommand, ...chunkCommands]) : backendSdk.asNonEmpty(chunkCommands);
+                logger.info("payments.database.batchChunk", {
+                  accountId,
+                  chunkIndex: Math.floor(i / BATCH_CHUNK_SIZE),
+                  chunkSize: chunkCommands.length,
+                  isLastChunk
+                });
+                await db.batch(batchCommands);
+                totalInserted += chunkCommands.length;
+              }
+              logger.info("payments.database.batchComplete", {
+                accountId,
+                inserted: totalInserted,
+                chunks: Math.ceil(createCommands.length / BATCH_CHUNK_SIZE)
+              });
+            } else {
+              logger.info("payments.database.updateOnly", {
+                accountId,
+                reason: "no new payments to insert"
+              });
+              await updateLastSyncCommand.execute();
+              logger.info("payments.database.updateComplete", {
+                accountId
+              });
+            }
+            if (eventsToEmit.length) {
+              logger.info("payments.queue.sending", {
+                accountId,
+                eventsCount: eventsToEmit.length
+              });
+              await pushToQueue(
+                this.env.QUEUE_BUS_QUEUE,
+                eventsToEmit
+              );
+              logger.info("payments.queue.sent", {
+                accountId,
+                eventsCount: eventsToEmit.length
+              });
+            }
+            logger.info("payments.process.complete", {
+              accountId,
+              ...lastSyncMetadata,
+              newLastSyncAt: now
+            });
+            return {
+              ...lastSyncMetadata,
+              newLastSyncAt: now
+            };
           } catch (err) {
             const message = err instanceof Error ? err.message : typeof err === "object" && err !== null && "message" in err ? String(err.message) : JSON.stringify(err);
-            logger.error("payments.fetch.failed", {
+            logger.error("payments.process.failed", {
               accountId,
               connectorKey: account.connectorKey,
               error: message
@@ -395,109 +573,6 @@ class BankSyncAccountPayments extends cloudflare_workers.WorkflowEntrypoint {
           }
         }
       );
-      const paymentsToProcess = payments.filter(
-        (p) => ott_zod.isPaymentCompleted(p.parsed)
-      );
-      const lastSyncBankRefIds = account.lastSyncMetadata?.lastSyncBankRefIds || [];
-      const paymentsToInsert = paymentsToProcess.filter(
-        (p) => !lastSyncBankRefIds.includes(p.parsed.bankRefId)
-      );
-      await step.do(
-        "process new payments and update lastSyncAt",
-        async (ctx) => {
-          const eventsToEmit = [];
-          const bankRefIds = paymentsToInsert.map((p) => p.parsed.bankRefId).filter((id) => id != null);
-          const BANK_REF_ID_CHUNK_SIZE = 90;
-          const matchingRequests = [];
-          for (let i = 0; i < bankRefIds.length; i += BANK_REF_ID_CHUNK_SIZE) {
-            const chunkIds = bankRefIds.slice(i, i + BANK_REF_ID_CHUNK_SIZE);
-            const rows = await db.select().from(ott_zod.tables.paymentRequest).where(
-              drizzleOrm.and(
-                drizzleOrm.inArray(ott_zod.tables.paymentRequest.bankRefId, chunkIds),
-                drizzleOrm.eq(ott_zod.tables.paymentRequest.accountId, account.id),
-                drizzleOrm.eq(ott_zod.tables.paymentRequest.connectorKey, account.connectorKey)
-              )
-            );
-            matchingRequests.push(...rows);
-          }
-          const requestByBankRefId = Object.fromEntries(
-            matchingRequests.map((r) => [r.bankRefId, r])
-          );
-          const enrichedPayments = paymentsToInsert.map((p) => {
-            const req = p.parsed.bankRefId && requestByBankRefId[p.parsed.bankRefId] || null;
-            if (!req) return p;
-            return {
-              ...p,
-              parsed: {
-                ...p.parsed,
-                // queue-bus: transaction matching (DBU doesn't echo these in statements)
-                vs: p.parsed.vs ?? req.vs,
-                ss: p.parsed.ss ?? req.ss,
-                ks: p.parsed.ks ?? req.ks,
-                message: p.parsed.message ?? req.message,
-                // queue-bus: party creation
-                creditor: {
-                  ...p.parsed.creditor,
-                  holderName: p.parsed.creditor?.holderName ?? req.creditor?.holderName
-                },
-                debtor: {
-                  ...p.parsed.debtor,
-                  holderName: p.parsed.debtor?.holderName ?? req.debtor?.holderName
-                }
-                // NOT enriched: chargeBearer, instructionPriority, refId, batchId,
-                //               createdAt, address, swiftBic — no downstream consumer in payment table
-              }
-            };
-          });
-          const createCommands = enrichedPayments.map(
-            (p) => credentialsResolver.createPaymentCommand(db, { payment: p.parsed }).command
-          );
-          eventsToEmit.push(
-            ...enrichedPayments.map((p) => ({
-              eventType: "BANK_PAYMENT",
-              eventSignal: "paymentFetched",
-              bankPayment: p.parsed,
-              metadata: {
-                correlationId: p.parsed.correlationId,
-                entityId: p.parsed.id,
-                timestamp: /* @__PURE__ */ new Date()
-              }
-            }))
-          );
-          const lastSyncMetadata = {
-            payments: payments.length,
-            paymentsToProcess: paymentsToProcess.length,
-            paymentsInserted: paymentsToInsert.length,
-            lastSyncBankRefIds: paymentsToProcess.filter((p) => p.parsed.status === "BOOKED").map((p) => p.parsed.bankRefId),
-            lastSyncPayments: lastSyncBankRefIds.length,
-            eventsEmitted: eventsToEmit.length,
-            iterationCount: getStepCount(ctx),
-            workflowStartedAt
-          };
-          const updateLastSyncCommand = updateAccountLastSyncCommand(db, {
-            accountId: account.id,
-            lastSyncAt: now,
-            lastSyncMetadata
-          }).command;
-          if (createCommands.length) {
-            await db.batch(
-              backendSdk.asNonEmpty([updateLastSyncCommand, ...createCommands])
-            );
-          } else {
-            await updateLastSyncCommand;
-          }
-          if (eventsToEmit.length) {
-            await pushToQueue(
-              this.env.QUEUE_BUS_QUEUE,
-              eventsToEmit
-            );
-          }
-          return {
-            ...lastSyncMetadata,
-            newLastSyncAt: now
-          };
-        }
-      );
       await step.sleep(
         "Sleep for next sync",
         `${account.syncIntervalS} seconds`