@develit-services/bank 5.7.1 → 5.9.0

package/dist/export/workflows.cjs

@@ -1,16 +1,16 @@
 'use strict';
 
 const backendSdk = require('@develit-io/backend-sdk');
-const paymentDirection = require('../shared/bank.CQjuudum.cjs');
+const paymentDirection = require('../shared/bank.BsWBG0gb.cjs');
 const batchLifecycle = require('../shared/bank.NF8bZBy0.cjs');
 const drizzleOrm = require('drizzle-orm');
-const bank = require('../shared/bank.B0DNtuUM.cjs');
+const bank = require('../shared/bank.DBTtRCcf.cjs');
 const cloudflare_workers = require('cloudflare:workers');
 const cloudflare_workflows = require('cloudflare:workflows');
 const d1 = require('drizzle-orm/d1');
 require('node:crypto');
 require('date-fns');
-require('../shared/bank.9Yw4KHyl.cjs');
+require('../shared/bank.Ca3jzmIb.cjs');
 require('drizzle-orm/sqlite-core');
 require('@develit-io/general-codes');
 require('jose');
@@ -20,12 +20,17 @@ require('drizzle-orm/zod');
 const updateAccountLastSyncCommand = (db, {
   lastSyncAt,
   accountId,
-  lastSyncMetadata
+  lastSyncMetadata,
+  seenInstanceId
 }) => {
+  const markerUnchanged = seenInstanceId === void 0 ? void 0 : (
+    // IS instead of = so a null marker compares equal to null.
+    drizzleOrm.sql`json_extract(${paymentDirection.tables.account.lastSyncMetadata}, '$.instanceId') IS ${seenInstanceId}`
+  );
   const command = db.update(paymentDirection.tables.account).set({
     lastSyncAt,
     lastSyncMetadata
-  }).where(drizzleOrm.eq(paymentDirection.tables.account.id, accountId)).returning();
+  }).where(drizzleOrm.and(drizzleOrm.eq(paymentDirection.tables.account.id, accountId), markerUnchanged)).returning();
   return {
     command
   };
@@ -340,6 +345,72 @@ function getStepCount(ctx) {
   return ctx.step?.count ?? 0;
 }
 
+const MAX_CATCH_UP_WINDOW_MS = 7 * 24 * 60 * 60 * 1e3;
+function resolveSyncWindow(lastSyncAt, nowMs, maxWindowMs) {
+  const openEnded = {
+    dateFrom: lastSyncAt,
+    dateTo: new Date(Math.max(nowMs, lastSyncAt.getTime())),
+    isCatchUp: false
+  };
+  if (!Number.isFinite(maxWindowMs) || maxWindowMs <= 0) return openEnded;
+  const clippedToMs = lastSyncAt.getTime() + maxWindowMs;
+  if (clippedToMs >= nowMs) return openEnded;
+  return {
+    dateFrom: lastSyncAt,
+    dateTo: new Date(clippedToMs),
+    isCatchUp: true
+  };
+}
+
+const STEP_RESULT_BYTE_CEILING = 800 * 1024;
+class StepResultTooLargeError extends cloudflare_workflows.NonRetryableError {
+}
+function guardStepResultSize(value, ceilingBytes = STEP_RESULT_BYTE_CEILING) {
+  const bytes = new TextEncoder().encode(JSON.stringify(value)).byteLength;
+  if (bytes > ceilingBytes) {
+    const count = Array.isArray(value) ? value.length : 1;
+    throw new StepResultTooLargeError(
+      `Fetched ${count} payments (${bytes} bytes), over the ${ceilingBytes} byte step result ceiling`
+    );
+  }
+}
+function rethrowFetchError(err) {
+  if (err instanceof cloudflare_workflows.NonRetryableError) throw err;
+  const message = err instanceof Error ? err.message : typeof err === "object" && err !== null && "message" in err ? String(err.message) : JSON.stringify(err);
+  throw new Error(message);
+}
+
+const MIN_CATCH_UP_WINDOW_MS = 6 * 60 * 60 * 1e3;
+async function fetchBoundedByWindow({
+  window,
+  fetchWindow,
+  guardResult,
+  minWindowMs,
+  onNarrowed
+}) {
+  let current = window;
+  while (true) {
+    const result = await fetchWindow(current);
+    try {
+      guardResult(result);
+      return { result, window: current };
+    } catch (tooLarge) {
+      if (!(tooLarge instanceof StepResultTooLargeError)) throw tooLarge;
+      const halvedSpanMs = Math.floor(
+        (current.dateTo.getTime() - current.dateFrom.getTime()) / 2
+      );
+      if (halvedSpanMs < minWindowMs) throw tooLarge;
+      current = {
+        dateFrom: current.dateFrom,
+        dateTo: new Date(current.dateFrom.getTime() + halvedSpanMs),
+        // Stops short of the original target, so another pass is needed.
+        isCatchUp: true
+      };
+      onNarrowed?.(current);
+    }
+  }
+}
+
 async function pushToQueue(queue, message) {
   if (!Array.isArray(message)) {
     await queue.send(message, { contentType: "v8" });
@@ -354,7 +425,8 @@ async function pushToQueue(queue, message) {
 }
 class BankSyncAccountPayments extends cloudflare_workers.WorkflowEntrypoint {
   async run(event, step) {
-    const { accountId } = event.payload;
+    const { accountId, maxIterations } = event.payload;
+    const iterationBudget = bank.resolveIterationBudget(maxIterations);
     const db = d1.drizzle(this.env.BANK_D1, { schema: paymentDirection.tables, relations: paymentDirection.relations });
     const logger = createWorkflowLogger(event.instanceId);
     if (!accountId) {
@@ -364,8 +436,7 @@ class BankSyncAccountPayments extends cloudflare_workers.WorkflowEntrypoint {
       "capture workflow start time",
       async () => Date.now()
     );
-    while (true) {
-      const now = /* @__PURE__ */ new Date();
+    for (let iteration = 0; iteration < iterationBudget; iteration++) {
       const account = await step.do("load account", async () => {
         const account2 = await bank.getAccountByIdQuery(db, { accountId });
         if (!account2) {
@@ -373,10 +444,49 @@ class BankSyncAccountPayments extends cloudflare_workers.WorkflowEntrypoint {
         }
         return account2;
       });
+      if (!account.syncEnabled) {
+        logger.info("sync.disabled", {
+          accountId,
+          instanceId: event.instanceId,
+          iteration
+        });
+        return;
+      }
       if (!account.lastSyncAt) {
         throw new Error(`lastSyncedAt is not set for account: ${accountId}`);
       }
-      const payments = await step.do(
+      if (bank.isSupersededBy(
+        event.instanceId,
+        account.lastSyncMetadata?.instanceId,
+        accountId
+      )) {
+        logger.info("sync.superseded", {
+          accountId,
+          instanceId: event.instanceId,
+          supersededBy: account.lastSyncMetadata?.instanceId,
+          iteration
+        });
+        return;
+      }
+      const lastSyncAtMs = account.lastSyncAt.getTime();
+      const windowMs = await step.do("resolve sync window", async () => {
+        const window = resolveSyncWindow(
+          new Date(lastSyncAtMs),
+          Date.now(),
+          MAX_CATCH_UP_WINDOW_MS
+        );
+        return {
+          dateFromMs: window.dateFrom.getTime(),
+          dateToMs: window.dateTo.getTime(),
+          isCatchUp: window.isCatchUp
+        };
+      });
+      const syncWindow = {
+        dateFrom: new Date(windowMs.dateFromMs),
+        dateTo: new Date(windowMs.dateToMs),
+        isCatchUp: windowMs.isCatchUp
+      };
+      const fetched = await step.do(
         "fetch bank payments",
         {
           retries: { limit: 5, delay: "2 minutes", backoff: "exponential" },
@@ -405,42 +515,88 @@ class BankSyncAccountPayments extends cloudflare_workers.WorkflowEntrypoint {
                 }
               ]
             });
-            const result = await connector.getAllAccountPayments({
-              account,
-              filter: { dateFrom: account.lastSyncAt }
-            });
+            const fetchWindow = async (window) => {
+              const result = await connector.getAllAccountPayments({
+                account,
+                filter: {
+                  dateFrom: window.dateFrom,
+                  dateTo: window.dateTo
+                }
+              });
+              const identifiable = result.filter(
+                (p) => p.parsed.bankRefId?.trim()
+              );
+              const skipped = result.length - identifiable.length;
+              if (skipped > 0) {
+                logger.warn("payments.fetch.skipped_blank_bank_ref", {
+                  accountId,
+                  connectorKey: account.connectorKey,
+                  skipped
+                });
+              }
+              return Promise.all(
+                identifiable.map(async (p) => ({
+                  parsed: {
+                    ...p.parsed,
+                    id: await deriveV4PaymentId(p.parsed)
+                  }
+                }))
+              );
+            };
+            let bounded;
+            try {
+              bounded = await fetchBoundedByWindow({
+                window: syncWindow,
+                fetchWindow,
+                guardResult: guardStepResultSize,
+                minWindowMs: MIN_CATCH_UP_WINDOW_MS,
+                onNarrowed: (narrowed) => logger.warn("payments.fetch.window-narrowed", {
+                  accountId,
+                  connectorKey: account.connectorKey,
+                  dateFrom: narrowed.dateFrom.toISOString(),
+                  dateTo: narrowed.dateTo.toISOString()
+                })
+              });
+            } catch (err) {
+              if (err instanceof StepResultTooLargeError) {
+                logger.error("payments.fetch.result-too-large", {
+                  accountId,
+                  connectorKey: account.connectorKey,
+                  dateFrom: syncWindow.dateFrom.toISOString(),
+                  dateTo: syncWindow.dateTo.toISOString()
+                });
+              }
+              throw err;
+            }
             logger.info("payments.fetch.completed", {
               accountId,
               connectorKey: account.connectorKey,
-              paymentsCount: result.length
+              paymentsCount: bounded.result.length
             });
-            const identifiable = result.filter(
-              (p) => p.parsed.bankRefId?.trim()
-            );
-            const skipped = result.length - identifiable.length;
-            if (skipped > 0) {
-              logger.warn("payments.fetch.skipped_blank_bank_ref", {
-                accountId,
-                connectorKey: account.connectorKey,
-                skipped
-              });
-            }
-            return Promise.all(
-              identifiable.map(async (p) => ({
-                parsed: { ...p.parsed, id: await deriveV4PaymentId(p.parsed) }
-              }))
-            );
+            return {
+              payments: bounded.result,
+              windowMs: {
+                dateFromMs: bounded.window.dateFrom.getTime(),
+                dateToMs: bounded.window.dateTo.getTime(),
+                isCatchUp: bounded.window.isCatchUp
+              }
+            };
           } catch (err) {
-            const message = err instanceof Error ? err.message : typeof err === "object" && err !== null && "message" in err ? String(err.message) : JSON.stringify(err);
             logger.error("payments.fetch.failed", {
               accountId,
               connectorKey: account.connectorKey,
-              error: message
+              error: err instanceof Error ? err.message : String(err)
             });
-            throw new Error(message);
+            rethrowFetchError(err);
           }
         }
       );
+      const payments = fetched.payments;
+      const effectiveWindow = {
+        dateFrom: new Date(fetched.windowMs.dateFromMs),
+        dateTo: new Date(fetched.windowMs.dateToMs),
+        isCatchUp: fetched.windowMs.isCatchUp
+      };
       const paymentsToProcess = payments.filter(
         (p) => paymentDirection.isPaymentCompleted(p.parsed)
       );
@@ -515,6 +671,7 @@ class BankSyncAccountPayments extends cloudflare_workers.WorkflowEntrypoint {
             }))
           );
           const lastSyncMetadata = {
+            instanceId: event.instanceId,
             payments: payments.length,
             paymentsToProcess: paymentsToProcess.length,
             paymentsInserted: paymentsToInsert.length,
@@ -531,15 +688,27 @@ class BankSyncAccountPayments extends cloudflare_workers.WorkflowEntrypoint {
           };
           const updateLastSyncCommand = updateAccountLastSyncCommand(db, {
             accountId: account.id,
-            lastSyncAt: now,
-            lastSyncMetadata
+            // End of the window just fetched, not `now`: while catching up the
+            // window stops short of now, and advancing past it would skip every
+            // payment in the gap.
+            lastSyncAt: effectiveWindow.dateTo,
+            lastSyncMetadata,
+            seenInstanceId: account.lastSyncMetadata?.instanceId ?? null
           }).command;
+          let syncStateRows;
           if (createCommands.length) {
-            await db.batch(
+            const [updateResult] = await db.batch(
               backendSdk.asNonEmpty([updateLastSyncCommand, ...createCommands])
             );
+            syncStateRows = updateResult;
           } else {
-            await updateLastSyncCommand;
+            syncStateRows = await updateLastSyncCommand;
+          }
+          if (syncStateRows.length === 0) {
+            logger.info("sync.write-superseded", {
+              accountId,
+              instanceId: event.instanceId
+            });
           }
           if (eventsToEmit.length) {
             await pushToQueue(
@@ -549,14 +718,16 @@ class BankSyncAccountPayments extends cloudflare_workers.WorkflowEntrypoint {
           }
           return {
             ...lastSyncMetadata,
-            newLastSyncAt: now
+            newLastSyncAt: effectiveWindow.dateTo
           };
         }
       );
-      await step.sleep(
-        "Sleep for next sync",
-        `${account.syncIntervalS} seconds`
-      );
+      if (!effectiveWindow.isCatchUp) {
+        await step.sleep(
+          "Sleep for next sync",
+          `${account.syncIntervalS} seconds`
+        );
+      }
     }
   }
 }

package/dist/export/workflows.d.cts

@@ -9,6 +9,7 @@ declare class BankProcessBatch extends WorkflowEntrypoint<BankEnv, Params$1> {
 
 type Params = {
     accountId: string;
+    maxIterations?: number;
 };
 declare class BankSyncAccountPayments extends WorkflowEntrypoint<BankEnv, Params> {
     run(event: WorkflowEvent<Params>, step: WorkflowStep): Promise<void>;

package/dist/export/workflows.d.mts

@@ -9,6 +9,7 @@ declare class BankProcessBatch extends WorkflowEntrypoint<BankEnv, Params$1> {
 
 type Params = {
     accountId: string;
+    maxIterations?: number;
 };
 declare class BankSyncAccountPayments extends WorkflowEntrypoint<BankEnv, Params> {
     run(event: WorkflowEvent<Params>, step: WorkflowStep): Promise<void>;

package/dist/export/workflows.d.ts

@@ -9,6 +9,7 @@ declare class BankProcessBatch extends WorkflowEntrypoint<BankEnv, Params$1> {
 
 type Params = {
     accountId: string;
+    maxIterations?: number;
 };
 declare class BankSyncAccountPayments extends WorkflowEntrypoint<BankEnv, Params> {
     run(event: WorkflowEvent<Params>, step: WorkflowStep): Promise<void>;