@seeka-labs/cli-apps 1.1.17 → 1.1.18

package/dist/init-templates/azure-function/README.md

@@ -100,6 +100,9 @@ This project comes ready to deploy for free to Azure functions with database bac
 ### Continuous delivery
 This template includes a GitLab CD pipeline that can be used to trigger deployments of your app when changes are pushed to your Git repository.
 
+### Configuring queues
+See [this article](https://learn.microsoft.com/en-us/azure/azure-functions/functions-bindings-storage-queue?tabs=isolated-process%2Cextensionv5%2Cextensionv3&pivots=programming-language-typescript#host-json) on settings that are settable in the `host.json` file.
+
 ## References
 - https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite
 - https://learn.microsoft.com/en-us/azure/storage/queues/storage-quickstart-queues-nodejs?tabs=connection-string%2Croles-azure-portal%2Cenvironment-variable-windows%2Csign-in-azure-cli

package/dist/init-templates/azure-function/host.json

@@ -14,7 +14,16 @@
     "version": "[4.*, 5.0.0)"
   },
   "concurrency": {
-    "dynamicConcurrencyEnabled": true,
+    "dynamicConcurrencyEnabled": false,
     "snapshotPersistenceEnabled": true
+  },
+  "extensions": {
+    "queues": {
+      "maxPollingInterval": "00:00:15",
+      "visibilityTimeout": "00:01:00",
+      "batchSize": 32,
+      "maxDequeueCount": 2,
+      "messageEncoding": "base64"
+    }
   }
 }

package/dist/init-templates/azure-function/package.json

@@ -13,7 +13,7 @@
     "lint": "eslint",
     "build": "<packageManagerRunPrefix> clean && tsc",
     "watch": "tsc -w",
-    "test": "yarn jest",
+    "test": "<packageManagerRunPrefix> jest",
     "clean": "<packageManagerRunPrefix> rimraf dist",
     "prestart": "<packageManagerRunPrefix> clean && <packageManagerRunPrefix> build",
     "dev": "<packageManagerRunPrefix> build && func start --port 7072",

package/dist/init-templates/azure-function/scripts/dev-queue-setup.js

@@ -15,4 +15,16 @@ const { QueueClient } = require("@azure/storage-queue");
     else {
         console.log("Queue already exists");
     }
+
+    // // https://learn.microsoft.com/en-us/azure/azure-functions/functions-bindings-storage-queue-trigger?tabs=python-v2%2Cisolated-process%2Cnodejs-v4%2Cextensionv5&pivots=programming-language-typescript#poison-messages
+    var poisonClient = new QueueClient(
+        `UseDevelopmentStorage=true`, `${process.argv[2]}-poison`
+    );
+    const resPoison = await poisonClient.createIfNotExists();
+    if (resPoison.succeeded) {
+        console.log("Poison queue created");
+    }
+    else {
+        console.log("Poison queue already exists");
+    }
 })();

package/dist/init-templates/azure-function/src/functions/queueExample.ts

@@ -2,9 +2,12 @@ import winston from 'winston';
 
 import { app, InvocationContext } from '@azure/functions';
 
-import { BackgroundJobRequestContext, deserialiseQueuePayload, queueNames } from '../lib/jobs';
+import { BackgroundJobRequestContext, deserialiseQueuePayload, queueNames, sendQueueMessageToPoisonQueue } from '../lib/jobs';
 import { backgroundJobLogger } from '../lib/logging';
 import { startServices } from '../lib/services';
+import { groupBy } from 'lodash';
+import { SeekaAppInstallState, tryGetInstallation } from '../lib/state/seeka/installations';
+import { SeekaActivityAcceptedWebhookContent } from '@seeka-labs/sdk-apps-server';
 
 app.storageQueue('queueExample', {
     queueName: queueNames.queueItemExampleQueueName,
@@ -12,23 +15,53 @@ app.storageQueue('queueExample', {
     handler: queueExample
 });
 
-export async function queueExample(queueItem: string, context: InvocationContext): Promise<void> {
-    const body = typeof queueItem === 'string' ? deserialiseQueuePayload<BackgroundJobRequestContext>(queueItem) : queueItem as BackgroundJobRequestContext;
+export interface MyQueueItem extends BackgroundJobRequestContext {
+    items: SeekaActivityAcceptedWebhookContent[];
+}
 
-    const logger = backgroundJobLogger(queueNames.queueItemExampleQueueName, body, context);
+export async function queueExample(queueItem: any, context: InvocationContext): Promise<void> {
+    const logger = backgroundJobLogger(queueNames.queueItemExampleQueueName, undefined, context);
     logger.profile(`queue.${queueNames.queueItemExampleQueueName}`)
+    try {
+        // queueItem can either be a single item or an array of items
+        const payload = deserialiseQueuePayload<MyQueueItem>(queueItem, logger);
 
-    await startServices(logger);
+        // Group by applicationInstallId
+        const grouped = groupBy(payload, e => e.applicationInstallId);
 
-    logger.verbose('Received request to trigger background job', { body });
+        logger.verbose('Received queue batch to handle queue message', { batchSize: payload.length });
 
-    try {
-        // Execute sync
-        // await executeLongRunningTask(body, logger);
+        // Process each group
+        await startServices(logger);
+        for (const [applicationInstallId, items] of Object.entries(grouped)) {
+            if (items.length === 0) {
+                logger.warn('No items to process for applicationInstallId', { applicationInstallId });
+                continue;
+            }
+            const thisLogger = backgroundJobLogger(queueNames.queueItemExampleQueueName, items[0], context);
+            try {
+                const installation = await tryGetInstallation(applicationInstallId, true, thisLogger) as SeekaAppInstallState;
+
+                // Execute sync
+                // const batchItems = items.flatMap(e => e.items || []).filter(Boolean)
+                // await executeLongRunningTask(batchItems, logger);
+            }
+            catch (err) {
+                thisLogger.error('Error handling queue item to handle queue message', { ex: winston.exceptions.getAllInfo(err) })
+                await sendQueueMessageToPoisonQueue(queueNames.queueItemExampleQueueName, {
+                    ...items[0],
+                    causationId: items[0].causationId,
+                    correlationId: context.invocationId,
+                    rows: items.flatMap(e => e.items || []).filter(Boolean)
+                } as MyQueueItem, thisLogger);
+            }
+        }
     }
     catch (err) {
-        logger.error('Error executing background job', { ex: winston.exceptions.getAllInfo(err) })
+        logger.error('Error handling queue item to handle queue message', { ex: winston.exceptions.getAllInfo(err) })
+        throw err; // Will retry based on host.json > extensions.queues.maxDequeueCount and then push to poison queue
+    }
+    finally {
+        logger.profile(`queue.${queueNames.queueItemExampleQueueName}`)
     }
-
-    logger.profile(`queue.${queueNames.queueItemExampleQueueName}`)
 }

package/dist/init-templates/azure-function/src/functions/seekaAppWebhook.ts

@@ -20,6 +20,7 @@ import {
 } from '../lib/state/seeka/installations';
 
 import type { Logger } from 'winston';
+import { MyQueueItem } from './queueExample';
 app.http('seekaAppWebhook', {
     methods: ['POST'],
     authLevel: 'anonymous',
@@ -226,31 +227,11 @@ const onInstallationSettingsUpdate = async (payload: SeekaAppInstallSettingsUpda
 }
 
 const handleSeekaActivity = async (activity: SeekaActivityAcceptedWebhookPayload, logger: Logger) => {
-    // const context = activity.context as SeekaAppWebhookContext;
-    // const helper = SeekaAppHelper.create(process.env['SEEKA_APP_SECRET'] as string, {
-    //     organisationId: context.organisationId as string,
-    //     applicationInstallId: context.applicationInstallId as string,
-    //     applicationId: process.env['SEEKA_APP_ID'] as string,
-    // }, { name, version }, logger);
-
-    // // Append a first name to the identity
-    // await helper.api.mergeIdentity({
-    //     seekaPId: activity.content?.personId,
-    //     firstName: [
-    //         'firstname_' + new Date().getTime()
-    //     ]
-    // }, {
-    //     method: 'toremove',
-    //     origin: TrackingEventSourceOriginType.Server
-    // })
-
-    // // Fire off a tracking event
-    // await helper.api.trackActivity({
-    //     activityName: TrackingActivityNames.Custom,
-    //     activityNameCustom: 'seeka-app-activity-accepted',
-    //     activityId: 'act' + new Date().getTime(),
-    // }, activity.content?.personId as string, {
-    //     method: 'toremove',
-    //     origin: TrackingEventSourceOriginType.Server
-    // })
+    // Will be handled by queueExample
+    await triggerBackgroundJob(queueNames.queueItemExampleQueueName, {
+        ...activity.context,
+        causationId: activity.causationId,
+        correlationId: activity.requestId,
+        items: [activity]
+    } as MyQueueItem, logger)
 }

package/dist/init-templates/azure-function/src/lib/jobs/index.ts

@@ -1,5 +1,7 @@
 import type { Logger } from 'winston';
 import { QueueClient } from '@azure/storage-queue';
+import { isArray } from 'lodash';
+import winston from 'winston';
 
 export interface BackgroundJobRequestContext {
     organisationId?: string;
@@ -30,9 +32,28 @@ const serialiseQueuePayload = (payload: unknown): string => {
     return Buffer.from(jsonString).toString('base64')
 }
 
-export const deserialiseQueuePayload = <TPayload>(payload: string): TPayload => {
-    const jsonString = Buffer.from(payload, 'base64').toString()
-    return JSON.parse(jsonString) as TPayload;
+export const deserialiseQueuePayload = <TPayload>(queueItem: any, logger: Logger): TPayload[] => {
+    try {
+        if (typeof queueItem !== 'string') {
+            if (isArray(queueItem)) {
+                return queueItem
+            }
+            return [queueItem]
+        }
+        else {
+            const jsonString = Buffer.from(queueItem, 'base64').toString()
+            const parsed = JSON.parse(jsonString);
+
+            if (isArray(parsed)) {
+                return parsed
+            }
+            return [parsed]
+        }
+    }
+    catch (err) {
+        logger.error('Failed to deserialise queue payload', { ex: winston.exceptions.getAllInfo(err), queueItem });
+        throw new Error(`Failed to deserialise queue payload`);
+    }
 }
 
 export const triggerBackgroundJobWithQueue = async (queueClient: QueueClient, context: BackgroundJobRequestContext, logger: Logger): Promise<void> => {
@@ -51,4 +72,25 @@ export const triggerBackgroundJobWithQueue = async (queueClient: QueueClient, co
     else {
         logger.debug("Background job triggered", { body, messageId: response.messageId, context })
     }
+}
+
+export const sendQueueMessageToPoisonQueue = async (queueName: string, context: BackgroundJobRequestContext, logger: Logger): Promise<void> => {
+    const body = {
+        ...context
+    }
+    const bodyStr = serialiseQueuePayload(body);
+
+    // https://learn.microsoft.com/en-us/azure/azure-functions/functions-bindings-storage-queue-trigger?tabs=python-v2%2Cisolated-process%2Cnodejs-v4%2Cextensionv5&pivots=programming-language-typescript#poison-messages
+    const queueClient = new QueueClient(process.env.AzureWebJobsStorage as string, `${queueName}-poison`);
+
+    const response = await queueClient.sendMessage(bodyStr, { messageTimeToLive: -1 });
+
+    if (response.errorCode) {
+        const { requestId, date, errorCode } = response;
+        logger.error("Failed to push to poison queue", { body, requestId, date, errorCode })
+        throw new Error(`Failed to push to poison queue: ${response.errorCode}`);
+    }
+    else {
+        logger.verbose("Message pushed to poison queue", { body, messageId: response.messageId, context })
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@seeka-labs/cli-apps",
-  "version": "1.1.17",
+  "version": "1.1.18",
   "description": "Seeka - Apps CLI",
   "author": "SEEKA <platform@seeka.co>",
   "license": "MIT",