@stamhoofd/backend 2.17.2 → 2.17.4

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stamhoofd/backend",
-    "version": "2.17.2",
+    "version": "2.17.4",
     "main": "./dist/index.js",
     "exports": {
         ".": {
@@ -39,10 +39,10 @@
         "@stamhoofd/backend-i18n": "^2.17.0",
         "@stamhoofd/backend-middleware": "^2.17.0",
         "@stamhoofd/email": "^2.17.0",
-        "@stamhoofd/models": "^2.17.0",
-        "@stamhoofd/queues": "^2.17.0",
-        "@stamhoofd/sql": "^2.17.0",
-        "@stamhoofd/structures": "^2.17.0",
+        "@stamhoofd/models": "^2.17.3",
+        "@stamhoofd/queues": "^2.17.3",
+        "@stamhoofd/sql": "^2.17.3",
+        "@stamhoofd/structures": "^2.17.3",
         "@stamhoofd/utility": "^2.17.0",
         "archiver": "^7.0.1",
         "aws-sdk": "^2.885.0",
@@ -60,5 +60,5 @@
         "postmark": "4.0.2",
         "stripe": "^16.6.0"
     },
-    "gitHead": "d43bd511d80deca4f467cd48fecf86a711c04565"
+    "gitHead": "56f46d516a4a18cffb011caf177b5de9a14607b6"
 }

package/src/endpoints/global/files/ExportToExcelEndpoint.ts

@@ -10,6 +10,7 @@ import { sleep } from "@stamhoofd/utility";
 import { Context } from '../../../helpers/Context';
 import { fetchToAsyncIterator } from '../../../helpers/fetchToAsyncIterator';
 import { FileCache } from '../../../helpers/FileCache';
+import { QueueHandler } from '@stamhoofd/queues';
 
 type Params = { type: string };
 type Query = undefined;
@@ -62,6 +63,15 @@ export class ExportToExcelEndpoint extends Endpoint<Params, Query, Body, Respons
             })
         }
 
+        if (QueueHandler.isRunning('user-export-to-excel-' + user.id)) {
+            throw new SimpleError({
+                code: "not_allowed",
+                message: "Export is pending",
+                human: 'Je hebt momenteel al een Excel export lopen. Wacht tot die klaar is voor je een nieuwe export start.',
+                statusCode: 403
+            })
+        }
+
         const loader = ExportToExcelEndpoint.loaders.get(request.params.type as ExcelExportType);
         
         if (!loader) {
@@ -137,27 +147,33 @@ export class ExportToExcelEndpoint extends Endpoint<Params, Query, Body, Respons
     }
 
     async job(loader: ExcelExporter<EncodableObject>, request: ExcelExportRequest, type: string): Promise<string> {
-        // Estimate how long it will take.
-        // If too long, we'll schedule it and write it to Digitalocean Spaces
-        // Otherwise we'll just return the file directly
-        const {file, stream} = await FileCache.getWriteStream('.xlsx');
-
-        const zipWriterAdapter = new ArchiverWriterAdapter(stream);
-        const writer = new XlsxWriter(zipWriterAdapter);
-
-        // Limit to pages of 100
-        request.filter.limit = 100;
-
-        await exportToExcel({
-            definitions: loader.sheets,
-            writer,
-            dataGenerator: fetchToAsyncIterator(request.filter, loader),
-            filter: request.workbookFilter
-        })
-
-        console.log('Done writing excel file')
-
-        const url = 'https://'+ STAMHOOFD.domains.api + '/v'+ Version +'/file-cache?file=' + encodeURIComponent(file) + '&name=' + encodeURIComponent(type)
-        return url;
+        // Only run 1 export per user at the same time
+        return await QueueHandler.schedule('user-export-to-excel-' + Context.user!.id, async () => {
+            // Allow maximum 2 running Excel jobs at the same time for all users
+            return await QueueHandler.schedule('export-to-excel', async () => {
+                // Estimate how long it will take.
+                // If too long, we'll schedule it and write it to Digitalocean Spaces
+                // Otherwise we'll just return the file directly
+                const {file, stream} = await FileCache.getWriteStream('.xlsx');
+
+                const zipWriterAdapter = new ArchiverWriterAdapter(stream);
+                const writer = new XlsxWriter(zipWriterAdapter);
+
+                // Limit to pages of 100
+                request.filter.limit = STAMHOOFD.environment === 'development' ? 1 : 100; // in development, we need to check if total count matches and pagination is working correctly
+
+                await exportToExcel({
+                    definitions: loader.sheets,
+                    writer,
+                    dataGenerator: fetchToAsyncIterator(request.filter, loader),
+                    filter: request.workbookFilter
+                })
+
+                console.log('Done writing excel file')
+
+                const url = 'https://'+ STAMHOOFD.domains.api + '/v'+ Version +'/file-cache?file=' + encodeURIComponent(file) + '&name=' + encodeURIComponent(type)
+                return url;
+            }, 2)
+        });
     }
 }

package/src/sql-filters/members.ts

@@ -9,6 +9,8 @@ import { registrationFilterCompilers } from "./registrations";
 export const memberFilterCompilers: SQLFilterDefinitions = {
     ...baseSQLFilterCompilers,
     id: createSQLColumnFilterCompiler('id'),
+    firstName: createSQLColumnFilterCompiler('firstName'),
+    lastName: createSQLColumnFilterCompiler('lastName'),
     name: createSQLExpressionFilterCompiler(
         new SQLConcat(
             SQL.column('firstName'),

package/src/sql-sorters/members.ts

@@ -3,6 +3,14 @@ import { SQL, SQLOrderBy, SQLOrderByDirection, SQLSortDefinitions } from "@stamh
 import { Formatter } from "@stamhoofd/utility"
 
 export const memberSorters: SQLSortDefinitions<MemberWithRegistrations> = {
+    // WARNING! TEST NEW SORTERS THOROUGHLY!
+    // Try to avoid creating sorters on fields that er not 1:1 with the database, that often causes pagination issues if not thought through
+    // An example: sorting on 'name' is not a good idea, because it is a concatenation of two fields. 
+    // You might be tempted to use ORDER BY firstName, lastName, but that will not work as expected and it needs to be ORDER BY CONCAT(firstName, ' ', lastName)
+    // Why? Because ORDER BY firstName, lastName produces a different order dan ORDER BY CONCAT(firstName, ' ', lastName) if there are multiple people with spaces in the first name
+    // And that again causes issues with pagination because the next query will append a filter of name > 'John Doe' - causing duplicate and/or skipped results
+    // What if you need mapping? simply map the sorters in the frontend: name -> firstname, lastname, age -> birthDay, etc.
+
     'id': {
         getValue(a) {
             return a.id
@@ -14,21 +22,26 @@ export const memberSorters: SQLSortDefinitions<MemberWithRegistrations> = {
             })
         }
     },
-    'name': {
+    'firstName': {
+        getValue(a) {
+            return a.firstName
+        },
+        toSQL: (direction: SQLOrderByDirection): SQLOrderBy => {
+            return new SQLOrderBy({
+                column: SQL.column('firstName'),
+                direction
+            })
+        }
+    },
+    'lastName': {
         getValue(a) {
-            return a.details.name
+            return a.lastName
         },
         toSQL: (direction: SQLOrderByDirection): SQLOrderBy => {
-            return SQLOrderBy.combine([
-                new SQLOrderBy({
-                    column: SQL.column('firstName'),
-                    direction
-                }),
-                new SQLOrderBy({
-                    column: SQL.column('lastName'),
-                    direction
-                })
-            ])
+            return new SQLOrderBy({
+                column: SQL.column('lastName'),
+                direction
+            })
         }
     },
     'birthDay': {
@@ -42,5 +55,4 @@ export const memberSorters: SQLSortDefinitions<MemberWithRegistrations> = {
             })
         }
     }
-    // Note: never add mapped sortings, that should happen in the frontend -> e.g. map age to birthDay
 }

package/src/sql-sorters/organizations.ts

@@ -2,6 +2,14 @@ import { Organization } from "@stamhoofd/models"
 import { SQL, SQLOrderBy, SQLOrderByDirection, SQLSortDefinitions } from "@stamhoofd/sql"
 
 export const organizationSorters: SQLSortDefinitions<Organization> = {
+    // WARNING! TEST NEW SORTERS THOROUGHLY!
+    // Try to avoid creating sorters on fields that er not 1:1 with the database, that often causes pagination issues if not thought through
+    // An example: sorting on 'name' is not a good idea, because it is a concatenation of two fields. 
+    // You might be tempted to use ORDER BY firstName, lastName, but that will not work as expected and it needs to be ORDER BY CONCAT(firstName, ' ', lastName)
+    // Why? Because ORDER BY firstName, lastName produces a different order dan ORDER BY CONCAT(firstName, ' ', lastName) if there are multiple people with spaces in the first name
+    // And that again causes issues with pagination because the next query will append a filter of name > 'John Doe' - causing duplicate and/or skipped results
+    // What if you need mapping? simply map the sorters in the frontend: name -> firstname, lastname, age -> birthDay, etc.
+    
     'id': {
         getValue(a) {
             return a.id

package/src/sql-sorters/payments.ts

@@ -3,6 +3,14 @@ import { SQL, SQLOrderBy, SQLOrderByDirection, SQLSortDefinitions } from "@stamh
 import { Formatter } from "@stamhoofd/utility"
 
 export const paymentSorters: SQLSortDefinitions<Payment> = {
+    // WARNING! TEST NEW SORTERS THOROUGHLY!
+    // Try to avoid creating sorters on fields that er not 1:1 with the database, that often causes pagination issues if not thought through
+    // An example: sorting on 'name' is not a good idea, because it is a concatenation of two fields. 
+    // You might be tempted to use ORDER BY firstName, lastName, but that will not work as expected and it needs to be ORDER BY CONCAT(firstName, ' ', lastName)
+    // Why? Because ORDER BY firstName, lastName produces a different order dan ORDER BY CONCAT(firstName, ' ', lastName) if there are multiple people with spaces in the first name
+    // And that again causes issues with pagination because the next query will append a filter of name > 'John Doe' - causing duplicate and/or skipped results
+    // What if you need mapping? simply map the sorters in the frontend: name -> firstname, lastname, age -> birthDay, etc.
+
     'id': {
         getValue(a) {
             return a.id