@directus/api 31.0.0 → 32.0.0

package/dist/services/import-export.js

@@ -1,5 +1,5 @@
 import { useEnv } from '@directus/env';
-import { ForbiddenError, InvalidPayloadError, ServiceUnavailableError, UnsupportedMediaTypeError, } from '@directus/errors';
+import { createError, ErrorCode, ForbiddenError, InvalidPayloadError, ServiceUnavailableError, UnsupportedMediaTypeError, } from '@directus/errors';
 import { isSystemCollection } from '@directus/system-data';
 import { parseJSON, toArray } from '@directus/utils';
 import { createTmpFile } from '@directus/utils/node';
@@ -28,6 +28,126 @@ import { parseFields } from '../database/get-ast-from-query/lib/parse-fields.js'
 import { set } from 'lodash-es';
 const env = useEnv();
 const logger = useLogger();
+const MAX_IMPORT_ERRORS = env['MAX_IMPORT_ERRORS'];
+export function createErrorTracker() {
+    let genericError;
+    // For errors with field / type (joi validation or DB with field)
+    const fieldErrors = new Map();
+    let capturedErrorCount = 0;
+    let isLimitReached = false;
+    function convertToRanges(rows, minRangeSize = 4) {
+        const sorted = Array.from(new Set(rows)).sort((a, b) => a - b);
+        const result = [];
+        if (sorted.length === 0)
+            return [];
+        let start = sorted[0];
+        let prev = sorted[0];
+        let count = 1;
+        const nonConsecutive = [];
+        const flush = () => {
+            if (count >= minRangeSize) {
+                result.push({ type: 'range', start, end: prev });
+            }
+            else {
+                for (let i = start; i <= prev; i++) {
+                    nonConsecutive.push(i);
+                }
+            }
+        };
+        for (let i = 1; i < sorted.length; i++) {
+            const current = sorted[i];
+            if (current === prev + 1) {
+                prev = current;
+                count++;
+            }
+            else {
+                flush();
+                start = prev = current;
+                count = 1;
+            }
+        }
+        flush();
+        // Add non-consecutive rows as a single "lines" entry
+        if (nonConsecutive.length > 0) {
+            result.push({ type: 'lines', rows: nonConsecutive });
+        }
+        return result;
+    }
+    function addCapturedError(err, rowNumber) {
+        const field = err.extensions?.field;
+        if (field) {
+            const type = err.extensions?.type;
+            const substring = err.extensions?.substring;
+            const valid = err.extensions?.valid;
+            const invalid = err.extensions?.invalid;
+            let key = type ? `${field}|${type}` : field;
+            if (substring !== undefined)
+                key += `|substring:${substring}`;
+            if (valid !== undefined)
+                key += `|valid:${JSON.stringify(valid)}`;
+            if (invalid !== undefined)
+                key += `|invalid:${JSON.stringify(invalid)}`;
+            if (!fieldErrors.has(err.code)) {
+                fieldErrors.set(err.code, new Map());
+            }
+            const errorsByCode = fieldErrors.get(err.code);
+            if (!errorsByCode.has(key)) {
+                errorsByCode.set(key, {
+                    message: err.message,
+                    rowNumbers: [],
+                });
+            }
+            errorsByCode.get(key).rowNumbers.push(rowNumber);
+        }
+        else {
+            genericError = err;
+        }
+        capturedErrorCount++;
+        if (capturedErrorCount >= MAX_IMPORT_ERRORS) {
+            isLimitReached = true;
+        }
+    }
+    function hasGenericError() {
+        return genericError !== undefined;
+    }
+    function buildFinalErrors() {
+        if (genericError) {
+            return [genericError];
+        }
+        return Array.from(fieldErrors.entries()).flatMap(([code, fieldMap]) => Array.from(fieldMap.entries()).map(([compositeKey, errorData]) => {
+            const parts = compositeKey.split('|');
+            const field = parts[0];
+            const type = parts[1];
+            const extensions = {};
+            for (let i = 2; i < parts.length; i++) {
+                const [paramType, paramValue] = parts[i]?.split(':', 2) ?? [];
+                if (!paramType || paramValue === undefined)
+                    continue;
+                try {
+                    extensions[paramType] = JSON.parse(paramValue);
+                }
+                catch {
+                    extensions[paramType] = paramValue;
+                }
+            }
+            const ErrorClass = createError(code, errorData.message, 400);
+            return new ErrorClass({
+                field,
+                type,
+                ...extensions,
+                rows: convertToRanges(errorData.rowNumbers),
+            });
+        }));
+    }
+    return {
+        addCapturedError,
+        buildFinalErrors,
+        getCount: () => capturedErrorCount,
+        hasErrors: () => capturedErrorCount > 0 || hasGenericError(),
+        shouldStop: () => isLimitReached || hasGenericError(),
+        hasGenericError,
+    };
+}
 export class ImportService {
     knex;
     accountability;
@@ -71,37 +191,72 @@ export class ImportService {
     async importJSON(collection, stream) {
         const extractJSON = StreamArray.withParser();
         const nestedActionEvents = [];
-        return transaction(this.knex, (trx) => {
+        const errorTracker = createErrorTracker();
+        return transaction(this.knex, async (trx) => {
             const service = getService(collection, {
                 knex: trx,
                 schema: this.schema,
                 accountability: this.accountability,
             });
-            const saveQueue = queue(async (value) => {
-                return await service.upsertOne(value, { bypassEmitAction: (params) => nestedActionEvents.push(params) });
-            });
-            return new Promise((resolve, reject) => {
-                stream.pipe(extractJSON);
-                extractJSON.on('data', ({ value }) => {
-                    saveQueue.push(value);
-                });
-                extractJSON.on('error', (err) => {
-                    destroyStream(stream);
-                    destroyStream(extractJSON);
-                    reject(new InvalidPayloadError({ reason: err.message }));
-                });
-                saveQueue.error((err) => {
-                    reject(err);
-                });
-                extractJSON.on('end', () => {
-                    saveQueue.drain(() => {
-                        for (const nestedActionEvent of nestedActionEvents) {
-                            emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
+            try {
+                await new Promise((resolve, reject) => {
+                    let rowNumber = 1;
+                    const saveQueue = queue(async (task) => {
+                        if (errorTracker.shouldStop())
+                            return;
+                        try {
+                            const result = await service.upsertOne(task.data, {
+                                bypassEmitAction: (params) => nestedActionEvents.push(params),
+                            });
+                            return result;
+                        }
+                        catch (error) {
+                            for (const err of toArray(error)) {
+                                errorTracker.addCapturedError(err, task.rowNumber);
+                                if (errorTracker.shouldStop()) {
+                                    break;
+                                }
+                            }
+                            if (errorTracker.shouldStop()) {
+                                saveQueue.kill();
+                                destroyStream(stream);
+                                destroyStream(extractJSON);
+                                reject();
+                            }
+                            return;
                         }
-                        return resolve();
+                    });
+                    stream.pipe(extractJSON);
+                    extractJSON.on('data', ({ value }) => {
+                        saveQueue.push({ data: value, rowNumber: rowNumber++ });
+                    });
+                    extractJSON.on('error', (err) => {
+                        destroyStream(stream);
+                        destroyStream(extractJSON);
+                        reject(new InvalidPayloadError({ reason: err.message }));
+                    });
+                    extractJSON.on('end', () => {
+                        // In case of empty JSON file
+                        if (!saveQueue.started)
+                            return resolve();
+                        saveQueue.drain(() => {
+                            if (errorTracker.hasErrors()) {
+                                return reject();
+                            }
+                            for (const nestedActionEvent of nestedActionEvents) {
+                                emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
+                            }
+                            return resolve();
+                        });
                     });
                 });
-            });
+            }
+            catch (error) {
+                if (!error && errorTracker.hasErrors()) {
+                    throw errorTracker.buildFinalErrors();
+                }
+                throw error;
+            }
         });
     }
     async importCSV(collection, stream) {
@@ -109,98 +264,129 @@ export class ImportService {
         if (!tmpFile)
             throw new Error('Failed to create temporary file for import');
         const nestedActionEvents = [];
-        return transaction(this.knex, (trx) => {
+        const errorTracker = createErrorTracker();
+        return transaction(this.knex, async (trx) => {
             const service = getService(collection, {
                 knex: trx,
                 schema: this.schema,
                 accountability: this.accountability,
             });
-            const saveQueue = queue(async (value) => {
-                return await service.upsertOne(value, { bypassEmitAction: (action) => nestedActionEvents.push(action) });
-            });
-            const transform = (value) => {
-                if (value.length === 0)
-                    return;
-                try {
-                    const parsedJson = parseJSON(value);
-                    if (typeof parsedJson === 'number') {
-                        return value;
-                    }
-                    return parsedJson;
-                }
-                catch {
-                    return value;
-                }
-            };
-            const PapaOptions = {
-                header: true,
-                // Trim whitespaces in headers, including the byte order mark (BOM) zero-width no-break space
-                transformHeader: (header) => header.trim(),
-                transform,
-            };
-            return new Promise((resolve, reject) => {
-                const streams = [stream];
-                const cleanup = (destroy = true) => {
-                    if (destroy) {
-                        for (const stream of streams) {
-                            destroyStream(stream);
+            try {
+                await new Promise((resolve, reject) => {
+                    const streams = [stream];
+                    let rowNumber = 0;
+                    const cleanup = (destroy = true) => {
+                        if (destroy) {
+                            for (const stream of streams) {
+                                destroyStream(stream);
+                            }
                         }
-                    }
-                    tmpFile.cleanup().catch(() => {
-                        logger.warn(`Failed to cleanup temporary import file (${tmpFile.path})`);
-                    });
-                };
-                saveQueue.error((error) => {
-                    reject(error);
-                });
-                const fileWriteStream = createWriteStream(tmpFile.path)
-                    .on('error', (error) => {
-                    cleanup();
-                    reject(new Error('Error while writing import data to temporary file', { cause: error }));
-                })
-                    .on('finish', () => {
-                    const fileReadStream = createReadStream(tmpFile.path).on('error', (error) => {
-                        cleanup();
-                        reject(new Error('Error while reading import data from temporary file', { cause: error }));
-                    });
-                    streams.push(fileReadStream);
-                    fileReadStream
-                        .pipe(Papa.parse(Papa.NODE_STREAM_INPUT, PapaOptions))
-                        .on('data', (obj) => {
-                        const result = {};
-                        // Filter out all undefined fields
-                        for (const field in obj) {
-                            if (obj[field] !== undefined) {
-                                set(result, field, obj[field]);
+                        tmpFile.cleanup().catch(() => {
+                            logger.warn(`Failed to cleanup temporary import file (${tmpFile.path})`);
+                        });
+                    };
+                    const saveQueue = queue(async (task) => {
+                        if (errorTracker.shouldStop())
+                            return;
+                        try {
+                            const result = await service.upsertOne(task.data, {
+                                bypassEmitAction: (action) => nestedActionEvents.push(action),
+                            });
+                            return result;
+                        }
+                        catch (error) {
+                            for (const err of toArray(error)) {
+                                errorTracker.addCapturedError(err, task.rowNumber);
+                                if (errorTracker.shouldStop()) {
+                                    break;
+                                }
+                            }
+                            if (errorTracker.shouldStop()) {
+                                saveQueue.kill();
+                                cleanup(true);
+                                reject();
                             }
+                            return;
                         }
-                        saveQueue.push(result);
-                    })
+                    });
+                    const fileWriteStream = createWriteStream(tmpFile.path)
                         .on('error', (error) => {
                         cleanup();
-                        reject(new InvalidPayloadError({ reason: error.message }));
+                        reject(new Error('Error while writing import data to temporary file', { cause: error }));
                     })
-                        .on('end', () => {
-                        cleanup(false);
-                        // In case of empty CSV file
-                        if (!saveQueue.started)
-                            return resolve();
-                        saveQueue.drain(() => {
-                            for (const nestedActionEvent of nestedActionEvents) {
-                                emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
+                        .on('finish', () => {
+                        const fileReadStream = createReadStream(tmpFile.path).on('error', (error) => {
+                            cleanup();
+                            reject(new Error('Error while reading import data from temporary file', { cause: error }));
+                        });
+                        streams.push(fileReadStream);
+                        fileReadStream
+                            .pipe(Papa.parse(Papa.NODE_STREAM_INPUT, {
+                            header: true,
+                            transformHeader: (header) => header.trim(),
+                            transform: (value) => {
+                                if (value.length === 0)
+                                    return;
+                                try {
+                                    const parsedJson = parseJSON(value);
+                                    if (typeof parsedJson === 'number') {
+                                        return value;
+                                    }
+                                    return parsedJson;
+                                }
+                                catch {
+                                    return value;
+                                }
+                            },
+                        }))
+                            .on('data', (obj) => {
+                            rowNumber++;
+                            const result = {};
+                            for (const field in obj) {
+                                if (obj[field] !== undefined) {
+                                    set(result, field, obj[field]);
+                                }
                             }
-                            return resolve();
+                            saveQueue.push({ data: result, rowNumber });
+                        })
+                            .on('error', (error) => {
+                            cleanup();
+                            reject(new InvalidPayloadError({ reason: error.message }));
+                        })
+                            .on('end', () => {
+                            // In case of empty CSV file
+                            if (!saveQueue.started) {
+                                cleanup(false);
+                                return resolve();
+                            }
+                            saveQueue.drain(() => {
+                                if (!errorTracker.shouldStop())
+                                    cleanup(false);
+                                if (errorTracker.hasErrors()) {
+                                    return reject();
+                                }
+                                for (const nestedActionEvent of nestedActionEvents) {
+                                    emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
+                                }
+                                return resolve();
+                            });
                         });
                     });
+                    streams.push(fileWriteStream);
+                    stream
+                        .on('error', (error) => {
+                        cleanup();
+                        reject(new Error('Error while retrieving import data', { cause: error }));
+                    })
+                        .pipe(fileWriteStream);
                 });
-                streams.push(fileWriteStream);
-                stream
-                    .on('error', (error) => {
-                    cleanup();
-                    reject(new Error('Error while retrieving import data', { cause: error }));
-                })
-                    .pipe(fileWriteStream);
-            });
+            }
+            catch (error) {
+                if (!error && errorTracker.hasErrors()) {
+                    throw errorTracker.buildFinalErrors();
+                }
+                throw error;
+            }
         });
     }
 }

package/dist/services/items.d.ts

@@ -1,4 +1,4 @@
-import type { AbstractService, AbstractServiceOptions, Accountability, Item as AnyItem, MutationTracker, MutationOptions, PrimaryKey, Query, QueryOptions, SchemaOverview } from '@directus/types';
+import type { AbstractService, AbstractServiceOptions, Accountability, Item as AnyItem, MutationOptions, MutationTracker, PrimaryKey, Query, QueryOptions, SchemaOverview } from '@directus/types';
 import type Keyv from 'keyv';
 import type { Knex } from 'knex';
 export declare class ItemsService<Item extends AnyItem = AnyItem, Collection extends string = string> implements AbstractService<Item> {

package/dist/services/items.js

@@ -8,7 +8,7 @@ import { getCache } from '../cache.js';
 import { translateDatabaseError } from '../database/errors/translate.js';
 import { getAstFromQuery } from '../database/get-ast-from-query/get-ast-from-query.js';
 import { getHelpers } from '../database/helpers/index.js';
-import getDatabase from '../database/index.js';
+import getDatabase, { getDatabaseClient } from '../database/index.js';
 import { runAst } from '../database/run-ast/run-ast.js';
 import emitter from '../emitter.js';
 import { processAst } from '../permissions/modules/process-ast/process-ast.js';
@@ -173,10 +173,15 @@ export class ItemsService {
                 autoIncrementSequenceNeedsToBeReset = true;
             }
             try {
+                let returningOptions = undefined;
+                // Support MSSQL tables that have triggers.
+                if (getDatabaseClient(trx) === 'mssql') {
+                    returningOptions = { includeTriggerModifications: true };
+                }
                 const result = await trx
                     .insert(payloadWithoutAliases)
                     .into(this.collection)
-                    .returning(primaryKeyField)
+                    .returning(primaryKeyField, returningOptions)
                     .then((result) => result[0]);
                 const returnedKey = typeof result === 'object' ? result[primaryKeyField] : result;
                 if (pkField.type === 'uuid') {
@@ -436,7 +441,7 @@ export class ItemsService {
         const filterWithKey = assign({}, query.filter, { [primaryKeyField]: { _eq: key } });
         const queryWithKey = assign({}, query, { filter: filterWithKey });
         let results = [];
-        if (query.version) {
+        if (query.version && query.version !== 'main') {
             results = [await handleVersion(this, key, queryWithKey, opts)];
         }
         else {
@@ -806,12 +811,23 @@ export class ItemsService {
         }
         const primaryKeyField = this.schema.collections[this.collection].primary;
         validateKeys(this.schema, this.collection, primaryKeyField, keys);
+        const keysAfterHooks = opts.emitEvents !== false
+            ? await emitter.emitFilter(this.eventScope === 'items'
+                ? ['items.delete', `${this.collection}.items.delete`]
+                : `${this.eventScope}.delete`, keys, {
+                collection: this.collection,
+            }, {
+                database: this.knex,
+                schema: this.schema,
+                accountability: this.accountability,
+            })
+            : keys;
         if (this.accountability) {
             await validateAccess({
                 accountability: this.accountability,
                 action: 'delete',
                 collection: this.collection,
-                primaryKeys: keys,
+                primaryKeys: keysAfterHooks,
             }, {
                 knex: this.knex,
                 schema: this.schema,
@@ -820,17 +836,8 @@ export class ItemsService {
         if (opts.preMutationError) {
             throw opts.preMutationError;
         }
-        if (opts.emitEvents !== false) {
-            await emitter.emitFilter(this.eventScope === 'items' ? ['items.delete', `${this.collection}.items.delete`] : `${this.eventScope}.delete`, keys, {
-                collection: this.collection,
-            }, {
-                database: this.knex,
-                schema: this.schema,
-                accountability: this.accountability,
-            });
-        }
         await transaction(this.knex, async (trx) => {
-            await trx(this.collection).whereIn(primaryKeyField, keys).delete();
+            await trx(this.collection).whereIn(primaryKeyField, keysAfterHooks).delete();
             if (opts.userIntegrityCheckFlags) {
                 if (opts.onRequireUserIntegrityCheck) {
                     opts.onRequireUserIntegrityCheck(opts.userIntegrityCheckFlags);
@@ -847,7 +854,7 @@ export class ItemsService {
                     knex: trx,
                     schema: this.schema,
                 });
-                await activityService.createMany(keys.map((key) => ({
+                await activityService.createMany(keysAfterHooks.map((key) => ({
                     action: Action.DELETE,
                     user: this.accountability.user,
                     collection: this.collection,
@@ -867,8 +874,8 @@ export class ItemsService {
                     ? ['items.delete', `${this.collection}.items.delete`]
                     : `${this.eventScope}.delete`,
                 meta: {
-                    payload: keys,
-                    keys: keys,
+                    payload: keysAfterHooks,
+                    keys: keysAfterHooks,
                     collection: this.collection,
                 },
                 context: {
@@ -884,7 +891,7 @@ export class ItemsService {
                 emitter.emitAction(actionEvent.event, actionEvent.meta, actionEvent.context);
             }
         }
-        return keys;
+        return keysAfterHooks;
     }
     /**
      * Read/treat collection as singleton.
@@ -892,8 +899,17 @@ export class ItemsService {
     async readSingleton(query, opts) {
         query = clone(query);
         query.limit = 1;
-        const records = await this.readByQuery(query, opts);
-        const record = records[0];
+        let record;
+        if (query.version && query.version !== 'main') {
+            const primaryKeyField = this.schema.collections[this.collection].primary;
+            const key = (await this.knex.select(primaryKeyField).from(this.collection).first())?.[primaryKeyField];
+            if (key) {
+                record = await handleVersion(this, key, query, opts);
+            }
+        }
+        else {
+            record = (await this.readByQuery(query, opts))[0];
+        }
         if (!record) {
             let fields = Object.entries(this.schema.collections[this.collection].fields);
             const defaults = {};

package/dist/services/mail/index.js

@@ -10,6 +10,7 @@ import emitter from '../../emitter.js';
 import { useLogger } from '../../logger/index.js';
 import getMailer from '../../mailer.js';
 import { Url } from '../../utils/url.js';
+import { useEmailRateLimiterQueue } from './rate-limiter.js';
 const env = useEnv();
 const logger = useLogger();
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -37,6 +38,7 @@ export class MailService {
         }
     }
     async send(options) {
+        await useEmailRateLimiterQueue();
         const payload = await emitter.emitFilter(`email.send`, options, {});
         if (!payload)
             return null;

package/dist/services/mail/rate-limiter.d.ts

@@ -0,0 +1 @@
+export declare function useEmailRateLimiterQueue(): Promise<void>;

package/dist/services/mail/rate-limiter.js

@@ -0,0 +1,29 @@
+import { useEnv } from '@directus/env';
+import { RateLimiterQueue } from 'rate-limiter-flexible';
+import { createRateLimiter } from '../../rate-limiter.js';
+import { toBoolean } from '@directus/utils';
+import { EmailLimitExceededError } from '@directus/errors';
+let emailRateLimiterQueue;
+const env = useEnv();
+if (toBoolean(env['RATE_LIMITER_EMAIL_ENABLED']) === true) {
+    emailRateLimiterQueue = new RateLimiterQueue(createRateLimiter('RATE_LIMITER_EMAIL'), {
+        maxQueueSize: Number(env['RATE_LIMITER_EMAIL_QUEUE_SIZE']),
+    });
+}
+export async function useEmailRateLimiterQueue() {
+    if (!emailRateLimiterQueue)
+        return;
+    try {
+        await emailRateLimiterQueue.removeTokens(1);
+    }
+    catch (err) {
+        if (err instanceof Error) {
+            throw new EmailLimitExceededError({
+                points: 'RATE_LIMITER_EMAIL_POINTS' in env ? Number(env['RATE_LIMITER_EMAIL_POINTS']) : undefined,
+                duration: 'RATE_LIMITER_EMAIL_DURATION' in env ? Number(env['RATE_LIMITER_EMAIL_DURATION']) : undefined,
+                message: 'RATE_LIMITER_EMAIL_ERROR_MESSAGE' in env ? String(env['RATE_LIMITER_EMAIL_ERROR_MESSAGE']) : undefined,
+            });
+        }
+        throw err;
+    }
+}