@directus/api 24.0.0 → 25.0.0

package/dist/metrics/lib/create-metrics.js

@@ -0,0 +1,239 @@
+import { useEnv } from '@directus/env';
+import { toArray } from '@directus/utils';
+import { randomUUID } from 'node:crypto';
+import { Readable } from 'node:stream';
+import { promisify } from 'node:util';
+import pm2 from 'pm2';
+import { AggregatorRegistry, Counter, Histogram, register } from 'prom-client';
+import { getCache } from '../../cache.js';
+import { hasDatabaseConnection } from '../../database/index.js';
+import { redisConfigAvailable, useRedis } from '../../redis/index.js';
+import { getStorage } from '../../storage/index.js';
+const isPM2 = 'PM2_HOME' in process.env;
+const METRICS_SYNC_PACKET = 'directus:metrics---data-sync';
+const listApps = promisify(pm2.list.bind(pm2));
+const sendDataToProcessId = promisify(pm2.sendDataToProcessId.bind(pm2));
+export function createMetrics() {
+    const env = useEnv();
+    const services = env['METRICS_SERVICES'] ?? [];
+    const aggregates = new Map();
+    /**
+     * Listen for PM2 metric data sync messages and add them to the aggregate
+     */
+    if (isPM2) {
+        process.on('message', (packet) => {
+            if (!packet.data || packet.topic !== METRICS_SYNC_PACKET)
+                return;
+            aggregate(packet.data);
+        });
+    }
+    async function generate() {
+        const checkId = randomUUID();
+        await Promise.all([
+            trackDatabaseMetric(),
+            trackCacheMetric(checkId),
+            trackRedisMetric(checkId),
+            trackStorageMetric(checkId),
+        ]);
+        /**
+         * Push generated metrics to all pm2 instances
+         */
+        if (isPM2) {
+            try {
+                const apps = await listApps();
+                const data = await register.getMetricsAsJSON();
+                const syncs = [];
+                for (const app of apps) {
+                    if (app.pm_id === undefined || app.pid === 0 || app.name !== 'directus') {
+                        continue;
+                    }
+                    syncs.push(sendDataToProcessId(app.pm_id, {
+                        data: { pid: process.pid, metrics: data },
+                        topic: METRICS_SYNC_PACKET,
+                    }));
+                }
+                await Promise.allSettled(syncs);
+            }
+            catch {
+                // ignore
+            }
+        }
+    }
+    /**
+     * Add PM2 synced metric to the aggregate store.
+     * Subsequent syncs for the given instance will override previous value.
+     */
+    async function aggregate(data) {
+        aggregates.set(data.pid, data.metrics);
+    }
+    async function readAll() {
+        /**
+         * In a PM2 context we must aggregate the metrics across instances ensuring
+         * only currently active instances are added to the aggregate
+         */
+        if (isPM2 && aggregates.size !== 0) {
+            const apps = await listApps();
+            const aggregate = [];
+            for (const app of apps) {
+                if (aggregates.has(app.pid)) {
+                    aggregate.push(aggregates.get(app.pid));
+                }
+            }
+            if (aggregate.length !== 0) {
+                return AggregatorRegistry.aggregate(aggregate).metrics();
+            }
+        }
+        return register.metrics();
+    }
+    function getDatabaseErrorMetric() {
+        if (services.includes('database') === false) {
+            return null;
+        }
+        const client = env['DB_CLIENT'];
+        let metric = register.getSingleMetric(`directus_db_${client}_connection_errors`);
+        if (!metric) {
+            metric = new Counter({
+                name: `directus_db_${client}_connection_errors`,
+                help: `${client} Database connection error count`,
+            });
+        }
+        return metric;
+    }
+    function getDatabaseResponseMetric() {
+        if (services.includes('database') === false) {
+            return null;
+        }
+        const client = env['DB_CLIENT'];
+        let metric = register.getSingleMetric(`directus_db_${client}_response_time_ms`);
+        if (!metric) {
+            metric = new Histogram({
+                name: `directus_db_${client}_response_time_ms`,
+                help: `${client} Database connection response time`,
+                buckets: [1, 10, 20, 40, 60, 80, 100, 200, 500, 750, 1000],
+            });
+        }
+        return metric;
+    }
+    function getCacheErrorMetric() {
+        if (services.includes('cache') === false || env['CACHE_ENABLED'] !== true) {
+            return null;
+        }
+        if (env['CACHE_STORE'] === 'redis' && redisConfigAvailable() !== true) {
+            return null;
+        }
+        let metric = register.getSingleMetric(`directus_cache_${env['CACHE_STORE']}_connection_errors`);
+        if (!metric) {
+            metric = new Counter({
+                name: `directus_cache_${env['CACHE_STORE']}_connection_errors`,
+                help: 'Cache connection error count',
+            });
+        }
+        return metric;
+    }
+    function getRedisErrorMetric() {
+        if (services.includes('redis') === false || redisConfigAvailable() !== true) {
+            return null;
+        }
+        let metric = register.getSingleMetric('directus_redis_connection_errors');
+        if (!metric) {
+            metric = new Counter({
+                name: `directus_redis_connection_errors`,
+                help: 'Redis connection error count',
+            });
+        }
+        return metric;
+    }
+    function getStorageErrorMetric(location) {
+        if (services.includes('storage') === false) {
+            return null;
+        }
+        let metric = register.getSingleMetric(`directus_storage_${location}_connection_errors`);
+        if (!metric) {
+            metric = new Counter({
+                name: `directus_storage_${location}_connection_errors`,
+                help: `${location} storage connection error count`,
+            });
+        }
+        return metric;
+    }
+    async function trackDatabaseMetric() {
+        const metric = getDatabaseErrorMetric();
+        if (metric === null) {
+            return;
+        }
+        try {
+            if (!(await hasDatabaseConnection())) {
+                metric.inc();
+            }
+        }
+        catch {
+            metric.inc();
+        }
+    }
+    async function trackCacheMetric(checkId) {
+        const metric = getCacheErrorMetric();
+        if (metric === null) {
+            return;
+        }
+        const { cache } = getCache();
+        if (!cache) {
+            return;
+        }
+        try {
+            await cache.set(`metrics-${checkId}`, '1', 5);
+            await cache.delete(`metrics-${checkId}`);
+        }
+        catch {
+            metric.inc();
+        }
+    }
+    async function trackRedisMetric(checkId) {
+        const metric = getRedisErrorMetric();
+        if (metric === null) {
+            return;
+        }
+        const redis = useRedis();
+        try {
+            await redis.set(`metrics-${checkId}`, '1');
+            await redis.del(`metrics-${checkId}`);
+        }
+        catch {
+            metric.inc();
+        }
+    }
+    async function trackStorageMetric(checkId) {
+        if (services.includes('storage') === false) {
+            return;
+        }
+        const storage = await getStorage();
+        for (const location of toArray(env['STORAGE_LOCATIONS'])) {
+            const disk = storage.location(location);
+            const metric = getStorageErrorMetric(location);
+            if (metric === null) {
+                continue;
+            }
+            try {
+                await disk.write(`metric-${checkId}`, Readable.from(['check']));
+                const fileStream = await disk.read(`metric-${checkId}`);
+                await new Promise((resolve) => fileStream.on('data', async () => {
+                    fileStream.destroy();
+                    await disk.delete(`metric-${checkId}`);
+                    return resolve(null);
+                }));
+            }
+            catch {
+                metric.inc();
+            }
+        }
+    }
+    return {
+        getDatabaseErrorMetric,
+        getDatabaseResponseMetric,
+        getCacheErrorMetric,
+        getRedisErrorMetric,
+        getStorageErrorMetric,
+        aggregate,
+        generate,
+        readAll,
+    };
+}

package/dist/metrics/lib/use-metrics.d.ts

@@ -0,0 +1,17 @@
+import { createMetrics } from './create-metrics.js';
+export declare const _cache: {
+    metrics: ReturnType<typeof createMetrics> | undefined;
+};
+export declare const useMetrics: () => {
+    getDatabaseErrorMetric: () => import("prom-client").Counter | null;
+    getDatabaseResponseMetric: () => import("prom-client").Histogram | null;
+    getCacheErrorMetric: () => import("prom-client").Counter | null;
+    getRedisErrorMetric: () => import("prom-client").Counter | null;
+    getStorageErrorMetric: (location: string) => import("prom-client").Counter | null;
+    aggregate: (data: {
+        pid: number;
+        metrics: import("prom-client").MetricObjectWithValues<import("prom-client").MetricValue<string>>[];
+    }) => Promise<void>;
+    generate: () => Promise<void>;
+    readAll: () => Promise<string>;
+} | undefined;

package/dist/metrics/lib/use-metrics.js

@@ -0,0 +1,15 @@
+import { useEnv } from '@directus/env';
+import { toBoolean } from '@directus/utils';
+import { createMetrics } from './create-metrics.js';
+export const _cache = { metrics: undefined };
+export const useMetrics = () => {
+    const env = useEnv();
+    if (!toBoolean(env['METRICS_ENABLED'])) {
+        return;
+    }
+    if (_cache.metrics) {
+        return _cache.metrics;
+    }
+    _cache.metrics = createMetrics();
+    return _cache.metrics;
+};

package/dist/metrics/types/metric.d.ts

@@ -0,0 +1 @@
+export type MetricService = 'database' | 'cache' | 'redis' | 'storage';

package/dist/metrics/types/metric.js

@@ -0,0 +1 @@
+export {};

package/dist/middleware/respond.js

@@ -1,6 +1,7 @@
 import { useEnv } from '@directus/env';
 import { parse as parseBytesConfiguration } from 'bytes';
 import { getCache, setCacheValue } from '../cache.js';
+import getDatabase from '../database/index.js';
 import { useLogger } from '../logger/index.js';
 import { ExportService } from '../services/import-export.js';
 import asyncHandler from '../utils/async-handler.js';
@@ -9,6 +10,7 @@ import { getCacheKey } from '../utils/get-cache-key.js';
 import { getDateFormatted } from '../utils/get-date-formatted.js';
 import { getMilliseconds } from '../utils/get-milliseconds.js';
 import { stringByteSize } from '../utils/get-string-byte-size.js';
+import { permissionsCachable } from '../utils/permissions-cachable.js';
 export const respond = asyncHandler(async (req, res) => {
     const env = useEnv();
     const logger = useLogger();
@@ -26,7 +28,11 @@ export const respond = asyncHandler(async (req, res) => {
         cache &&
         !req.sanitizedQuery.export &&
         res.locals['cache'] !== false &&
-        exceedsMaxSize === false) {
+        exceedsMaxSize === false &&
+        (await permissionsCachable(req.collection, {
+            knex: getDatabase(),
+            schema: req.schema,
+        }, req.accountability))) {
         const key = await getCacheKey(req);
         try {
             await setCacheValue(cache, key, res.locals['payload'], getMilliseconds(env['CACHE_TTL']));

package/dist/operations/condition/index.js

@@ -1,11 +1,16 @@
-import { validatePayload } from '@directus/utils';
 import { defineOperationApi } from '@directus/extensions';
+import { validatePayload } from '@directus/utils';
+import { FailedValidationError, joiValidationErrorItemToErrorExtensions } from '@directus/validation';
 export default defineOperationApi({
     id: 'condition',
     handler: ({ filter }, { data }) => {
         const errors = validatePayload(filter, data, { requireAll: true });
         if (errors.length > 0) {
-            throw errors;
+            // sanitize and format errors
+            const validationErrors = errors
+                .map((error) => error.details.map((details) => new FailedValidationError(joiValidationErrorItemToErrorExtensions(details))))
+                .flat();
+            throw validationErrors;
         }
         else {
             return null;

package/dist/operations/mail/index.d.ts

@@ -1,10 +1,13 @@
 export type Options = {
-    body?: string;
-    template?: string;
-    data?: Record<string, any>;
     to: string;
     type: 'wysiwyg' | 'markdown' | 'template';
     subject: string;
+    body?: string;
+    template?: string;
+    data?: Record<string, any>;
+    cc?: string;
+    bcc?: string;
+    replyTo?: string;
 };
 declare const _default: import("@directus/extensions").OperationApiConfig<Options>;
 export default _default;

package/dist/operations/mail/index.js

@@ -5,9 +5,9 @@ import { useLogger } from '../../logger/index.js';
 const logger = useLogger();
 export default defineOperationApi({
     id: 'mail',
-    handler: async ({ body, template, data, to, type, subject }, { accountability, database, getSchema }) => {
+    handler: async ({ body, template, data, to, type, subject, cc, bcc, replyTo }, { accountability, database, getSchema }) => {
         const mailService = new MailService({ schema: await getSchema({ database }), accountability, knex: database });
-        const mailObject = { to, subject };
+        const mailObject = { to, subject, cc, bcc, replyTo };
         const safeBody = typeof body !== 'string' ? JSON.stringify(body) : body;
         if (type === 'template') {
             mailObject.template = {

package/dist/schedules/metrics.d.ts

@@ -0,0 +1,7 @@
+export declare function handleMetricsJob(): Promise<void>;
+/**
+ * Schedule the metric generation
+ *
+ * @returns Whether or not metrics has been initialized
+ */
+export default function schedule(): Promise<boolean>;

package/dist/schedules/metrics.js

@@ -0,0 +1,44 @@
+import { useEnv } from '@directus/env';
+import { toBoolean } from '@directus/utils';
+import { scheduleJob } from 'node-schedule';
+import { useLogger } from '../logger/index.js';
+import { useMetrics } from '../metrics/index.js';
+import { validateCron } from '../utils/schedule.js';
+const METRICS_LOCK_TIMEOUT = 10 * 60 * 1000; // 10 mins
+let lockedAt = 0;
+const logger = useLogger();
+const metrics = useMetrics();
+export async function handleMetricsJob() {
+    const now = Date.now();
+    if (lockedAt !== 0 && lockedAt > now - METRICS_LOCK_TIMEOUT) {
+        // ensure only generating metrics once per node
+        return;
+    }
+    lockedAt = Date.now();
+    try {
+        await metrics?.generate();
+    }
+    catch (err) {
+        logger.warn(`An error was thrown while attempting metric generation`);
+        logger.warn(err);
+    }
+    finally {
+        lockedAt = 0;
+    }
+}
+/**
+ * Schedule the metric generation
+ *
+ * @returns Whether or not metrics has been initialized
+ */
+export default async function schedule() {
+    const env = useEnv();
+    if (!toBoolean(env['METRICS_ENABLED'])) {
+        return false;
+    }
+    if (!validateCron(String(env['METRICS_SCHEDULE']))) {
+        return false;
+    }
+    scheduleJob('metrics', String(env['METRICS_SCHEDULE']), handleMetricsJob);
+    return true;
+}

package/dist/services/assets.d.ts

@@ -10,9 +10,14 @@ export declare class AssetsService {
     schema: SchemaOverview;
     filesService: FilesService;
     constructor(options: AbstractServiceOptions);
-    getAsset(id: string, transformation?: TransformationSet, range?: Range): Promise<{
+    getAsset(id: string, transformation?: TransformationSet, range?: Range, deferStream?: false): Promise<{
         stream: Readable;
         file: any;
         stat: Stat;
     }>;
+    getAsset(id: string, transformation?: TransformationSet, range?: Range, deferStream?: true): Promise<{
+        stream: () => Promise<Readable>;
+        file: any;
+        stat: Stat;
+    }>;
 }

package/dist/services/assets.js

@@ -28,7 +28,7 @@ export class AssetsService {
         this.schema = options.schema;
         this.filesService = new FilesService({ ...options, accountability: null });
     }
-    async getAsset(id, transformation, range) {
+    async getAsset(id, transformation, range, deferStream = false) {
         const storage = await getStorage();
         const publicSettings = await this.knex
             .select('project_logo', 'public_background', 'public_foreground', 'public_favicon')
@@ -99,8 +99,9 @@ export class AssetsService {
                 file.type = contentType(assetFilename) || null;
             }
             if (exists) {
+                const assetStream = () => storage.location(file.storage).read(assetFilename, { range });
                 return {
-                    stream: await storage.location(file.storage).read(assetFilename, { range }),
+                    stream: deferStream ? assetStream : await assetStream(),
                     file,
                     stat: await storage.location(file.storage).stat(assetFilename),
                 };
@@ -123,7 +124,6 @@ export class AssetsService {
                     reason: 'Server too busy',
                 });
             }
-            const readStream = await storage.location(file.storage).read(file.filename_disk, { range, version });
             const transformer = getSharpInstance();
             transformer.timeout({
                 seconds: clamp(Math.round(getMilliseconds(env['ASSETS_TRANSFORM_TIMEOUT'], 0) / 1000), 1, 3600),
@@ -131,6 +131,7 @@ export class AssetsService {
             if (transforms.find((transform) => transform[0] === 'rotate') === undefined)
                 transformer.rotate();
             transforms.forEach(([method, ...args]) => transformer[method].apply(transformer, args));
+            const readStream = await storage.location(file.storage).read(file.filename_disk, { range, version });
             readStream.on('error', (e) => {
                 logger.error(e, `Couldn't transform file ${file.id}`);
                 readStream.unpipe(transformer);
@@ -152,16 +153,17 @@ export class AssetsService {
                     throw error;
                 }
             }
+            const assetStream = () => storage.location(file.storage).read(assetFilename, { range, version });
             return {
-                stream: await storage.location(file.storage).read(assetFilename, { range, version }),
+                stream: deferStream ? assetStream : await assetStream(),
                 stat: await storage.location(file.storage).stat(assetFilename),
                 file,
             };
         }
         else {
-            const readStream = await storage.location(file.storage).read(file.filename_disk, { range, version });
+            const assetStream = () => storage.location(file.storage).read(file.filename_disk, { range, version });
             const stat = await storage.location(file.storage).stat(file.filename_disk);
-            return { stream: readStream, file, stat };
+            return { stream: deferStream ? assetStream : await assetStream(), file, stat };
         }
     }
 }

package/dist/services/fields.js

@@ -58,7 +58,7 @@ export class FieldsService {
         if (!columnInfo) {
             columnInfo = await this.schemaInspector.columnInfo();
             if (schemaCacheIsEnabled) {
-                setCacheValue(this.schemaCache, 'columnInfo', columnInfo);
+                await setCacheValue(this.schemaCache, 'columnInfo', columnInfo);
             }
         }
         if (collection) {
@@ -674,7 +674,17 @@ export class FieldsService {
         else {
             throw new InvalidPayloadError({ reason: `Illegal type passed: "${field.type}"` });
         }
-        const setDefaultValue = (defaultValue) => {
+        /**
+         * The column nullability must be set on every alter or it will be dropped
+         * This is due to column.alter() not being incremental per https://knexjs.org/guide/schema-builder.html#alter
+         */
+        this.helpers.schema.setNullable(column, field, existing);
+        /**
+         * The default value must be set on every alter or it will be dropped
+         * This is due to column.alter() not being incremental per https://knexjs.org/guide/schema-builder.html#alter
+         */
+        const defaultValue = field.schema?.default_value !== undefined ? field.schema?.default_value : existing?.default_value;
+        if (defaultValue !== undefined) {
             const newDefaultValueIsString = typeof defaultValue === 'string';
             const newDefaultIsNowFunction = newDefaultValueIsString && defaultValue.toLowerCase() === 'now()';
             const newDefaultIsCurrentTimestamp = newDefaultValueIsString && defaultValue === 'CURRENT_TIMESTAMP';
@@ -694,57 +704,31 @@ export class FieldsService {
             else {
                 column.defaultTo(defaultValue);
             }
-        };
-        // for a new item, set the default value and nullable as provided without any further considerations
-        if (!existing) {
-            if (field.schema?.default_value !== undefined) {
-                setDefaultValue(field.schema.default_value);
-            }
-            if (field.schema?.is_nullable || field.schema?.is_nullable === undefined) {
-                column.nullable();
-            }
-            else {
-                column.notNullable();
-            }
-        }
-        else {
-            // for an existing item: if nullable option changed, we have to provide the default values as well and actually vice versa
-            // see https://knexjs.org/guide/schema-builder.html#alter
-            // To overwrite a nullable option with the same value this is not possible for Oracle though, hence the DB helper
-            if (field.schema?.default_value !== undefined || field.schema?.is_nullable !== undefined) {
-                this.helpers.nullableUpdate.updateNullableValue(column, field, existing);
-                let defaultValue = null;
-                if (field.schema?.default_value !== undefined) {
-                    defaultValue = field.schema.default_value;
-                }
-                else if (existing.default_value !== undefined) {
-                    defaultValue = existing.default_value;
-                }
-                setDefaultValue(defaultValue);
-            }
         }
         if (field.schema?.is_primary_key) {
             column.primary().notNullable();
         }
         else if (!existing?.is_primary_key) {
             // primary key will already have unique/index constraints
-            const uniqueIndexName = this.helpers.schema.generateIndexName('unique', collection, field.field);
             if (field.schema?.is_unique === true) {
                 if (!existing || existing.is_unique === false) {
-                    column.unique({ indexName: uniqueIndexName });
+                    column.unique({ indexName: this.helpers.schema.generateIndexName('unique', collection, field.field) });
                 }
             }
             else if (field.schema?.is_unique === false) {
-                if (existing && existing.is_unique === true) {
-                    table.dropUnique([field.field], uniqueIndexName);
+                if (existing?.is_unique === true) {
+                    table.dropUnique([field.field], this.helpers.schema.generateIndexName('unique', collection, field.field));
                 }
             }
-            const indexName = this.helpers.schema.generateIndexName('index', collection, field.field);
-            if (field.schema?.is_indexed === true && !existing?.is_indexed) {
-                column.index(indexName);
+            if (field.schema?.is_indexed === true) {
+                if (!existing || existing.is_indexed === false) {
+                    column.index(this.helpers.schema.generateIndexName('index', collection, field.field));
+                }
             }
-            else if (field.schema?.is_indexed === false && existing?.is_indexed) {
-                table.dropIndex([field.field], indexName);
+            else if (field.schema?.is_indexed === false) {
+                if (existing?.is_indexed === true) {
+                    table.dropIndex([field.field], this.helpers.schema.generateIndexName('index', collection, field.field));
+                }
             }
         }
         if (existing) {

package/dist/services/files.js

@@ -1,5 +1,5 @@
 import { useEnv } from '@directus/env';
-import { ContentTooLargeError, ForbiddenError, InvalidPayloadError, ServiceUnavailableError } from '@directus/errors';
+import { ContentTooLargeError, InvalidPayloadError, ServiceUnavailableError } from '@directus/errors';
 import formatTitle from '@directus/format-title';
 import { toArray } from '@directus/utils';
 import encodeURL from 'encodeurl';
@@ -211,10 +211,11 @@ export class FilesService extends ItemsService {
      */
     async deleteMany(keys) {
         const storage = await getStorage();
-        const files = await super.readMany(keys, { fields: ['id', 'storage', 'filename_disk'], limit: -1 });
-        if (!files) {
-            throw new ForbiddenError();
-        }
+        const sudoFilesItemsService = new FilesService({
+            knex: this.knex,
+            schema: this.schema,
+        });
+        const files = await sudoFilesItemsService.readMany(keys, { fields: ['id', 'storage', 'filename_disk'], limit: -1 });
         await super.deleteMany(keys);
         for (const file of files) {
             const disk = storage.location(file['storage']);

package/dist/services/graphql/errors/format.d.ts

@@ -0,0 +1,6 @@
+import { type DirectusError } from '@directus/errors';
+import { GraphQLError } from 'graphql';
+/**
+ * Convert Directus-Exception into a GraphQL format, so it can be returned by GraphQL properly.
+ */
+export declare function formatError(error: DirectusError | DirectusError[]): GraphQLError;

package/dist/services/graphql/errors/format.js

@@ -0,0 +1,14 @@
+import {} from '@directus/errors';
+import { GraphQLError } from 'graphql';
+import { set } from 'lodash-es';
+/**
+ * Convert Directus-Exception into a GraphQL format, so it can be returned by GraphQL properly.
+ */
+export function formatError(error) {
+    if (Array.isArray(error)) {
+        set(error[0], 'extensions.code', error[0].code);
+        return new GraphQLError(error[0].message, undefined, undefined, undefined, undefined, error[0]);
+    }
+    set(error, 'extensions.code', error.code);
+    return new GraphQLError(error.message, undefined, undefined, undefined, undefined, error);
+}