@wbce-d9/api 12.6.0 → 12.7.0

package/dist/auth/drivers/baseoauth.js

@@ -39,11 +39,16 @@ export class BaseOAuthDriver extends LocalAuthDriver {
                 provider: this.getConfig()['provider'],
                 providerPayload: { accessToken: tokenSet.access_token, userInfo },
             }, { database: getDatabase(), schema: this.schema, accountability: null });
-            // Update user to update refresh_token and other properties that might have changed
-            await this.getUserService().updateOne(userId, {
-                ...updatedUserPayload,
-                auth_data: userPayload.auth_data ?? null,
-            });
+            // Update user to update refresh_token and other properties that might have changed.
+            // Keep the stored refresh_token if the provider didn't issue a new one (e.g. Google when consent was
+            // already granted): refresh() still checks it against the provider
+            const updateUserPayload = userPayload.auth_data
+                ? { ...updatedUserPayload, auth_data: userPayload.auth_data }
+                : updatedUserPayload;
+            // Skip empty updates, they crash (#127)
+            if (Object.keys(updateUserPayload).length > 0) {
+                await this.getUserService().updateOne(userId, updateUserPayload);
+            }
             return userId;
         }
         //  --- NEW USER ----
@@ -131,10 +136,16 @@ export class BaseOAuthDriver extends LocalAuthDriver {
                 throw this.handleError(e);
             }
         }
+        else if (this.getConfig()['requireRefreshToken'] === false) {
+            // Opt-in: the provider issued no refresh token (e.g. Google when consent was already granted),
+            // so the Directus session is the only source of truth for this user
+            logger.debug(`[${this.getClientName()}] No refresh token stored, continuing (REQUIRE_REFRESH_TOKEN=false)`);
+        }
         else {
-            // If no auth connexion, throw user out
+            // If no auth connexion, throw user out. On login, the callback retries once with prompt=consent
+            // so the provider can issue a refresh_token
             logger.error(`No [${this.getClientName()}] Session found`);
-            throw new InvalidCredentialsException();
+            throw new InvalidTokenException();
         }
     }
     handleError = (e) => {

package/dist/auth/drivers/baseoauth.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/env.js

@@ -141,6 +141,7 @@ const allowedEnvironmentVars = [
     'AUTH_.+_FIRST_NAME_KEY',
     'AUTH_.+_LAST_NAME_KEY',
     'AUTH_.+_ALLOW_PUBLIC_REGISTRATION',
+    'AUTH_.+_REQUIRE_REFRESH_TOKEN',
     'AUTH_.+_DEFAULT_ROLE_ID',
     'AUTH_.+_REGISTER_URL_ALLOW_LIST',
     'AUTH_.+_ICON',

package/dist/flows.d.ts

@@ -15,8 +15,22 @@ declare class FlowManager {
     runOperationFlow(id: string, data: unknown, context: Record<string, unknown>): Promise<unknown>;
     runWebhookFlow(id: string, data: unknown, context: Record<string, unknown>): Promise<unknown>;
     private load;
+    /**
+     * Manual flows act on behalf of the triggering user: make sure that user can see the flow itself
+     * and read the collection / items passed in the payload before running it.
+     */
+    private checkManualTriggerAccess;
     private unload;
     private executeFlow;
+    /**
+     * Return a copy of the event data whose payload has its concealed fields masked, as they would be on read.
+     * The data is returned as is when there is nothing to conceal.
+     */
+    private concealEventPayload;
+    /**
+     * Options are resolved against `concealedKeyedData` for the log operation and for the returned options (stored in
+     * the flow revisions), and against `keyedData` for every other operation.
+     */
     private executeOperation;
 }
 export {};

package/dist/flows.js

@@ -2,7 +2,7 @@ import * as sharedExceptions from '@wbce-d9/exceptions';
 import { Action } from '@wbce-d9/constants';
 import { applyOptionsData, isValidJSON, parseJSON, toArray } from '@wbce-d9/utils';
 import fastRedact from 'fast-redact';
-import { omit, pick } from 'lodash-es';
+import { cloneDeep, omit, pick } from 'lodash-es';
 import { get } from 'micromustache';
 import { schedule, validate } from 'node-cron';
 import getDatabase from './database/index.js';
@@ -15,6 +15,7 @@ import { getMessenger } from './messenger.js';
 import { ActivityService } from './services/activity.js';
 import * as services from './services/index.js';
 import { FlowsService } from './services/flows.js';
+import { PayloadService } from './services/payload.js';
 import { RevisionsService } from './services/revisions.js';
 import { constructFlowTree } from './utils/construct-flow-tree.js';
 import { getSchema } from './utils/get-schema.js';
@@ -197,13 +198,16 @@ class FlowManager {
                         logger.warn(`Specified collection must be one of: ${enabledCollections.join(', ')}.`);
                         throw new exceptions.ForbiddenException();
                     }
-                    if (flow.options['async']) {
-                        this.executeFlow(flow, data, context);
-                        return undefined;
-                    }
-                    else {
-                        return this.executeFlow(flow, data, context);
-                    }
+                    const run = () => {
+                        if (flow.options['async']) {
+                            this.executeFlow(flow, data, context);
+                            return undefined;
+                        }
+                        else {
+                            return this.executeFlow(flow, data, context);
+                        }
+                    };
+                    return this.checkManualTriggerAccess(flow, targetCollection, data, context).then(run);
                 };
                 // Default return to $last for manual
                 flow.options['return'] = '$last';
@@ -212,6 +216,34 @@ class FlowManager {
         }
         this.isLoaded = true;
     }
+    /**
+     * Manual flows act on behalf of the triggering user: make sure that user can see the flow itself
+     * and read the collection / items passed in the payload before running it.
+     */
+    async checkManualTriggerAccess(flow, collection, data, context) {
+        const accountability = context['accountability'];
+        if (!accountability)
+            throw new exceptions.ForbiddenException();
+        if (accountability.admin)
+            return;
+        const database = context['database'] ?? getDatabase();
+        const schema = context['schema'] ?? (await getSchema({ database }));
+        const flowsService = new FlowsService({ accountability, knex: database, schema });
+        await flowsService.readOne(flow.id, { fields: ['id'] });
+        const keys = data?.['body']?.keys;
+        if (keys === null || keys === undefined || (Array.isArray(keys) && keys.length === 0)) {
+            if (flow.options?.['requireSelection'] !== false)
+                throw new exceptions.ForbiddenException();
+            const canReadCollection = accountability.permissions?.some((permission) => permission.collection === collection && permission.action === 'read');
+            if (!canReadCollection)
+                throw new exceptions.ForbiddenException();
+            return;
+        }
+        if (!Array.isArray(keys))
+            throw new exceptions.ForbiddenException();
+        const authorizationService = new services.AuthorizationService({ accountability, knex: database, schema });
+        await authorizationService.checkAccess('read', collection, keys);
+    }
     async unload() {
         for (const trigger of this.triggerHandlers) {
             trigger.events.forEach((event) => {
@@ -242,13 +274,22 @@ class FlowManager {
             [ACCOUNTABILITY_KEY]: context?.['accountability'] ?? null,
             [ENV_KEY]: pick(env, env['FLOWS_ENV_ALLOW_LIST'] ? toArray(env['FLOWS_ENV_ALLOW_LIST']) : []),
         };
+        // Concealed fields of the event payload (e.g. passwords, tokens) must never end up in logs or revisions
+        const concealedData = await this.concealEventPayload(flow, data, database, schema);
+        const concealedKeyedData = {
+            ...keyedData,
+            [TRIGGER_KEY]: concealedData,
+            [LAST_KEY]: concealedData,
+        };
         let nextOperation = flow.operation;
         let lastOperationStatus = 'unknown';
         const steps = [];
         while (nextOperation !== null) {
-            const { successor, data, status, options } = await this.executeOperation(nextOperation, keyedData, context);
+            const { successor, data, status, options } = await this.executeOperation(nextOperation, keyedData, concealedKeyedData, context);
             keyedData[nextOperation.key] = data;
             keyedData[LAST_KEY] = data;
+            concealedKeyedData[nextOperation.key] = data;
+            concealedKeyedData[LAST_KEY] = data;
             lastOperationStatus = status;
             steps.push({ operation: nextOperation.id, key: nextOperation.key, status, options });
             nextOperation = successor;
@@ -279,7 +320,7 @@ class FlowManager {
                     item: flow.id,
                     data: {
                         steps: steps,
-                        data: redactLogs(omit(keyedData, '$accountability.permissions')), // Permissions is a ton of data, and is just a copy of what's in the directus_permissions table
+                        data: redactLogs(omit(concealedKeyedData, '$accountability.permissions')), // Permissions is a ton of data, and is just a copy of what's in the directus_permissions table
                     },
                 });
             }
@@ -295,15 +336,35 @@ class FlowManager {
         }
         return undefined;
     }
-    async executeOperation(operation, keyedData, context = {}) {
+    /**
+     * Return a copy of the event data whose payload has its concealed fields masked, as they would be on read.
+     * The data is returned as is when there is nothing to conceal.
+     */
+    async concealEventPayload(flow, data, knex, schema) {
+        if (flow.trigger !== 'event')
+            return data;
+        const { collection, payload } = data;
+        if (!schema.collections[collection] || !payload)
+            return data;
+        const payloadService = new PayloadService(collection, { knex, schema });
+        const concealedData = cloneDeep(data);
+        concealedData.payload = await payloadService.processValues('read', concealedData.payload);
+        return concealedData;
+    }
+    /**
+     * Options are resolved against `concealedKeyedData` for the log operation and for the returned options (stored in
+     * the flow revisions), and against `keyedData` for every other operation.
+     */
+    async executeOperation(operation, keyedData, concealedKeyedData, context = {}) {
         if (!(operation.type in this.operations)) {
             logger.warn(`Couldn't find operation ${operation.type}`);
             return { successor: null, status: 'unknown', data: null, options: null };
         }
         const handler = this.operations[operation.type];
         const options = applyOptionsData(operation.options, keyedData);
+        const concealedOptions = applyOptionsData(operation.options, concealedKeyedData);
         try {
-            let result = await handler(options, {
+            let result = await handler(operation.type === 'log' ? concealedOptions : options, {
                 services,
                 exceptions: { ...exceptions, ...sharedExceptions },
                 env,
@@ -321,7 +382,7 @@ class FlowManager {
             if (typeof result === 'object' && result !== null) {
                 result = mapValuesDeep(result, (_, value) => (value === undefined ? null : value));
             }
-            return { successor: operation.resolve, status: 'resolve', data: result ?? null, options };
+            return { successor: operation.resolve, status: 'resolve', data: result ?? null, options: concealedOptions };
         }
         catch (error) {
             let data;
@@ -343,7 +404,7 @@ class FlowManager {
                 successor: operation.reject,
                 status: 'reject',
                 data,
-                options,
+                options: concealedOptions,
             };
         }
     }

package/dist/flows.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/logger.js

@@ -6,17 +6,18 @@ import { URL } from 'url';
 import env from './env.js';
 import { REDACT_TEXT } from './constants.js';
 import { getConfigFromEnv } from './utils/get-config-from-env.js';
+const REDACTED_PATHS = ['req.headers.authorization', 'req.headers.cookie', 'req.query.access_token'];
 const pinoOptions = {
     level: env['LOG_LEVEL'] || 'info',
     redact: {
-        paths: ['req.headers.authorization', 'req.headers.cookie'],
+        paths: REDACTED_PATHS,
         censor: REDACT_TEXT,
     },
 };
 export const httpLoggerOptions = {
     level: env['LOG_LEVEL'] || 'info',
     redact: {
-        paths: ['req.headers.authorization', 'req.headers.cookie'],
+        paths: REDACTED_PATHS,
         censor: REDACT_TEXT,
     },
 };
@@ -43,7 +44,7 @@ if (env['LOG_STYLE'] !== 'raw') {
 }
 if (env['LOG_STYLE'] === 'raw') {
     httpLoggerOptions.redact = {
-        paths: ['req.headers.authorization', 'req.headers.cookie', 'res.headers'],
+        paths: [...REDACTED_PATHS, 'res.headers'],
         censor: (value, pathParts) => {
             const path = pathParts.join('.');
             if (path === 'res.headers') {

package/dist/services/fields.js

@@ -458,6 +458,18 @@ export class FieldsService {
                 if (Object.keys(collectionMetaUpdates).length > 0) {
                     await trx('directus_collections').update(collectionMetaUpdates).where({ collection });
                 }
+                const permissions = await trx('directus_permissions')
+                    .select('id', 'fields')
+                    .where({ collection });
+                for (const permission of permissions) {
+                    const permittedFields = permission.fields?.split(',') ?? [];
+                    if (permittedFields.includes(field) === false)
+                        continue;
+                    const remainingFields = permittedFields.filter((permittedField) => permittedField !== field);
+                    await trx('directus_permissions')
+                        .update({ fields: remainingFields.length > 0 ? remainingFields.join(',') : null })
+                        .where({ id: permission.id });
+                }
                 // Cleanup directus_fields
                 const metaRow = await trx
                     .select('collection', 'field')

package/dist/services/graphql/index.js

@@ -104,6 +104,9 @@ export class GraphQLService {
         return formattedResult;
     }
     getSchema(type = 'schema') {
+        if (type === 'sdl' && env['GRAPHQL_INTROSPECTION'] === false) {
+            throw new ForbiddenException();
+        }
         // eslint-disable-next-line @typescript-eslint/no-this-alias
         const self = this;
         const schemaComposer = new SchemaComposer();

package/dist/services/payload.js

@@ -3,7 +3,7 @@ import { format, isValid, parseISO } from 'date-fns';
 import flat from 'flat';
 import Joi from 'joi';
 import { clone, cloneDeep, isNil, isObject, isPlainObject, omit, pick } from 'lodash-es';
-import { v4 as uuid } from 'uuid';
+import { v4 as uuid, validate as validateUUID } from 'uuid';
 import { parse as wktToGeoJSON } from 'wellknown';
 import { getHelpers } from '../database/helpers/index.js';
 import getDatabase from '../database/index.js';
@@ -35,13 +35,19 @@ export class PayloadService {
             if (action === 'create' || action === 'update') {
                 return await generateHash(String(value));
             }
+            // Same as conceal
+            if (action === 'read')
+                return '**********';
             return value;
         },
         async uuid({ action, value }) {
             if (action === 'create' && !value) {
                 return uuid();
             }
-            return value;
+            if (action === 'create' && !validateUUID(String(value))) {
+                throw new InvalidPayloadException(`Invalid UUID "${value}"`);
+            }
+            return action === 'create' ? String(value).toLowerCase() : value;
         },
         async 'cast-boolean'({ action, value }) {
             if (action === 'read') {

package/dist/services/specifications.js

@@ -62,7 +62,8 @@ class OASSpecsService {
             info: {
                 title: 'Dynamic API Specification',
                 description: 'This is a dynamically generated API specification for all endpoints existing on the current project.',
-                version: version,
+                // Only admins see the real version, as on /server/info
+                version: this.accountability?.admin ? version : '0.0.0-hidden',
             },
             servers: [
                 {

package/dist/services/users.js

@@ -5,6 +5,7 @@ import { cloneDeep, isEmpty } from 'lodash-es';
 import { performance } from 'perf_hooks';
 import getDatabase from '../database/index.js';
 import env from '../env.js';
+import logger from '../logger.js';
 import { RecordNotUniqueException } from '../exceptions/database/record-not-unique.js';
 import { ForbiddenException, InvalidPayloadException, UnprocessableEntityException } from '../exceptions/index.js';
 import isUrlAllowed from '../utils/is-url-allowed.js';
@@ -340,37 +341,42 @@ export class UsersService extends ItemsService {
     async requestPasswordReset(email, url, subject) {
         const STALL_TIME = 500;
         const timeStart = performance.now();
-        const user = await this.getUserByEmail(email);
-        if (user?.status !== 'active') {
-            await stall(STALL_TIME, timeStart);
-            throw new ForbiddenException();
+        try {
+            if (url && isUrlAllowed(url, env['PASSWORD_RESET_URL_ALLOW_LIST']) === false) {
+                throw new InvalidPayloadException(`Url "${url}" can't be used to reset passwords.`);
+            }
+            const user = await this.getUserByEmail(email);
+            if (user?.status !== 'active') {
+                throw new ForbiddenException();
+            }
+            const mailService = new MailService({
+                schema: this.schema,
+                knex: this.knex,
+                accountability: this.accountability,
+            });
+            const payload = { email: user.email, scope: 'password-reset', hash: getSimpleHash('' + user.password) };
+            const token = jwt.sign(payload, env['SECRET'], { expiresIn: '1d', issuer: 'directus' });
+            const acceptURL = url
+                ? new Url(url).setQuery('token', token).toString()
+                : new Url(env['PUBLIC_URL']).addPath('admin', 'reset-password').setQuery('token', token).toString();
+            const subjectLine = subject ? subject : 'Password Reset Request';
+            mailService
+                .send({
+                to: user.email,
+                subject: subjectLine,
+                template: {
+                    name: 'password-reset',
+                    data: {
+                        url: acceptURL,
+                        email: user.email,
+                    },
+                },
+            })
+                .catch((err) => logger.warn(err, `[email] ${err}`));
         }
-        if (url && isUrlAllowed(url, env['PASSWORD_RESET_URL_ALLOW_LIST']) === false) {
-            throw new InvalidPayloadException(`Url "${url}" can't be used to reset passwords.`);
+        finally {
+            await stall(STALL_TIME, timeStart);
         }
-        const mailService = new MailService({
-            schema: this.schema,
-            knex: this.knex,
-            accountability: this.accountability,
-        });
-        const payload = { email: user.email, scope: 'password-reset', hash: getSimpleHash('' + user.password) };
-        const token = jwt.sign(payload, env['SECRET'], { expiresIn: '1d', issuer: 'directus' });
-        const acceptURL = url
-            ? new Url(url).setQuery('token', token).toString()
-            : new Url(env['PUBLIC_URL']).addPath('admin', 'reset-password').setQuery('token', token).toString();
-        const subjectLine = subject ? subject : 'Password Reset Request';
-        await mailService.send({
-            to: user.email,
-            subject: subjectLine,
-            template: {
-                name: 'password-reset',
-                data: {
-                    url: acceptURL,
-                    email: user.email,
-                },
-            },
-        });
-        await stall(STALL_TIME, timeStart);
     }
     async resetPassword(token, password) {
         const { email, scope, hash } = jwt.verify(token, env['SECRET'], { issuer: 'directus' });

package/dist/utils/apply-query.d.ts

@@ -32,4 +32,4 @@ export declare function applyFilter(knex: Knex, schema: SchemaOverview, rootQuer
 };
 export declare function addWhereClauses(knex: Knex, schema: SchemaOverview, dbQuery: Knex.QueryBuilder, filter: Filter, collection: string, aliasMap: AliasMap, logical?: 'and' | 'or'): void;
 export declare function applySearch(schema: SchemaOverview, dbQuery: Knex.QueryBuilder, searchQuery: string, collection: string): Promise<void>;
-export declare function applyAggregate(dbQuery: Knex.QueryBuilder, aggregate: Aggregate, collection: string): void;
+export declare function applyAggregate(schema: SchemaOverview, dbQuery: Knex.QueryBuilder, aggregate: Aggregate, collection: string): void;

package/dist/utils/apply-query.js

@@ -33,7 +33,7 @@ export default function applyQuery(knex, collection, dbQuery, query, schema, opt
         dbQuery.groupBy(query.group.map((column) => getColumn(knex, collection, column, false, schema)));
     }
     if (query.aggregate) {
-        applyAggregate(dbQuery, query.aggregate, collection);
+        applyAggregate(schema, dbQuery, query.aggregate, collection);
     }
     if (query.filter) {
         hasMultiRelationalFilter = applyFilter(knex, schema, dbQuery, query.filter, collection, aliasMap).hasMultiRelationalFilter;
@@ -166,6 +166,9 @@ export function applySort(knex, schema, rootQuery, rootSort, collection, aliasMa
     rootQuery.orderBy(sortRecords);
     return undefined;
 }
+function isConcealedField(schema, collection, field) {
+    return schema.collections[collection]?.fields[stripFunction(field)]?.special.includes('conceal') ?? false;
+}
 export function applyLimit(knex, rootQuery, limit) {
     if (typeof limit === 'number') {
         getHelpers(knex).schema.applyLimit(rootQuery, limit);
@@ -480,6 +483,8 @@ export async function applySearch(schema, dbQuery, searchQuery, collection) {
     const fields = Object.entries(schema.collections[collection].fields);
     dbQuery.andWhere(function () {
         fields.forEach(([name, field]) => {
+            if (field.special.includes('conceal'))
+                return;
             if (['text', 'string'].includes(field.type)) {
                 this.orWhereRaw(`LOWER(??) LIKE ?`, [`${collection}.${name}`, `%${searchQuery.toLowerCase()}%`]);
             }
@@ -494,11 +499,14 @@ export async function applySearch(schema, dbQuery, searchQuery, collection) {
         });
     });
 }
-export function applyAggregate(dbQuery, aggregate, collection) {
+export function applyAggregate(schema, dbQuery, aggregate, collection) {
     for (const [operation, fields] of Object.entries(aggregate)) {
         if (!fields)
             continue;
         for (const field of fields) {
+            if (!operation.startsWith('count') && isConcealedField(schema, collection, field)) {
+                throw new InvalidQueryException(`Field with "conceal" special does not allow the "${operation}" aggregate function`);
+            }
             if (operation === 'avg') {
                 dbQuery.avg(`${collection}.${field}`, { as: `avg->${field}` });
             }

package/dist/utils/package.d.ts

@@ -1,3 +1,3 @@
 declare const name: string, version: string;
-declare const mainVersion: any;
+declare const mainVersion: string;
 export { name, version, mainVersion };

package/dist/utils/package.js

@@ -4,6 +4,18 @@ import { fileURLToPath } from 'node:url';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const apiPkgPath = resolve(__dirname, '../../package.json');
 const { name, version } = JSON.parse(readFileSync(apiPkgPath, 'utf8'));
-const mainPkgPath = resolve(dirname(apiPkgPath), '../../../../../../package.json');
-const mainVersion = existsSync(mainPkgPath) ? JSON.parse(readFileSync(mainPkgPath, 'utf8')).version : null;
+function pkgVersion(path) {
+    try {
+        return existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')).version ?? null : null;
+    }
+    catch {
+        return null;
+    }
+}
+// d9 release version (the bundle, matching the Docker image tag): dev-checkout root, then the
+// installed bundle (published/projen image), then the api version as a last resort.
+const mainVersion = pkgVersion(resolve(dirname(apiPkgPath), '../../../../../../package.json')) ??
+    pkgVersion(resolve(process.cwd(), 'node_modules/@wbce-d9/directus9/package.json')) ??
+    pkgVersion(resolve(process.cwd(), 'node_modules/directus9/package.json')) ??
+    version;
 export { name, version, mainVersion };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wbce-d9/api",
-  "version": "12.6.0",
+  "version": "12.7.0",
   "description": "Directus is a real-time API and App dashboard for managing SQL database content",
   "type": "module",
   "keywords": [
@@ -144,20 +144,20 @@
     "tmp-promise": "3.0.3",
     "tsx": "3.12.6",
     "uuid-validate": "0.0.3",
-    "@wbce-d9/app": "11.0.1",
-    "@wbce-d9/exceptions": "10.0.1",
+    "@wbce-d9/app": "11.2.0",
     "@wbce-d9/extensions-sdk": "10.0.5",
-    "@wbce-d9/specs": "9.26.5",
-    "@wbce-d9/constants": "10.0.0",
     "@wbce-d9/schema": "10.0.0",
+    "@wbce-d9/specs": "9.26.5",
     "@wbce-d9/storage-driver-azure": "10.0.4",
-    "@wbce-d9/storage": "10.0.0",
     "@wbce-d9/storage-driver-cloudinary": "10.0.5",
+    "@wbce-d9/storage-driver-local": "10.0.4",
     "@wbce-d9/storage-driver-gcs": "10.0.4",
     "@wbce-d9/storage-driver-s3": "10.0.4",
     "@wbce-d9/update-check": "10.0.4",
+    "@wbce-d9/storage": "10.0.0",
     "@wbce-d9/utils": "10.0.4",
-    "@wbce-d9/storage-driver-local": "10.0.4"
+    "@wbce-d9/constants": "10.0.0",
+    "@wbce-d9/exceptions": "10.0.1"
   },
   "devDependencies": {
     "@ngneat/falso": "6.4.0",