@directus/api 30.0.0 → 32.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app.js +7 -0
- package/dist/auth/auth.d.ts +2 -1
- package/dist/auth/auth.js +7 -2
- package/dist/auth/drivers/ldap.d.ts +0 -2
- package/dist/auth/drivers/ldap.js +9 -7
- package/dist/auth/drivers/oauth2.d.ts +0 -2
- package/dist/auth/drivers/oauth2.js +28 -11
- package/dist/auth/drivers/openid.d.ts +0 -2
- package/dist/auth/drivers/openid.js +28 -11
- package/dist/auth/drivers/saml.d.ts +0 -2
- package/dist/auth/drivers/saml.js +5 -5
- package/dist/auth.js +1 -2
- package/dist/cli/commands/bootstrap/index.js +12 -33
- package/dist/cli/commands/init/index.js +1 -1
- package/dist/cli/commands/schema/apply.d.ts +4 -0
- package/dist/cli/commands/schema/apply.js +26 -3
- package/dist/controllers/collections.js +7 -2
- package/dist/controllers/fields.js +31 -8
- package/dist/controllers/mcp.d.ts +2 -0
- package/dist/controllers/mcp.js +33 -0
- package/dist/controllers/server.js +26 -1
- package/dist/controllers/settings.js +9 -2
- package/dist/controllers/users.js +17 -7
- package/dist/controllers/versions.js +3 -2
- package/dist/database/errors/dialects/mssql.d.ts +1 -1
- package/dist/database/errors/dialects/mssql.js +18 -10
- package/dist/database/helpers/fn/types.js +3 -3
- package/dist/database/helpers/schema/dialects/cockroachdb.d.ts +2 -1
- package/dist/database/helpers/schema/dialects/cockroachdb.js +13 -0
- package/dist/database/helpers/schema/dialects/mssql.d.ts +2 -1
- package/dist/database/helpers/schema/dialects/mssql.js +23 -0
- package/dist/database/helpers/schema/dialects/mysql.d.ts +2 -1
- package/dist/database/helpers/schema/dialects/mysql.js +25 -0
- package/dist/database/helpers/schema/dialects/oracle.d.ts +2 -1
- package/dist/database/helpers/schema/dialects/oracle.js +13 -0
- package/dist/database/helpers/schema/dialects/postgres.d.ts +2 -1
- package/dist/database/helpers/schema/dialects/postgres.js +13 -0
- package/dist/database/helpers/schema/types.d.ts +5 -0
- package/dist/database/helpers/schema/types.js +6 -0
- package/dist/database/migrations/20250813A-add-mcp.d.ts +3 -0
- package/dist/database/migrations/20250813A-add-mcp.js +18 -0
- package/dist/database/migrations/20251012A-add-field-searchable.d.ts +3 -0
- package/dist/database/migrations/20251012A-add-field-searchable.js +10 -0
- package/dist/database/migrations/20251014A-add-project-owner.d.ts +3 -0
- package/dist/database/migrations/20251014A-add-project-owner.js +37 -0
- package/dist/database/migrations/20251028A-add-retention-indexes.d.ts +3 -0
- package/dist/database/migrations/20251028A-add-retention-indexes.js +42 -0
- package/dist/database/run-ast/README.md +46 -0
- package/dist/database/run-ast/lib/apply-query/add-join.js +2 -2
- package/dist/database/run-ast/lib/apply-query/filter/get-filter-type.d.ts +2 -2
- package/dist/database/run-ast/lib/apply-query/index.d.ts +0 -1
- package/dist/database/run-ast/lib/apply-query/index.js +4 -6
- package/dist/database/run-ast/lib/apply-query/search.js +2 -0
- package/dist/database/run-ast/lib/get-db-query.js +7 -6
- package/dist/database/run-ast/utils/generate-alias.d.ts +6 -0
- package/dist/database/run-ast/utils/generate-alias.js +57 -0
- package/dist/flows.js +1 -0
- package/dist/mcp/define.d.ts +2 -0
- package/dist/mcp/define.js +3 -0
- package/dist/mcp/index.d.ts +1 -0
- package/dist/mcp/index.js +1 -0
- package/dist/mcp/schema.d.ts +485 -0
- package/dist/mcp/schema.js +219 -0
- package/dist/mcp/server.d.ts +103 -0
- package/dist/mcp/server.js +310 -0
- package/dist/mcp/tools/assets.d.ts +3 -0
- package/dist/mcp/tools/assets.js +54 -0
- package/dist/mcp/tools/collections.d.ts +84 -0
- package/dist/mcp/tools/collections.js +90 -0
- package/dist/mcp/tools/fields.d.ts +101 -0
- package/dist/mcp/tools/fields.js +157 -0
- package/dist/mcp/tools/files.d.ts +235 -0
- package/dist/mcp/tools/files.js +103 -0
- package/dist/mcp/tools/flows.d.ts +323 -0
- package/dist/mcp/tools/flows.js +85 -0
- package/dist/mcp/tools/folders.d.ts +95 -0
- package/dist/mcp/tools/folders.js +96 -0
- package/dist/mcp/tools/index.d.ts +15 -0
- package/dist/mcp/tools/index.js +29 -0
- package/dist/mcp/tools/items.d.ts +87 -0
- package/dist/mcp/tools/items.js +141 -0
- package/dist/mcp/tools/operations.d.ts +171 -0
- package/dist/mcp/tools/operations.js +77 -0
- package/dist/mcp/tools/prompts/assets.md +8 -0
- package/dist/mcp/tools/prompts/collections.md +336 -0
- package/dist/mcp/tools/prompts/fields.md +521 -0
- package/dist/mcp/tools/prompts/files.md +180 -0
- package/dist/mcp/tools/prompts/flows.md +495 -0
- package/dist/mcp/tools/prompts/folders.md +34 -0
- package/dist/mcp/tools/prompts/index.d.ts +16 -0
- package/dist/mcp/tools/prompts/index.js +19 -0
- package/dist/mcp/tools/prompts/items.md +317 -0
- package/dist/mcp/tools/prompts/operations.md +721 -0
- package/dist/mcp/tools/prompts/relations.md +386 -0
- package/dist/mcp/tools/prompts/schema.md +130 -0
- package/dist/mcp/tools/prompts/system-prompt-description.md +1 -0
- package/dist/mcp/tools/prompts/system-prompt.md +44 -0
- package/dist/mcp/tools/prompts/trigger-flow.md +214 -0
- package/dist/mcp/tools/relations.d.ts +73 -0
- package/dist/mcp/tools/relations.js +93 -0
- package/dist/mcp/tools/schema.d.ts +54 -0
- package/dist/mcp/tools/schema.js +317 -0
- package/dist/mcp/tools/system.d.ts +3 -0
- package/dist/mcp/tools/system.js +22 -0
- package/dist/mcp/tools/trigger-flow.d.ts +8 -0
- package/dist/mcp/tools/trigger-flow.js +48 -0
- package/dist/mcp/transport.d.ts +13 -0
- package/dist/mcp/transport.js +18 -0
- package/dist/mcp/types.d.ts +56 -0
- package/dist/mcp/types.js +1 -0
- package/dist/metrics/lib/create-metrics.js +16 -25
- package/dist/middleware/collection-exists.js +2 -2
- package/dist/operations/mail/index.js +3 -1
- package/dist/operations/mail/rate-limiter.d.ts +1 -0
- package/dist/operations/mail/rate-limiter.js +29 -0
- package/dist/permissions/modules/process-payload/process-payload.js +3 -10
- package/dist/permissions/modules/validate-access/validate-access.js +2 -3
- package/dist/schedules/metrics.js +6 -2
- package/dist/schedules/project.d.ts +4 -0
- package/dist/schedules/project.js +27 -0
- package/dist/services/authentication.js +36 -0
- package/dist/services/collections.d.ts +3 -3
- package/dist/services/collections.js +16 -1
- package/dist/services/fields.d.ts +21 -5
- package/dist/services/fields.js +109 -32
- package/dist/services/graphql/resolvers/query.js +1 -1
- package/dist/services/graphql/resolvers/system-admin.js +49 -5
- package/dist/services/graphql/schema/parse-query.js +8 -8
- package/dist/services/graphql/utils/aggregate-query.d.ts +1 -1
- package/dist/services/graphql/utils/aggregate-query.js +5 -1
- package/dist/services/graphql/utils/filter-replace-m2a.js +2 -1
- package/dist/services/import-export.d.ts +9 -1
- package/dist/services/import-export.js +287 -101
- package/dist/services/items.d.ts +1 -1
- package/dist/services/items.js +50 -24
- package/dist/services/mail/index.js +2 -0
- package/dist/services/mail/rate-limiter.d.ts +1 -0
- package/dist/services/mail/rate-limiter.js +29 -0
- package/dist/services/meta.js +28 -24
- package/dist/services/payload.d.ts +7 -3
- package/dist/services/payload.js +26 -12
- package/dist/services/schema.js +4 -1
- package/dist/services/server.d.ts +1 -0
- package/dist/services/server.js +15 -18
- package/dist/services/settings.d.ts +2 -1
- package/dist/services/settings.js +15 -0
- package/dist/services/tfa.d.ts +1 -1
- package/dist/services/tfa.js +20 -5
- package/dist/services/tus/server.js +14 -9
- package/dist/services/versions.d.ts +6 -4
- package/dist/services/versions.js +84 -25
- package/dist/telemetry/lib/get-report.js +4 -4
- package/dist/telemetry/lib/send-report.d.ts +6 -1
- package/dist/telemetry/lib/send-report.js +3 -1
- package/dist/telemetry/types/report.d.ts +17 -1
- package/dist/telemetry/utils/get-settings.d.ts +9 -0
- package/dist/telemetry/utils/get-settings.js +14 -0
- package/dist/test-utils/README.md +760 -0
- package/dist/test-utils/cache.d.ts +51 -0
- package/dist/test-utils/cache.js +59 -0
- package/dist/test-utils/database.d.ts +48 -0
- package/dist/test-utils/database.js +52 -0
- package/dist/test-utils/emitter.d.ts +35 -0
- package/dist/test-utils/emitter.js +38 -0
- package/dist/test-utils/fields-service.d.ts +28 -0
- package/dist/test-utils/fields-service.js +36 -0
- package/dist/test-utils/items-service.d.ts +23 -0
- package/dist/test-utils/items-service.js +37 -0
- package/dist/test-utils/knex.d.ts +164 -0
- package/dist/test-utils/knex.js +268 -0
- package/dist/test-utils/schema.d.ts +26 -0
- package/dist/test-utils/schema.js +35 -0
- package/dist/types/auth.d.ts +2 -3
- package/dist/utils/apply-diff.js +15 -0
- package/dist/utils/create-admin.d.ts +11 -0
- package/dist/utils/create-admin.js +50 -0
- package/dist/utils/get-schema.js +5 -3
- package/dist/utils/get-snapshot-diff.js +49 -5
- package/dist/utils/get-snapshot.js +13 -7
- package/dist/utils/sanitize-schema.d.ts +11 -4
- package/dist/utils/sanitize-schema.js +9 -6
- package/dist/utils/schedule.js +15 -19
- package/dist/utils/validate-diff.js +31 -0
- package/dist/utils/validate-snapshot.js +7 -0
- package/dist/utils/versioning/deep-map-with-schema.d.ts +23 -0
- package/dist/utils/versioning/deep-map-with-schema.js +81 -0
- package/dist/utils/versioning/handle-version.d.ts +2 -2
- package/dist/utils/versioning/handle-version.js +47 -43
- package/dist/utils/versioning/split-recursive.d.ts +4 -0
- package/dist/utils/versioning/split-recursive.js +27 -0
- package/dist/websocket/controllers/hooks.js +12 -20
- package/dist/websocket/messages.d.ts +3 -3
- package/package.json +65 -66
- package/dist/cli/utils/defaults.d.ts +0 -4
- package/dist/cli/utils/defaults.js +0 -17
- package/dist/telemetry/utils/get-project-id.d.ts +0 -2
- package/dist/telemetry/utils/get-project-id.js +0 -4
|
@@ -1,13 +1,16 @@
|
|
|
1
|
+
import { InvalidPayloadError } from '@directus/errors';
|
|
2
|
+
import { isSystemField } from '@directus/system-data';
|
|
1
3
|
import { GraphQLBoolean, GraphQLID, GraphQLList, GraphQLNonNull, GraphQLString } from 'graphql';
|
|
2
4
|
import { SchemaComposer, toInputObjectType } from 'graphql-compose';
|
|
5
|
+
import { fromZodError } from 'zod-validation-error';
|
|
3
6
|
import { CollectionsService } from '../../collections.js';
|
|
4
7
|
import { ExtensionsService } from '../../extensions.js';
|
|
5
|
-
import { FieldsService } from '../../fields.js';
|
|
8
|
+
import { FieldsService, systemFieldUpdateSchema } from '../../fields.js';
|
|
6
9
|
import { RelationsService } from '../../relations.js';
|
|
7
10
|
import { GraphQLService } from '../index.js';
|
|
11
|
+
import { getCollectionType } from './get-collection-type.js';
|
|
8
12
|
import { getFieldType } from './get-field-type.js';
|
|
9
13
|
import { getRelationType } from './get-relation-type.js';
|
|
10
|
-
import { getCollectionType } from './get-collection-type.js';
|
|
11
14
|
export function resolveSystemAdmin(gql, schema, schemaComposer) {
|
|
12
15
|
if (!gql.accountability?.admin) {
|
|
13
16
|
return;
|
|
@@ -24,13 +27,16 @@ export function resolveSystemAdmin(gql, schema, schemaComposer) {
|
|
|
24
27
|
}).addFields({
|
|
25
28
|
fields: [toInputObjectType(Field, { postfix: '_input' }).NonNull],
|
|
26
29
|
}).NonNull,
|
|
30
|
+
concurrentIndexCreation: { type: GraphQLBoolean, defaultValue: false },
|
|
27
31
|
},
|
|
28
32
|
resolve: async (_, args) => {
|
|
29
33
|
const collectionsService = new CollectionsService({
|
|
30
34
|
accountability: gql.accountability,
|
|
31
35
|
schema: gql.schema,
|
|
32
36
|
});
|
|
33
|
-
const collectionKey = await collectionsService.createOne(args['data']
|
|
37
|
+
const collectionKey = await collectionsService.createOne(args['data'], {
|
|
38
|
+
attemptConcurrentIndex: Boolean(args['concurrentIndexCreation']),
|
|
39
|
+
});
|
|
34
40
|
return await collectionsService.readOne(collectionKey);
|
|
35
41
|
},
|
|
36
42
|
},
|
|
@@ -77,13 +83,16 @@ export function resolveSystemAdmin(gql, schema, schemaComposer) {
|
|
|
77
83
|
args: {
|
|
78
84
|
collection: new GraphQLNonNull(GraphQLString),
|
|
79
85
|
data: toInputObjectType(Field, { postfix: '_input' }).NonNull,
|
|
86
|
+
concurrentIndexCreation: { type: GraphQLBoolean, defaultValue: false },
|
|
80
87
|
},
|
|
81
88
|
resolve: async (_, args) => {
|
|
82
89
|
const service = new FieldsService({
|
|
83
90
|
accountability: gql.accountability,
|
|
84
91
|
schema: gql.schema,
|
|
85
92
|
});
|
|
86
|
-
await service.createField(args['collection'], args['data']
|
|
93
|
+
await service.createField(args['collection'], args['data'], undefined, {
|
|
94
|
+
attemptConcurrentIndex: Boolean(args['concurrentIndexCreation']),
|
|
95
|
+
});
|
|
87
96
|
return await service.readOne(args['collection'], args['data'].field);
|
|
88
97
|
},
|
|
89
98
|
},
|
|
@@ -93,17 +102,52 @@ export function resolveSystemAdmin(gql, schema, schemaComposer) {
|
|
|
93
102
|
collection: new GraphQLNonNull(GraphQLString),
|
|
94
103
|
field: new GraphQLNonNull(GraphQLString),
|
|
95
104
|
data: toInputObjectType(Field, { postfix: '_input' }).NonNull,
|
|
105
|
+
concurrentIndexCreation: { type: GraphQLBoolean, defaultValue: false },
|
|
96
106
|
},
|
|
97
107
|
resolve: async (_, args) => {
|
|
98
108
|
const service = new FieldsService({
|
|
99
109
|
accountability: gql.accountability,
|
|
100
110
|
schema: gql.schema,
|
|
101
111
|
});
|
|
112
|
+
if (isSystemField(args['collection'], args['field'])) {
|
|
113
|
+
const validationResult = systemFieldUpdateSchema.safeParse(args['data']);
|
|
114
|
+
if (!validationResult.success) {
|
|
115
|
+
throw new InvalidPayloadError({ reason: fromZodError(validationResult.error).message });
|
|
116
|
+
}
|
|
117
|
+
}
|
|
102
118
|
await service.updateField(args['collection'], {
|
|
103
119
|
...args['data'],
|
|
104
120
|
field: args['field'],
|
|
121
|
+
}, {
|
|
122
|
+
attemptConcurrentIndex: Boolean(args['concurrentIndexCreation']),
|
|
105
123
|
});
|
|
106
|
-
return await service.readOne(args['collection'], args['
|
|
124
|
+
return await service.readOne(args['collection'], args['field']);
|
|
125
|
+
},
|
|
126
|
+
},
|
|
127
|
+
update_fields_items: {
|
|
128
|
+
type: Field,
|
|
129
|
+
args: {
|
|
130
|
+
collection: new GraphQLNonNull(GraphQLString),
|
|
131
|
+
data: [toInputObjectType(Field, { postfix: '_input' }).NonNull],
|
|
132
|
+
concurrentIndexCreation: { type: GraphQLBoolean, defaultValue: false },
|
|
133
|
+
},
|
|
134
|
+
resolve: async (_, args) => {
|
|
135
|
+
const service = new FieldsService({
|
|
136
|
+
accountability: gql.accountability,
|
|
137
|
+
schema: gql.schema,
|
|
138
|
+
});
|
|
139
|
+
for (const fieldData of args['data']) {
|
|
140
|
+
if (isSystemField(args['collection'], fieldData['field'])) {
|
|
141
|
+
const validationResult = systemFieldUpdateSchema.safeParse(fieldData);
|
|
142
|
+
if (!validationResult.success) {
|
|
143
|
+
throw new InvalidPayloadError({ reason: fromZodError(validationResult.error).message });
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
await service.updateFields(args['collection'], args['data'], {
|
|
148
|
+
attemptConcurrentIndex: Boolean(args['concurrentIndexCreation']),
|
|
149
|
+
});
|
|
150
|
+
return await service.readOne(args['collection'], args['field']);
|
|
107
151
|
},
|
|
108
152
|
},
|
|
109
153
|
delete_fields_item: {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { get, mapKeys, merge, set, uniq } from 'lodash-es';
|
|
2
2
|
import { sanitizeQuery } from '../../../utils/sanitize-query.js';
|
|
3
3
|
import { validateQuery } from '../../../utils/validate-query.js';
|
|
4
|
+
import { filterReplaceM2A, filterReplaceM2ADeep } from '../utils/filter-replace-m2a.js';
|
|
4
5
|
import { replaceFuncs } from '../utils/replace-funcs.js';
|
|
5
6
|
import { parseArgs } from './parse-args.js';
|
|
6
|
-
import { filterReplaceM2A, filterReplaceM2ADeep } from '../utils/filter-replace-m2a.js';
|
|
7
7
|
/**
|
|
8
8
|
* Get a Directus Query object from the parsed arguments (rawQuery) and GraphQL AST selectionSet. Converts SelectionSet into
|
|
9
9
|
* Directus' `fields` query for use in the resolver. Also applies variables where appropriate.
|
|
@@ -52,7 +52,8 @@ export async function getQuery(rawQuery, schema, selections, variableValues, acc
|
|
|
52
52
|
if (selection.selectionSet) {
|
|
53
53
|
if (!query.deep)
|
|
54
54
|
query.deep = {};
|
|
55
|
-
|
|
55
|
+
const path = parent.replaceAll(':', '__');
|
|
56
|
+
set(query.deep, path, merge({}, get(query.deep, parent), { _alias: { [selection.alias.value]: selection.name.value } }));
|
|
56
57
|
}
|
|
57
58
|
}
|
|
58
59
|
}
|
|
@@ -79,12 +80,11 @@ export async function getQuery(rawQuery, schema, selections, variableValues, acc
|
|
|
79
80
|
fields.push(current);
|
|
80
81
|
}
|
|
81
82
|
if (selection.kind === 'Field' && selection.arguments && selection.arguments.length > 0) {
|
|
82
|
-
if (
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
}
|
|
83
|
+
if (!query.deep)
|
|
84
|
+
query.deep = {};
|
|
85
|
+
const args = parseArgs(selection.arguments, variableValues);
|
|
86
|
+
const path = (currentAlias ?? current).replaceAll(':', '__');
|
|
87
|
+
set(query.deep, path, merge({}, get(query.deep, path), mapKeys(await sanitizeQuery(args, schema, accountability), (_value, key) => `_${key}`)));
|
|
88
88
|
}
|
|
89
89
|
}
|
|
90
90
|
return uniq(fields);
|
|
@@ -3,4 +3,4 @@ import type { SelectionNode } from 'graphql';
|
|
|
3
3
|
/**
|
|
4
4
|
* Resolve the aggregation query based on the requested aggregated fields
|
|
5
5
|
*/
|
|
6
|
-
export declare function getAggregateQuery(rawQuery: Query, selections: readonly SelectionNode[], schema: SchemaOverview, accountability?: Accountability | null): Promise<Query>;
|
|
6
|
+
export declare function getAggregateQuery(rawQuery: Query, selections: readonly SelectionNode[], schema: SchemaOverview, accountability?: Accountability | null, collection?: string): Promise<Query>;
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { sanitizeQuery } from '../../../utils/sanitize-query.js';
|
|
2
2
|
import { validateQuery } from '../../../utils/validate-query.js';
|
|
3
|
+
import { filterReplaceM2A } from './filter-replace-m2a.js';
|
|
3
4
|
import { replaceFuncs } from './replace-funcs.js';
|
|
4
5
|
/**
|
|
5
6
|
* Resolve the aggregation query based on the requested aggregated fields
|
|
6
7
|
*/
|
|
7
|
-
export async function getAggregateQuery(rawQuery, selections, schema, accountability) {
|
|
8
|
+
export async function getAggregateQuery(rawQuery, selections, schema, accountability, collection) {
|
|
8
9
|
const query = await sanitizeQuery(rawQuery, schema, accountability);
|
|
9
10
|
query.aggregate = {};
|
|
10
11
|
for (let aggregationGroup of selections) {
|
|
@@ -27,6 +28,9 @@ export async function getAggregateQuery(rawQuery, selections, schema, accountabi
|
|
|
27
28
|
if (query.filter) {
|
|
28
29
|
query.filter = replaceFuncs(query.filter);
|
|
29
30
|
}
|
|
31
|
+
if (collection && query.filter) {
|
|
32
|
+
query.filter = filterReplaceM2A(query.filter, collection, schema);
|
|
33
|
+
}
|
|
30
34
|
validateQuery(query);
|
|
31
35
|
return query;
|
|
32
36
|
}
|
|
@@ -48,7 +48,8 @@ export function filterReplaceM2ADeep(deep_arg, collection, schema) {
|
|
|
48
48
|
deep[key] = filterReplaceM2ADeep(deep[key], relation.related_collection, schema);
|
|
49
49
|
}
|
|
50
50
|
else if (type === 'a2o' && any_collection && relation.meta?.one_allowed_collections?.includes(any_collection)) {
|
|
51
|
-
deep[
|
|
51
|
+
deep[`${field}:${any_collection}`] = filterReplaceM2ADeep(deep[key], any_collection, schema);
|
|
52
|
+
delete deep[key];
|
|
52
53
|
}
|
|
53
54
|
}
|
|
54
55
|
if (key === '_filter') {
|
|
@@ -1,7 +1,15 @@
|
|
|
1
|
-
import type { AbstractServiceOptions, Accountability, ExportFormat, File, Query, SchemaOverview } from '@directus/types';
|
|
1
|
+
import type { AbstractServiceOptions, Accountability, DirectusError, ExportFormat, File, Query, SchemaOverview } from '@directus/types';
|
|
2
2
|
import type { Knex } from 'knex';
|
|
3
3
|
import type { Readable } from 'node:stream';
|
|
4
4
|
import type { FunctionFieldNode, FieldNode, NestedCollectionNode } from '../types/index.js';
|
|
5
|
+
export declare function createErrorTracker(): {
|
|
6
|
+
addCapturedError: (err: any, rowNumber: number) => void;
|
|
7
|
+
buildFinalErrors: () => DirectusError<any>[];
|
|
8
|
+
getCount: () => number;
|
|
9
|
+
hasErrors: () => boolean;
|
|
10
|
+
shouldStop: () => boolean;
|
|
11
|
+
hasGenericError: () => boolean;
|
|
12
|
+
};
|
|
5
13
|
export declare class ImportService {
|
|
6
14
|
knex: Knex;
|
|
7
15
|
accountability: Accountability | null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { useEnv } from '@directus/env';
|
|
2
|
-
import { ForbiddenError, InvalidPayloadError, ServiceUnavailableError, UnsupportedMediaTypeError, } from '@directus/errors';
|
|
2
|
+
import { createError, ErrorCode, ForbiddenError, InvalidPayloadError, ServiceUnavailableError, UnsupportedMediaTypeError, } from '@directus/errors';
|
|
3
3
|
import { isSystemCollection } from '@directus/system-data';
|
|
4
4
|
import { parseJSON, toArray } from '@directus/utils';
|
|
5
5
|
import { createTmpFile } from '@directus/utils/node';
|
|
@@ -28,6 +28,126 @@ import { parseFields } from '../database/get-ast-from-query/lib/parse-fields.js'
|
|
|
28
28
|
import { set } from 'lodash-es';
|
|
29
29
|
const env = useEnv();
|
|
30
30
|
const logger = useLogger();
|
|
31
|
+
const MAX_IMPORT_ERRORS = env['MAX_IMPORT_ERRORS'];
|
|
32
|
+
export function createErrorTracker() {
|
|
33
|
+
let genericError;
|
|
34
|
+
// For errors with field / type (joi validation or DB with field)
|
|
35
|
+
const fieldErrors = new Map();
|
|
36
|
+
let capturedErrorCount = 0;
|
|
37
|
+
let isLimitReached = false;
|
|
38
|
+
function convertToRanges(rows, minRangeSize = 4) {
|
|
39
|
+
const sorted = Array.from(new Set(rows)).sort((a, b) => a - b);
|
|
40
|
+
const result = [];
|
|
41
|
+
if (sorted.length === 0)
|
|
42
|
+
return [];
|
|
43
|
+
let start = sorted[0];
|
|
44
|
+
let prev = sorted[0];
|
|
45
|
+
let count = 1;
|
|
46
|
+
const nonConsecutive = [];
|
|
47
|
+
const flush = () => {
|
|
48
|
+
if (count >= minRangeSize) {
|
|
49
|
+
result.push({ type: 'range', start, end: prev });
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
for (let i = start; i <= prev; i++) {
|
|
53
|
+
nonConsecutive.push(i);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
58
|
+
const current = sorted[i];
|
|
59
|
+
if (current === prev + 1) {
|
|
60
|
+
prev = current;
|
|
61
|
+
count++;
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
flush();
|
|
65
|
+
start = prev = current;
|
|
66
|
+
count = 1;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
flush();
|
|
70
|
+
// Add non-consecutive rows as a single "lines" entry
|
|
71
|
+
if (nonConsecutive.length > 0) {
|
|
72
|
+
result.push({ type: 'lines', rows: nonConsecutive });
|
|
73
|
+
}
|
|
74
|
+
return result;
|
|
75
|
+
}
|
|
76
|
+
function addCapturedError(err, rowNumber) {
|
|
77
|
+
const field = err.extensions?.field;
|
|
78
|
+
if (field) {
|
|
79
|
+
const type = err.extensions?.type;
|
|
80
|
+
const substring = err.extensions?.substring;
|
|
81
|
+
const valid = err.extensions?.valid;
|
|
82
|
+
const invalid = err.extensions?.invalid;
|
|
83
|
+
let key = type ? `${field}|${type}` : field;
|
|
84
|
+
if (substring !== undefined)
|
|
85
|
+
key += `|substring:${substring}`;
|
|
86
|
+
if (valid !== undefined)
|
|
87
|
+
key += `|valid:${JSON.stringify(valid)}`;
|
|
88
|
+
if (invalid !== undefined)
|
|
89
|
+
key += `|invalid:${JSON.stringify(invalid)}`;
|
|
90
|
+
if (!fieldErrors.has(err.code)) {
|
|
91
|
+
fieldErrors.set(err.code, new Map());
|
|
92
|
+
}
|
|
93
|
+
const errorsByCode = fieldErrors.get(err.code);
|
|
94
|
+
if (!errorsByCode.has(key)) {
|
|
95
|
+
errorsByCode.set(key, {
|
|
96
|
+
message: err.message,
|
|
97
|
+
rowNumbers: [],
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
errorsByCode.get(key).rowNumbers.push(rowNumber);
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
genericError = err;
|
|
104
|
+
}
|
|
105
|
+
capturedErrorCount++;
|
|
106
|
+
if (capturedErrorCount >= MAX_IMPORT_ERRORS) {
|
|
107
|
+
isLimitReached = true;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
function hasGenericError() {
|
|
111
|
+
return genericError !== undefined;
|
|
112
|
+
}
|
|
113
|
+
function buildFinalErrors() {
|
|
114
|
+
if (genericError) {
|
|
115
|
+
return [genericError];
|
|
116
|
+
}
|
|
117
|
+
return Array.from(fieldErrors.entries()).flatMap(([code, fieldMap]) => Array.from(fieldMap.entries()).map(([compositeKey, errorData]) => {
|
|
118
|
+
const parts = compositeKey.split('|');
|
|
119
|
+
const field = parts[0];
|
|
120
|
+
const type = parts[1];
|
|
121
|
+
const extensions = {};
|
|
122
|
+
for (let i = 2; i < parts.length; i++) {
|
|
123
|
+
const [paramType, paramValue] = parts[i]?.split(':', 2) ?? [];
|
|
124
|
+
if (!paramType || paramValue === undefined)
|
|
125
|
+
continue;
|
|
126
|
+
try {
|
|
127
|
+
extensions[paramType] = JSON.parse(paramValue);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
extensions[paramType] = paramValue;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const ErrorClass = createError(code, errorData.message, 400);
|
|
134
|
+
return new ErrorClass({
|
|
135
|
+
field,
|
|
136
|
+
type,
|
|
137
|
+
...extensions,
|
|
138
|
+
rows: convertToRanges(errorData.rowNumbers),
|
|
139
|
+
});
|
|
140
|
+
}));
|
|
141
|
+
}
|
|
142
|
+
return {
|
|
143
|
+
addCapturedError,
|
|
144
|
+
buildFinalErrors,
|
|
145
|
+
getCount: () => capturedErrorCount,
|
|
146
|
+
hasErrors: () => capturedErrorCount > 0 || hasGenericError(),
|
|
147
|
+
shouldStop: () => isLimitReached || hasGenericError(),
|
|
148
|
+
hasGenericError,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
31
151
|
export class ImportService {
|
|
32
152
|
knex;
|
|
33
153
|
accountability;
|
|
@@ -71,37 +191,72 @@ export class ImportService {
|
|
|
71
191
|
async importJSON(collection, stream) {
|
|
72
192
|
const extractJSON = StreamArray.withParser();
|
|
73
193
|
const nestedActionEvents = [];
|
|
74
|
-
|
|
194
|
+
const errorTracker = createErrorTracker();
|
|
195
|
+
return transaction(this.knex, async (trx) => {
|
|
75
196
|
const service = getService(collection, {
|
|
76
197
|
knex: trx,
|
|
77
198
|
schema: this.schema,
|
|
78
199
|
accountability: this.accountability,
|
|
79
200
|
});
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
201
|
+
try {
|
|
202
|
+
await new Promise((resolve, reject) => {
|
|
203
|
+
let rowNumber = 1;
|
|
204
|
+
const saveQueue = queue(async (task) => {
|
|
205
|
+
if (errorTracker.shouldStop())
|
|
206
|
+
return;
|
|
207
|
+
try {
|
|
208
|
+
const result = await service.upsertOne(task.data, {
|
|
209
|
+
bypassEmitAction: (params) => nestedActionEvents.push(params),
|
|
210
|
+
});
|
|
211
|
+
return result;
|
|
212
|
+
}
|
|
213
|
+
catch (error) {
|
|
214
|
+
for (const err of toArray(error)) {
|
|
215
|
+
errorTracker.addCapturedError(err, task.rowNumber);
|
|
216
|
+
if (errorTracker.shouldStop()) {
|
|
217
|
+
break;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
if (errorTracker.shouldStop()) {
|
|
221
|
+
saveQueue.kill();
|
|
222
|
+
destroyStream(stream);
|
|
223
|
+
destroyStream(extractJSON);
|
|
224
|
+
reject();
|
|
225
|
+
}
|
|
226
|
+
return;
|
|
100
227
|
}
|
|
101
|
-
|
|
228
|
+
});
|
|
229
|
+
stream.pipe(extractJSON);
|
|
230
|
+
extractJSON.on('data', ({ value }) => {
|
|
231
|
+
saveQueue.push({ data: value, rowNumber: rowNumber++ });
|
|
232
|
+
});
|
|
233
|
+
extractJSON.on('error', (err) => {
|
|
234
|
+
destroyStream(stream);
|
|
235
|
+
destroyStream(extractJSON);
|
|
236
|
+
reject(new InvalidPayloadError({ reason: err.message }));
|
|
237
|
+
});
|
|
238
|
+
extractJSON.on('end', () => {
|
|
239
|
+
// In case of empty JSON file
|
|
240
|
+
if (!saveQueue.started)
|
|
241
|
+
return resolve();
|
|
242
|
+
saveQueue.drain(() => {
|
|
243
|
+
if (errorTracker.hasErrors()) {
|
|
244
|
+
return reject();
|
|
245
|
+
}
|
|
246
|
+
for (const nestedActionEvent of nestedActionEvents) {
|
|
247
|
+
emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
|
|
248
|
+
}
|
|
249
|
+
return resolve();
|
|
250
|
+
});
|
|
102
251
|
});
|
|
103
252
|
});
|
|
104
|
-
}
|
|
253
|
+
}
|
|
254
|
+
catch (error) {
|
|
255
|
+
if (!error && errorTracker.hasErrors()) {
|
|
256
|
+
throw errorTracker.buildFinalErrors();
|
|
257
|
+
}
|
|
258
|
+
throw error;
|
|
259
|
+
}
|
|
105
260
|
});
|
|
106
261
|
}
|
|
107
262
|
async importCSV(collection, stream) {
|
|
@@ -109,98 +264,129 @@ export class ImportService {
|
|
|
109
264
|
if (!tmpFile)
|
|
110
265
|
throw new Error('Failed to create temporary file for import');
|
|
111
266
|
const nestedActionEvents = [];
|
|
112
|
-
|
|
267
|
+
const errorTracker = createErrorTracker();
|
|
268
|
+
return transaction(this.knex, async (trx) => {
|
|
113
269
|
const service = getService(collection, {
|
|
114
270
|
knex: trx,
|
|
115
271
|
schema: this.schema,
|
|
116
272
|
accountability: this.accountability,
|
|
117
273
|
});
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
return value;
|
|
128
|
-
}
|
|
129
|
-
return parsedJson;
|
|
130
|
-
}
|
|
131
|
-
catch {
|
|
132
|
-
return value;
|
|
133
|
-
}
|
|
134
|
-
};
|
|
135
|
-
const PapaOptions = {
|
|
136
|
-
header: true,
|
|
137
|
-
// Trim whitespaces in headers, including the byte order mark (BOM) zero-width no-break space
|
|
138
|
-
transformHeader: (header) => header.trim(),
|
|
139
|
-
transform,
|
|
140
|
-
};
|
|
141
|
-
return new Promise((resolve, reject) => {
|
|
142
|
-
const streams = [stream];
|
|
143
|
-
const cleanup = (destroy = true) => {
|
|
144
|
-
if (destroy) {
|
|
145
|
-
for (const stream of streams) {
|
|
146
|
-
destroyStream(stream);
|
|
274
|
+
try {
|
|
275
|
+
await new Promise((resolve, reject) => {
|
|
276
|
+
const streams = [stream];
|
|
277
|
+
let rowNumber = 0;
|
|
278
|
+
const cleanup = (destroy = true) => {
|
|
279
|
+
if (destroy) {
|
|
280
|
+
for (const stream of streams) {
|
|
281
|
+
destroyStream(stream);
|
|
282
|
+
}
|
|
147
283
|
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
for (const field in obj) {
|
|
173
|
-
if (obj[field] !== undefined) {
|
|
174
|
-
set(result, field, obj[field]);
|
|
284
|
+
tmpFile.cleanup().catch(() => {
|
|
285
|
+
logger.warn(`Failed to cleanup temporary import file (${tmpFile.path})`);
|
|
286
|
+
});
|
|
287
|
+
};
|
|
288
|
+
const saveQueue = queue(async (task) => {
|
|
289
|
+
if (errorTracker.shouldStop())
|
|
290
|
+
return;
|
|
291
|
+
try {
|
|
292
|
+
const result = await service.upsertOne(task.data, {
|
|
293
|
+
bypassEmitAction: (action) => nestedActionEvents.push(action),
|
|
294
|
+
});
|
|
295
|
+
return result;
|
|
296
|
+
}
|
|
297
|
+
catch (error) {
|
|
298
|
+
for (const err of toArray(error)) {
|
|
299
|
+
errorTracker.addCapturedError(err, task.rowNumber);
|
|
300
|
+
if (errorTracker.shouldStop()) {
|
|
301
|
+
break;
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
if (errorTracker.shouldStop()) {
|
|
305
|
+
saveQueue.kill();
|
|
306
|
+
cleanup(true);
|
|
307
|
+
reject();
|
|
175
308
|
}
|
|
309
|
+
return;
|
|
176
310
|
}
|
|
177
|
-
|
|
178
|
-
|
|
311
|
+
});
|
|
312
|
+
const fileWriteStream = createWriteStream(tmpFile.path)
|
|
179
313
|
.on('error', (error) => {
|
|
180
314
|
cleanup();
|
|
181
|
-
reject(new
|
|
315
|
+
reject(new Error('Error while writing import data to temporary file', { cause: error }));
|
|
182
316
|
})
|
|
183
|
-
.on('
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
317
|
+
.on('finish', () => {
|
|
318
|
+
const fileReadStream = createReadStream(tmpFile.path).on('error', (error) => {
|
|
319
|
+
cleanup();
|
|
320
|
+
reject(new Error('Error while reading import data from temporary file', { cause: error }));
|
|
321
|
+
});
|
|
322
|
+
streams.push(fileReadStream);
|
|
323
|
+
fileReadStream
|
|
324
|
+
.pipe(Papa.parse(Papa.NODE_STREAM_INPUT, {
|
|
325
|
+
header: true,
|
|
326
|
+
transformHeader: (header) => header.trim(),
|
|
327
|
+
transform: (value) => {
|
|
328
|
+
if (value.length === 0)
|
|
329
|
+
return;
|
|
330
|
+
try {
|
|
331
|
+
const parsedJson = parseJSON(value);
|
|
332
|
+
if (typeof parsedJson === 'number') {
|
|
333
|
+
return value;
|
|
334
|
+
}
|
|
335
|
+
return parsedJson;
|
|
336
|
+
}
|
|
337
|
+
catch {
|
|
338
|
+
return value;
|
|
339
|
+
}
|
|
340
|
+
},
|
|
341
|
+
}))
|
|
342
|
+
.on('data', (obj) => {
|
|
343
|
+
rowNumber++;
|
|
344
|
+
const result = {};
|
|
345
|
+
for (const field in obj) {
|
|
346
|
+
if (obj[field] !== undefined) {
|
|
347
|
+
set(result, field, obj[field]);
|
|
348
|
+
}
|
|
191
349
|
}
|
|
192
|
-
|
|
350
|
+
saveQueue.push({ data: result, rowNumber });
|
|
351
|
+
})
|
|
352
|
+
.on('error', (error) => {
|
|
353
|
+
cleanup();
|
|
354
|
+
reject(new InvalidPayloadError({ reason: error.message }));
|
|
355
|
+
})
|
|
356
|
+
.on('end', () => {
|
|
357
|
+
// In case of empty CSV file
|
|
358
|
+
if (!saveQueue.started) {
|
|
359
|
+
cleanup(false);
|
|
360
|
+
return resolve();
|
|
361
|
+
}
|
|
362
|
+
saveQueue.drain(() => {
|
|
363
|
+
if (!errorTracker.shouldStop())
|
|
364
|
+
cleanup(false);
|
|
365
|
+
if (errorTracker.hasErrors()) {
|
|
366
|
+
return reject();
|
|
367
|
+
}
|
|
368
|
+
for (const nestedActionEvent of nestedActionEvents) {
|
|
369
|
+
emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
|
|
370
|
+
}
|
|
371
|
+
return resolve();
|
|
372
|
+
});
|
|
193
373
|
});
|
|
194
374
|
});
|
|
375
|
+
streams.push(fileWriteStream);
|
|
376
|
+
stream
|
|
377
|
+
.on('error', (error) => {
|
|
378
|
+
cleanup();
|
|
379
|
+
reject(new Error('Error while retrieving import data', { cause: error }));
|
|
380
|
+
})
|
|
381
|
+
.pipe(fileWriteStream);
|
|
195
382
|
});
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
});
|
|
383
|
+
}
|
|
384
|
+
catch (error) {
|
|
385
|
+
if (!error && errorTracker.hasErrors()) {
|
|
386
|
+
throw errorTracker.buildFinalErrors();
|
|
387
|
+
}
|
|
388
|
+
throw error;
|
|
389
|
+
}
|
|
204
390
|
});
|
|
205
391
|
}
|
|
206
392
|
}
|
package/dist/services/items.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AbstractService, AbstractServiceOptions, Accountability, Item as AnyItem,
|
|
1
|
+
import type { AbstractService, AbstractServiceOptions, Accountability, Item as AnyItem, MutationOptions, MutationTracker, PrimaryKey, Query, QueryOptions, SchemaOverview } from '@directus/types';
|
|
2
2
|
import type Keyv from 'keyv';
|
|
3
3
|
import type { Knex } from 'knex';
|
|
4
4
|
export declare class ItemsService<Item extends AnyItem = AnyItem, Collection extends string = string> implements AbstractService<Item> {
|