@directus/api 30.0.0 → 32.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. package/dist/app.js +7 -0
  2. package/dist/auth/auth.d.ts +2 -1
  3. package/dist/auth/auth.js +7 -2
  4. package/dist/auth/drivers/ldap.d.ts +0 -2
  5. package/dist/auth/drivers/ldap.js +9 -7
  6. package/dist/auth/drivers/oauth2.d.ts +0 -2
  7. package/dist/auth/drivers/oauth2.js +28 -11
  8. package/dist/auth/drivers/openid.d.ts +0 -2
  9. package/dist/auth/drivers/openid.js +28 -11
  10. package/dist/auth/drivers/saml.d.ts +0 -2
  11. package/dist/auth/drivers/saml.js +5 -5
  12. package/dist/auth.js +1 -2
  13. package/dist/cli/commands/bootstrap/index.js +12 -33
  14. package/dist/cli/commands/init/index.js +1 -1
  15. package/dist/cli/commands/schema/apply.d.ts +4 -0
  16. package/dist/cli/commands/schema/apply.js +26 -3
  17. package/dist/controllers/collections.js +7 -2
  18. package/dist/controllers/fields.js +31 -8
  19. package/dist/controllers/mcp.d.ts +2 -0
  20. package/dist/controllers/mcp.js +33 -0
  21. package/dist/controllers/server.js +26 -1
  22. package/dist/controllers/settings.js +9 -2
  23. package/dist/controllers/users.js +17 -7
  24. package/dist/controllers/versions.js +3 -2
  25. package/dist/database/errors/dialects/mssql.d.ts +1 -1
  26. package/dist/database/errors/dialects/mssql.js +18 -10
  27. package/dist/database/helpers/fn/types.js +3 -3
  28. package/dist/database/helpers/schema/dialects/cockroachdb.d.ts +2 -1
  29. package/dist/database/helpers/schema/dialects/cockroachdb.js +13 -0
  30. package/dist/database/helpers/schema/dialects/mssql.d.ts +2 -1
  31. package/dist/database/helpers/schema/dialects/mssql.js +23 -0
  32. package/dist/database/helpers/schema/dialects/mysql.d.ts +2 -1
  33. package/dist/database/helpers/schema/dialects/mysql.js +25 -0
  34. package/dist/database/helpers/schema/dialects/oracle.d.ts +2 -1
  35. package/dist/database/helpers/schema/dialects/oracle.js +13 -0
  36. package/dist/database/helpers/schema/dialects/postgres.d.ts +2 -1
  37. package/dist/database/helpers/schema/dialects/postgres.js +13 -0
  38. package/dist/database/helpers/schema/types.d.ts +5 -0
  39. package/dist/database/helpers/schema/types.js +6 -0
  40. package/dist/database/migrations/20250813A-add-mcp.d.ts +3 -0
  41. package/dist/database/migrations/20250813A-add-mcp.js +18 -0
  42. package/dist/database/migrations/20251012A-add-field-searchable.d.ts +3 -0
  43. package/dist/database/migrations/20251012A-add-field-searchable.js +10 -0
  44. package/dist/database/migrations/20251014A-add-project-owner.d.ts +3 -0
  45. package/dist/database/migrations/20251014A-add-project-owner.js +37 -0
  46. package/dist/database/migrations/20251028A-add-retention-indexes.d.ts +3 -0
  47. package/dist/database/migrations/20251028A-add-retention-indexes.js +42 -0
  48. package/dist/database/run-ast/README.md +46 -0
  49. package/dist/database/run-ast/lib/apply-query/add-join.js +2 -2
  50. package/dist/database/run-ast/lib/apply-query/filter/get-filter-type.d.ts +2 -2
  51. package/dist/database/run-ast/lib/apply-query/index.d.ts +0 -1
  52. package/dist/database/run-ast/lib/apply-query/index.js +4 -6
  53. package/dist/database/run-ast/lib/apply-query/search.js +2 -0
  54. package/dist/database/run-ast/lib/get-db-query.js +7 -6
  55. package/dist/database/run-ast/utils/generate-alias.d.ts +6 -0
  56. package/dist/database/run-ast/utils/generate-alias.js +57 -0
  57. package/dist/flows.js +1 -0
  58. package/dist/mcp/define.d.ts +2 -0
  59. package/dist/mcp/define.js +3 -0
  60. package/dist/mcp/index.d.ts +1 -0
  61. package/dist/mcp/index.js +1 -0
  62. package/dist/mcp/schema.d.ts +485 -0
  63. package/dist/mcp/schema.js +219 -0
  64. package/dist/mcp/server.d.ts +103 -0
  65. package/dist/mcp/server.js +310 -0
  66. package/dist/mcp/tools/assets.d.ts +3 -0
  67. package/dist/mcp/tools/assets.js +54 -0
  68. package/dist/mcp/tools/collections.d.ts +84 -0
  69. package/dist/mcp/tools/collections.js +90 -0
  70. package/dist/mcp/tools/fields.d.ts +101 -0
  71. package/dist/mcp/tools/fields.js +157 -0
  72. package/dist/mcp/tools/files.d.ts +235 -0
  73. package/dist/mcp/tools/files.js +103 -0
  74. package/dist/mcp/tools/flows.d.ts +323 -0
  75. package/dist/mcp/tools/flows.js +85 -0
  76. package/dist/mcp/tools/folders.d.ts +95 -0
  77. package/dist/mcp/tools/folders.js +96 -0
  78. package/dist/mcp/tools/index.d.ts +15 -0
  79. package/dist/mcp/tools/index.js +29 -0
  80. package/dist/mcp/tools/items.d.ts +87 -0
  81. package/dist/mcp/tools/items.js +141 -0
  82. package/dist/mcp/tools/operations.d.ts +171 -0
  83. package/dist/mcp/tools/operations.js +77 -0
  84. package/dist/mcp/tools/prompts/assets.md +8 -0
  85. package/dist/mcp/tools/prompts/collections.md +336 -0
  86. package/dist/mcp/tools/prompts/fields.md +521 -0
  87. package/dist/mcp/tools/prompts/files.md +180 -0
  88. package/dist/mcp/tools/prompts/flows.md +495 -0
  89. package/dist/mcp/tools/prompts/folders.md +34 -0
  90. package/dist/mcp/tools/prompts/index.d.ts +16 -0
  91. package/dist/mcp/tools/prompts/index.js +19 -0
  92. package/dist/mcp/tools/prompts/items.md +317 -0
  93. package/dist/mcp/tools/prompts/operations.md +721 -0
  94. package/dist/mcp/tools/prompts/relations.md +386 -0
  95. package/dist/mcp/tools/prompts/schema.md +130 -0
  96. package/dist/mcp/tools/prompts/system-prompt-description.md +1 -0
  97. package/dist/mcp/tools/prompts/system-prompt.md +44 -0
  98. package/dist/mcp/tools/prompts/trigger-flow.md +214 -0
  99. package/dist/mcp/tools/relations.d.ts +73 -0
  100. package/dist/mcp/tools/relations.js +93 -0
  101. package/dist/mcp/tools/schema.d.ts +54 -0
  102. package/dist/mcp/tools/schema.js +317 -0
  103. package/dist/mcp/tools/system.d.ts +3 -0
  104. package/dist/mcp/tools/system.js +22 -0
  105. package/dist/mcp/tools/trigger-flow.d.ts +8 -0
  106. package/dist/mcp/tools/trigger-flow.js +48 -0
  107. package/dist/mcp/transport.d.ts +13 -0
  108. package/dist/mcp/transport.js +18 -0
  109. package/dist/mcp/types.d.ts +56 -0
  110. package/dist/mcp/types.js +1 -0
  111. package/dist/metrics/lib/create-metrics.js +16 -25
  112. package/dist/middleware/collection-exists.js +2 -2
  113. package/dist/operations/mail/index.js +3 -1
  114. package/dist/operations/mail/rate-limiter.d.ts +1 -0
  115. package/dist/operations/mail/rate-limiter.js +29 -0
  116. package/dist/permissions/modules/process-payload/process-payload.js +3 -10
  117. package/dist/permissions/modules/validate-access/validate-access.js +2 -3
  118. package/dist/schedules/metrics.js +6 -2
  119. package/dist/schedules/project.d.ts +4 -0
  120. package/dist/schedules/project.js +27 -0
  121. package/dist/services/authentication.js +36 -0
  122. package/dist/services/collections.d.ts +3 -3
  123. package/dist/services/collections.js +16 -1
  124. package/dist/services/fields.d.ts +21 -5
  125. package/dist/services/fields.js +109 -32
  126. package/dist/services/graphql/resolvers/query.js +1 -1
  127. package/dist/services/graphql/resolvers/system-admin.js +49 -5
  128. package/dist/services/graphql/schema/parse-query.js +8 -8
  129. package/dist/services/graphql/utils/aggregate-query.d.ts +1 -1
  130. package/dist/services/graphql/utils/aggregate-query.js +5 -1
  131. package/dist/services/graphql/utils/filter-replace-m2a.js +2 -1
  132. package/dist/services/import-export.d.ts +9 -1
  133. package/dist/services/import-export.js +287 -101
  134. package/dist/services/items.d.ts +1 -1
  135. package/dist/services/items.js +50 -24
  136. package/dist/services/mail/index.js +2 -0
  137. package/dist/services/mail/rate-limiter.d.ts +1 -0
  138. package/dist/services/mail/rate-limiter.js +29 -0
  139. package/dist/services/meta.js +28 -24
  140. package/dist/services/payload.d.ts +7 -3
  141. package/dist/services/payload.js +26 -12
  142. package/dist/services/schema.js +4 -1
  143. package/dist/services/server.d.ts +1 -0
  144. package/dist/services/server.js +15 -18
  145. package/dist/services/settings.d.ts +2 -1
  146. package/dist/services/settings.js +15 -0
  147. package/dist/services/tfa.d.ts +1 -1
  148. package/dist/services/tfa.js +20 -5
  149. package/dist/services/tus/server.js +14 -9
  150. package/dist/services/versions.d.ts +6 -4
  151. package/dist/services/versions.js +84 -25
  152. package/dist/telemetry/lib/get-report.js +4 -4
  153. package/dist/telemetry/lib/send-report.d.ts +6 -1
  154. package/dist/telemetry/lib/send-report.js +3 -1
  155. package/dist/telemetry/types/report.d.ts +17 -1
  156. package/dist/telemetry/utils/get-settings.d.ts +9 -0
  157. package/dist/telemetry/utils/get-settings.js +14 -0
  158. package/dist/test-utils/README.md +760 -0
  159. package/dist/test-utils/cache.d.ts +51 -0
  160. package/dist/test-utils/cache.js +59 -0
  161. package/dist/test-utils/database.d.ts +48 -0
  162. package/dist/test-utils/database.js +52 -0
  163. package/dist/test-utils/emitter.d.ts +35 -0
  164. package/dist/test-utils/emitter.js +38 -0
  165. package/dist/test-utils/fields-service.d.ts +28 -0
  166. package/dist/test-utils/fields-service.js +36 -0
  167. package/dist/test-utils/items-service.d.ts +23 -0
  168. package/dist/test-utils/items-service.js +37 -0
  169. package/dist/test-utils/knex.d.ts +164 -0
  170. package/dist/test-utils/knex.js +268 -0
  171. package/dist/test-utils/schema.d.ts +26 -0
  172. package/dist/test-utils/schema.js +35 -0
  173. package/dist/types/auth.d.ts +2 -3
  174. package/dist/utils/apply-diff.js +15 -0
  175. package/dist/utils/create-admin.d.ts +11 -0
  176. package/dist/utils/create-admin.js +50 -0
  177. package/dist/utils/get-schema.js +5 -3
  178. package/dist/utils/get-snapshot-diff.js +49 -5
  179. package/dist/utils/get-snapshot.js +13 -7
  180. package/dist/utils/sanitize-schema.d.ts +11 -4
  181. package/dist/utils/sanitize-schema.js +9 -6
  182. package/dist/utils/schedule.js +15 -19
  183. package/dist/utils/validate-diff.js +31 -0
  184. package/dist/utils/validate-snapshot.js +7 -0
  185. package/dist/utils/versioning/deep-map-with-schema.d.ts +23 -0
  186. package/dist/utils/versioning/deep-map-with-schema.js +81 -0
  187. package/dist/utils/versioning/handle-version.d.ts +2 -2
  188. package/dist/utils/versioning/handle-version.js +47 -43
  189. package/dist/utils/versioning/split-recursive.d.ts +4 -0
  190. package/dist/utils/versioning/split-recursive.js +27 -0
  191. package/dist/websocket/controllers/hooks.js +12 -20
  192. package/dist/websocket/messages.d.ts +3 -3
  193. package/package.json +65 -66
  194. package/dist/cli/utils/defaults.d.ts +0 -4
  195. package/dist/cli/utils/defaults.js +0 -17
  196. package/dist/telemetry/utils/get-project-id.d.ts +0 -2
  197. package/dist/telemetry/utils/get-project-id.js +0 -4
@@ -1,13 +1,16 @@
1
+ import { InvalidPayloadError } from '@directus/errors';
2
+ import { isSystemField } from '@directus/system-data';
1
3
  import { GraphQLBoolean, GraphQLID, GraphQLList, GraphQLNonNull, GraphQLString } from 'graphql';
2
4
  import { SchemaComposer, toInputObjectType } from 'graphql-compose';
5
+ import { fromZodError } from 'zod-validation-error';
3
6
  import { CollectionsService } from '../../collections.js';
4
7
  import { ExtensionsService } from '../../extensions.js';
5
- import { FieldsService } from '../../fields.js';
8
+ import { FieldsService, systemFieldUpdateSchema } from '../../fields.js';
6
9
  import { RelationsService } from '../../relations.js';
7
10
  import { GraphQLService } from '../index.js';
11
+ import { getCollectionType } from './get-collection-type.js';
8
12
  import { getFieldType } from './get-field-type.js';
9
13
  import { getRelationType } from './get-relation-type.js';
10
- import { getCollectionType } from './get-collection-type.js';
11
14
  export function resolveSystemAdmin(gql, schema, schemaComposer) {
12
15
  if (!gql.accountability?.admin) {
13
16
  return;
@@ -24,13 +27,16 @@ export function resolveSystemAdmin(gql, schema, schemaComposer) {
24
27
  }).addFields({
25
28
  fields: [toInputObjectType(Field, { postfix: '_input' }).NonNull],
26
29
  }).NonNull,
30
+ concurrentIndexCreation: { type: GraphQLBoolean, defaultValue: false },
27
31
  },
28
32
  resolve: async (_, args) => {
29
33
  const collectionsService = new CollectionsService({
30
34
  accountability: gql.accountability,
31
35
  schema: gql.schema,
32
36
  });
33
- const collectionKey = await collectionsService.createOne(args['data']);
37
+ const collectionKey = await collectionsService.createOne(args['data'], {
38
+ attemptConcurrentIndex: Boolean(args['concurrentIndexCreation']),
39
+ });
34
40
  return await collectionsService.readOne(collectionKey);
35
41
  },
36
42
  },
@@ -77,13 +83,16 @@ export function resolveSystemAdmin(gql, schema, schemaComposer) {
77
83
  args: {
78
84
  collection: new GraphQLNonNull(GraphQLString),
79
85
  data: toInputObjectType(Field, { postfix: '_input' }).NonNull,
86
+ concurrentIndexCreation: { type: GraphQLBoolean, defaultValue: false },
80
87
  },
81
88
  resolve: async (_, args) => {
82
89
  const service = new FieldsService({
83
90
  accountability: gql.accountability,
84
91
  schema: gql.schema,
85
92
  });
86
- await service.createField(args['collection'], args['data']);
93
+ await service.createField(args['collection'], args['data'], undefined, {
94
+ attemptConcurrentIndex: Boolean(args['concurrentIndexCreation']),
95
+ });
87
96
  return await service.readOne(args['collection'], args['data'].field);
88
97
  },
89
98
  },
@@ -93,17 +102,52 @@ export function resolveSystemAdmin(gql, schema, schemaComposer) {
93
102
  collection: new GraphQLNonNull(GraphQLString),
94
103
  field: new GraphQLNonNull(GraphQLString),
95
104
  data: toInputObjectType(Field, { postfix: '_input' }).NonNull,
105
+ concurrentIndexCreation: { type: GraphQLBoolean, defaultValue: false },
96
106
  },
97
107
  resolve: async (_, args) => {
98
108
  const service = new FieldsService({
99
109
  accountability: gql.accountability,
100
110
  schema: gql.schema,
101
111
  });
112
+ if (isSystemField(args['collection'], args['field'])) {
113
+ const validationResult = systemFieldUpdateSchema.safeParse(args['data']);
114
+ if (!validationResult.success) {
115
+ throw new InvalidPayloadError({ reason: fromZodError(validationResult.error).message });
116
+ }
117
+ }
102
118
  await service.updateField(args['collection'], {
103
119
  ...args['data'],
104
120
  field: args['field'],
121
+ }, {
122
+ attemptConcurrentIndex: Boolean(args['concurrentIndexCreation']),
105
123
  });
106
- return await service.readOne(args['collection'], args['data'].field);
124
+ return await service.readOne(args['collection'], args['field']);
125
+ },
126
+ },
127
+ update_fields_items: {
128
+ type: Field,
129
+ args: {
130
+ collection: new GraphQLNonNull(GraphQLString),
131
+ data: [toInputObjectType(Field, { postfix: '_input' }).NonNull],
132
+ concurrentIndexCreation: { type: GraphQLBoolean, defaultValue: false },
133
+ },
134
+ resolve: async (_, args) => {
135
+ const service = new FieldsService({
136
+ accountability: gql.accountability,
137
+ schema: gql.schema,
138
+ });
139
+ for (const fieldData of args['data']) {
140
+ if (isSystemField(args['collection'], fieldData['field'])) {
141
+ const validationResult = systemFieldUpdateSchema.safeParse(fieldData);
142
+ if (!validationResult.success) {
143
+ throw new InvalidPayloadError({ reason: fromZodError(validationResult.error).message });
144
+ }
145
+ }
146
+ }
147
+ await service.updateFields(args['collection'], args['data'], {
148
+ attemptConcurrentIndex: Boolean(args['concurrentIndexCreation']),
149
+ });
150
+ return await service.readOne(args['collection'], args['field']);
107
151
  },
108
152
  },
109
153
  delete_fields_item: {
@@ -1,9 +1,9 @@
1
1
  import { get, mapKeys, merge, set, uniq } from 'lodash-es';
2
2
  import { sanitizeQuery } from '../../../utils/sanitize-query.js';
3
3
  import { validateQuery } from '../../../utils/validate-query.js';
4
+ import { filterReplaceM2A, filterReplaceM2ADeep } from '../utils/filter-replace-m2a.js';
4
5
  import { replaceFuncs } from '../utils/replace-funcs.js';
5
6
  import { parseArgs } from './parse-args.js';
6
- import { filterReplaceM2A, filterReplaceM2ADeep } from '../utils/filter-replace-m2a.js';
7
7
  /**
8
8
  * Get a Directus Query object from the parsed arguments (rawQuery) and GraphQL AST selectionSet. Converts SelectionSet into
9
9
  * Directus' `fields` query for use in the resolver. Also applies variables where appropriate.
@@ -52,7 +52,8 @@ export async function getQuery(rawQuery, schema, selections, variableValues, acc
52
52
  if (selection.selectionSet) {
53
53
  if (!query.deep)
54
54
  query.deep = {};
55
- set(query.deep, parent, merge({}, get(query.deep, parent), { _alias: { [selection.alias.value]: selection.name.value } }));
55
+ const path = parent.replaceAll(':', '__');
56
+ set(query.deep, path, merge({}, get(query.deep, parent), { _alias: { [selection.alias.value]: selection.name.value } }));
56
57
  }
57
58
  }
58
59
  }
@@ -79,12 +80,11 @@ export async function getQuery(rawQuery, schema, selections, variableValues, acc
79
80
  fields.push(current);
80
81
  }
81
82
  if (selection.kind === 'Field' && selection.arguments && selection.arguments.length > 0) {
82
- if (selection.arguments && selection.arguments.length > 0) {
83
- if (!query.deep)
84
- query.deep = {};
85
- const args = parseArgs(selection.arguments, variableValues);
86
- set(query.deep, currentAlias ?? current, merge({}, get(query.deep, currentAlias ?? current), mapKeys(await sanitizeQuery(args, schema, accountability), (_value, key) => `_${key}`)));
87
- }
83
+ if (!query.deep)
84
+ query.deep = {};
85
+ const args = parseArgs(selection.arguments, variableValues);
86
+ const path = (currentAlias ?? current).replaceAll(':', '__');
87
+ set(query.deep, path, merge({}, get(query.deep, path), mapKeys(await sanitizeQuery(args, schema, accountability), (_value, key) => `_${key}`)));
88
88
  }
89
89
  }
90
90
  return uniq(fields);
@@ -3,4 +3,4 @@ import type { SelectionNode } from 'graphql';
3
3
  /**
4
4
  * Resolve the aggregation query based on the requested aggregated fields
5
5
  */
6
- export declare function getAggregateQuery(rawQuery: Query, selections: readonly SelectionNode[], schema: SchemaOverview, accountability?: Accountability | null): Promise<Query>;
6
+ export declare function getAggregateQuery(rawQuery: Query, selections: readonly SelectionNode[], schema: SchemaOverview, accountability?: Accountability | null, collection?: string): Promise<Query>;
@@ -1,10 +1,11 @@
1
1
  import { sanitizeQuery } from '../../../utils/sanitize-query.js';
2
2
  import { validateQuery } from '../../../utils/validate-query.js';
3
+ import { filterReplaceM2A } from './filter-replace-m2a.js';
3
4
  import { replaceFuncs } from './replace-funcs.js';
4
5
  /**
5
6
  * Resolve the aggregation query based on the requested aggregated fields
6
7
  */
7
- export async function getAggregateQuery(rawQuery, selections, schema, accountability) {
8
+ export async function getAggregateQuery(rawQuery, selections, schema, accountability, collection) {
8
9
  const query = await sanitizeQuery(rawQuery, schema, accountability);
9
10
  query.aggregate = {};
10
11
  for (let aggregationGroup of selections) {
@@ -27,6 +28,9 @@ export async function getAggregateQuery(rawQuery, selections, schema, accountabi
27
28
  if (query.filter) {
28
29
  query.filter = replaceFuncs(query.filter);
29
30
  }
31
+ if (collection && query.filter) {
32
+ query.filter = filterReplaceM2A(query.filter, collection, schema);
33
+ }
30
34
  validateQuery(query);
31
35
  return query;
32
36
  }
@@ -48,7 +48,8 @@ export function filterReplaceM2ADeep(deep_arg, collection, schema) {
48
48
  deep[key] = filterReplaceM2ADeep(deep[key], relation.related_collection, schema);
49
49
  }
50
50
  else if (type === 'a2o' && any_collection && relation.meta?.one_allowed_collections?.includes(any_collection)) {
51
- deep[key] = filterReplaceM2ADeep(deep[key], any_collection, schema);
51
+ deep[`${field}:${any_collection}`] = filterReplaceM2ADeep(deep[key], any_collection, schema);
52
+ delete deep[key];
52
53
  }
53
54
  }
54
55
  if (key === '_filter') {
@@ -1,7 +1,15 @@
1
- import type { AbstractServiceOptions, Accountability, ExportFormat, File, Query, SchemaOverview } from '@directus/types';
1
+ import type { AbstractServiceOptions, Accountability, DirectusError, ExportFormat, File, Query, SchemaOverview } from '@directus/types';
2
2
  import type { Knex } from 'knex';
3
3
  import type { Readable } from 'node:stream';
4
4
  import type { FunctionFieldNode, FieldNode, NestedCollectionNode } from '../types/index.js';
5
+ export declare function createErrorTracker(): {
6
+ addCapturedError: (err: any, rowNumber: number) => void;
7
+ buildFinalErrors: () => DirectusError<any>[];
8
+ getCount: () => number;
9
+ hasErrors: () => boolean;
10
+ shouldStop: () => boolean;
11
+ hasGenericError: () => boolean;
12
+ };
5
13
  export declare class ImportService {
6
14
  knex: Knex;
7
15
  accountability: Accountability | null;
@@ -1,5 +1,5 @@
1
1
  import { useEnv } from '@directus/env';
2
- import { ForbiddenError, InvalidPayloadError, ServiceUnavailableError, UnsupportedMediaTypeError, } from '@directus/errors';
2
+ import { createError, ErrorCode, ForbiddenError, InvalidPayloadError, ServiceUnavailableError, UnsupportedMediaTypeError, } from '@directus/errors';
3
3
  import { isSystemCollection } from '@directus/system-data';
4
4
  import { parseJSON, toArray } from '@directus/utils';
5
5
  import { createTmpFile } from '@directus/utils/node';
@@ -28,6 +28,126 @@ import { parseFields } from '../database/get-ast-from-query/lib/parse-fields.js'
28
28
  import { set } from 'lodash-es';
29
29
  const env = useEnv();
30
30
  const logger = useLogger();
31
+ const MAX_IMPORT_ERRORS = env['MAX_IMPORT_ERRORS'];
32
+ export function createErrorTracker() {
33
+ let genericError;
34
+ // For errors with field / type (joi validation or DB with field)
35
+ const fieldErrors = new Map();
36
+ let capturedErrorCount = 0;
37
+ let isLimitReached = false;
38
+ function convertToRanges(rows, minRangeSize = 4) {
39
+ const sorted = Array.from(new Set(rows)).sort((a, b) => a - b);
40
+ const result = [];
41
+ if (sorted.length === 0)
42
+ return [];
43
+ let start = sorted[0];
44
+ let prev = sorted[0];
45
+ let count = 1;
46
+ const nonConsecutive = [];
47
+ const flush = () => {
48
+ if (count >= minRangeSize) {
49
+ result.push({ type: 'range', start, end: prev });
50
+ }
51
+ else {
52
+ for (let i = start; i <= prev; i++) {
53
+ nonConsecutive.push(i);
54
+ }
55
+ }
56
+ };
57
+ for (let i = 1; i < sorted.length; i++) {
58
+ const current = sorted[i];
59
+ if (current === prev + 1) {
60
+ prev = current;
61
+ count++;
62
+ }
63
+ else {
64
+ flush();
65
+ start = prev = current;
66
+ count = 1;
67
+ }
68
+ }
69
+ flush();
70
+ // Add non-consecutive rows as a single "lines" entry
71
+ if (nonConsecutive.length > 0) {
72
+ result.push({ type: 'lines', rows: nonConsecutive });
73
+ }
74
+ return result;
75
+ }
76
+ function addCapturedError(err, rowNumber) {
77
+ const field = err.extensions?.field;
78
+ if (field) {
79
+ const type = err.extensions?.type;
80
+ const substring = err.extensions?.substring;
81
+ const valid = err.extensions?.valid;
82
+ const invalid = err.extensions?.invalid;
83
+ let key = type ? `${field}|${type}` : field;
84
+ if (substring !== undefined)
85
+ key += `|substring:${substring}`;
86
+ if (valid !== undefined)
87
+ key += `|valid:${JSON.stringify(valid)}`;
88
+ if (invalid !== undefined)
89
+ key += `|invalid:${JSON.stringify(invalid)}`;
90
+ if (!fieldErrors.has(err.code)) {
91
+ fieldErrors.set(err.code, new Map());
92
+ }
93
+ const errorsByCode = fieldErrors.get(err.code);
94
+ if (!errorsByCode.has(key)) {
95
+ errorsByCode.set(key, {
96
+ message: err.message,
97
+ rowNumbers: [],
98
+ });
99
+ }
100
+ errorsByCode.get(key).rowNumbers.push(rowNumber);
101
+ }
102
+ else {
103
+ genericError = err;
104
+ }
105
+ capturedErrorCount++;
106
+ if (capturedErrorCount >= MAX_IMPORT_ERRORS) {
107
+ isLimitReached = true;
108
+ }
109
+ }
110
+ function hasGenericError() {
111
+ return genericError !== undefined;
112
+ }
113
+ function buildFinalErrors() {
114
+ if (genericError) {
115
+ return [genericError];
116
+ }
117
+ return Array.from(fieldErrors.entries()).flatMap(([code, fieldMap]) => Array.from(fieldMap.entries()).map(([compositeKey, errorData]) => {
118
+ const parts = compositeKey.split('|');
119
+ const field = parts[0];
120
+ const type = parts[1];
121
+ const extensions = {};
122
+ for (let i = 2; i < parts.length; i++) {
123
+ const [paramType, paramValue] = parts[i]?.split(':', 2) ?? [];
124
+ if (!paramType || paramValue === undefined)
125
+ continue;
126
+ try {
127
+ extensions[paramType] = JSON.parse(paramValue);
128
+ }
129
+ catch {
130
+ extensions[paramType] = paramValue;
131
+ }
132
+ }
133
+ const ErrorClass = createError(code, errorData.message, 400);
134
+ return new ErrorClass({
135
+ field,
136
+ type,
137
+ ...extensions,
138
+ rows: convertToRanges(errorData.rowNumbers),
139
+ });
140
+ }));
141
+ }
142
+ return {
143
+ addCapturedError,
144
+ buildFinalErrors,
145
+ getCount: () => capturedErrorCount,
146
+ hasErrors: () => capturedErrorCount > 0 || hasGenericError(),
147
+ shouldStop: () => isLimitReached || hasGenericError(),
148
+ hasGenericError,
149
+ };
150
+ }
31
151
  export class ImportService {
32
152
  knex;
33
153
  accountability;
@@ -71,37 +191,72 @@ export class ImportService {
71
191
  async importJSON(collection, stream) {
72
192
  const extractJSON = StreamArray.withParser();
73
193
  const nestedActionEvents = [];
74
- return transaction(this.knex, (trx) => {
194
+ const errorTracker = createErrorTracker();
195
+ return transaction(this.knex, async (trx) => {
75
196
  const service = getService(collection, {
76
197
  knex: trx,
77
198
  schema: this.schema,
78
199
  accountability: this.accountability,
79
200
  });
80
- const saveQueue = queue(async (value) => {
81
- return await service.upsertOne(value, { bypassEmitAction: (params) => nestedActionEvents.push(params) });
82
- });
83
- return new Promise((resolve, reject) => {
84
- stream.pipe(extractJSON);
85
- extractJSON.on('data', ({ value }) => {
86
- saveQueue.push(value);
87
- });
88
- extractJSON.on('error', (err) => {
89
- destroyStream(stream);
90
- destroyStream(extractJSON);
91
- reject(new InvalidPayloadError({ reason: err.message }));
92
- });
93
- saveQueue.error((err) => {
94
- reject(err);
95
- });
96
- extractJSON.on('end', () => {
97
- saveQueue.drain(() => {
98
- for (const nestedActionEvent of nestedActionEvents) {
99
- emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
201
+ try {
202
+ await new Promise((resolve, reject) => {
203
+ let rowNumber = 1;
204
+ const saveQueue = queue(async (task) => {
205
+ if (errorTracker.shouldStop())
206
+ return;
207
+ try {
208
+ const result = await service.upsertOne(task.data, {
209
+ bypassEmitAction: (params) => nestedActionEvents.push(params),
210
+ });
211
+ return result;
212
+ }
213
+ catch (error) {
214
+ for (const err of toArray(error)) {
215
+ errorTracker.addCapturedError(err, task.rowNumber);
216
+ if (errorTracker.shouldStop()) {
217
+ break;
218
+ }
219
+ }
220
+ if (errorTracker.shouldStop()) {
221
+ saveQueue.kill();
222
+ destroyStream(stream);
223
+ destroyStream(extractJSON);
224
+ reject();
225
+ }
226
+ return;
100
227
  }
101
- return resolve();
228
+ });
229
+ stream.pipe(extractJSON);
230
+ extractJSON.on('data', ({ value }) => {
231
+ saveQueue.push({ data: value, rowNumber: rowNumber++ });
232
+ });
233
+ extractJSON.on('error', (err) => {
234
+ destroyStream(stream);
235
+ destroyStream(extractJSON);
236
+ reject(new InvalidPayloadError({ reason: err.message }));
237
+ });
238
+ extractJSON.on('end', () => {
239
+ // In case of empty JSON file
240
+ if (!saveQueue.started)
241
+ return resolve();
242
+ saveQueue.drain(() => {
243
+ if (errorTracker.hasErrors()) {
244
+ return reject();
245
+ }
246
+ for (const nestedActionEvent of nestedActionEvents) {
247
+ emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
248
+ }
249
+ return resolve();
250
+ });
102
251
  });
103
252
  });
104
- });
253
+ }
254
+ catch (error) {
255
+ if (!error && errorTracker.hasErrors()) {
256
+ throw errorTracker.buildFinalErrors();
257
+ }
258
+ throw error;
259
+ }
105
260
  });
106
261
  }
107
262
  async importCSV(collection, stream) {
@@ -109,98 +264,129 @@ export class ImportService {
109
264
  if (!tmpFile)
110
265
  throw new Error('Failed to create temporary file for import');
111
266
  const nestedActionEvents = [];
112
- return transaction(this.knex, (trx) => {
267
+ const errorTracker = createErrorTracker();
268
+ return transaction(this.knex, async (trx) => {
113
269
  const service = getService(collection, {
114
270
  knex: trx,
115
271
  schema: this.schema,
116
272
  accountability: this.accountability,
117
273
  });
118
- const saveQueue = queue(async (value) => {
119
- return await service.upsertOne(value, { bypassEmitAction: (action) => nestedActionEvents.push(action) });
120
- });
121
- const transform = (value) => {
122
- if (value.length === 0)
123
- return;
124
- try {
125
- const parsedJson = parseJSON(value);
126
- if (typeof parsedJson === 'number') {
127
- return value;
128
- }
129
- return parsedJson;
130
- }
131
- catch {
132
- return value;
133
- }
134
- };
135
- const PapaOptions = {
136
- header: true,
137
- // Trim whitespaces in headers, including the byte order mark (BOM) zero-width no-break space
138
- transformHeader: (header) => header.trim(),
139
- transform,
140
- };
141
- return new Promise((resolve, reject) => {
142
- const streams = [stream];
143
- const cleanup = (destroy = true) => {
144
- if (destroy) {
145
- for (const stream of streams) {
146
- destroyStream(stream);
274
+ try {
275
+ await new Promise((resolve, reject) => {
276
+ const streams = [stream];
277
+ let rowNumber = 0;
278
+ const cleanup = (destroy = true) => {
279
+ if (destroy) {
280
+ for (const stream of streams) {
281
+ destroyStream(stream);
282
+ }
147
283
  }
148
- }
149
- tmpFile.cleanup().catch(() => {
150
- logger.warn(`Failed to cleanup temporary import file (${tmpFile.path})`);
151
- });
152
- };
153
- saveQueue.error((error) => {
154
- reject(error);
155
- });
156
- const fileWriteStream = createWriteStream(tmpFile.path)
157
- .on('error', (error) => {
158
- cleanup();
159
- reject(new Error('Error while writing import data to temporary file', { cause: error }));
160
- })
161
- .on('finish', () => {
162
- const fileReadStream = createReadStream(tmpFile.path).on('error', (error) => {
163
- cleanup();
164
- reject(new Error('Error while reading import data from temporary file', { cause: error }));
165
- });
166
- streams.push(fileReadStream);
167
- fileReadStream
168
- .pipe(Papa.parse(Papa.NODE_STREAM_INPUT, PapaOptions))
169
- .on('data', (obj) => {
170
- const result = {};
171
- // Filter out all undefined fields
172
- for (const field in obj) {
173
- if (obj[field] !== undefined) {
174
- set(result, field, obj[field]);
284
+ tmpFile.cleanup().catch(() => {
285
+ logger.warn(`Failed to cleanup temporary import file (${tmpFile.path})`);
286
+ });
287
+ };
288
+ const saveQueue = queue(async (task) => {
289
+ if (errorTracker.shouldStop())
290
+ return;
291
+ try {
292
+ const result = await service.upsertOne(task.data, {
293
+ bypassEmitAction: (action) => nestedActionEvents.push(action),
294
+ });
295
+ return result;
296
+ }
297
+ catch (error) {
298
+ for (const err of toArray(error)) {
299
+ errorTracker.addCapturedError(err, task.rowNumber);
300
+ if (errorTracker.shouldStop()) {
301
+ break;
302
+ }
303
+ }
304
+ if (errorTracker.shouldStop()) {
305
+ saveQueue.kill();
306
+ cleanup(true);
307
+ reject();
175
308
  }
309
+ return;
176
310
  }
177
- saveQueue.push(result);
178
- })
311
+ });
312
+ const fileWriteStream = createWriteStream(tmpFile.path)
179
313
  .on('error', (error) => {
180
314
  cleanup();
181
- reject(new InvalidPayloadError({ reason: error.message }));
315
+ reject(new Error('Error while writing import data to temporary file', { cause: error }));
182
316
  })
183
- .on('end', () => {
184
- cleanup(false);
185
- // In case of empty CSV file
186
- if (!saveQueue.started)
187
- return resolve();
188
- saveQueue.drain(() => {
189
- for (const nestedActionEvent of nestedActionEvents) {
190
- emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
317
+ .on('finish', () => {
318
+ const fileReadStream = createReadStream(tmpFile.path).on('error', (error) => {
319
+ cleanup();
320
+ reject(new Error('Error while reading import data from temporary file', { cause: error }));
321
+ });
322
+ streams.push(fileReadStream);
323
+ fileReadStream
324
+ .pipe(Papa.parse(Papa.NODE_STREAM_INPUT, {
325
+ header: true,
326
+ transformHeader: (header) => header.trim(),
327
+ transform: (value) => {
328
+ if (value.length === 0)
329
+ return;
330
+ try {
331
+ const parsedJson = parseJSON(value);
332
+ if (typeof parsedJson === 'number') {
333
+ return value;
334
+ }
335
+ return parsedJson;
336
+ }
337
+ catch {
338
+ return value;
339
+ }
340
+ },
341
+ }))
342
+ .on('data', (obj) => {
343
+ rowNumber++;
344
+ const result = {};
345
+ for (const field in obj) {
346
+ if (obj[field] !== undefined) {
347
+ set(result, field, obj[field]);
348
+ }
191
349
  }
192
- return resolve();
350
+ saveQueue.push({ data: result, rowNumber });
351
+ })
352
+ .on('error', (error) => {
353
+ cleanup();
354
+ reject(new InvalidPayloadError({ reason: error.message }));
355
+ })
356
+ .on('end', () => {
357
+ // In case of empty CSV file
358
+ if (!saveQueue.started) {
359
+ cleanup(false);
360
+ return resolve();
361
+ }
362
+ saveQueue.drain(() => {
363
+ if (!errorTracker.shouldStop())
364
+ cleanup(false);
365
+ if (errorTracker.hasErrors()) {
366
+ return reject();
367
+ }
368
+ for (const nestedActionEvent of nestedActionEvents) {
369
+ emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context);
370
+ }
371
+ return resolve();
372
+ });
193
373
  });
194
374
  });
375
+ streams.push(fileWriteStream);
376
+ stream
377
+ .on('error', (error) => {
378
+ cleanup();
379
+ reject(new Error('Error while retrieving import data', { cause: error }));
380
+ })
381
+ .pipe(fileWriteStream);
195
382
  });
196
- streams.push(fileWriteStream);
197
- stream
198
- .on('error', (error) => {
199
- cleanup();
200
- reject(new Error('Error while retrieving import data', { cause: error }));
201
- })
202
- .pipe(fileWriteStream);
203
- });
383
+ }
384
+ catch (error) {
385
+ if (!error && errorTracker.hasErrors()) {
386
+ throw errorTracker.buildFinalErrors();
387
+ }
388
+ throw error;
389
+ }
204
390
  });
205
391
  }
206
392
  }
@@ -1,4 +1,4 @@
1
- import type { AbstractService, AbstractServiceOptions, Accountability, Item as AnyItem, MutationTracker, MutationOptions, PrimaryKey, Query, QueryOptions, SchemaOverview } from '@directus/types';
1
+ import type { AbstractService, AbstractServiceOptions, Accountability, Item as AnyItem, MutationOptions, MutationTracker, PrimaryKey, Query, QueryOptions, SchemaOverview } from '@directus/types';
2
2
  import type Keyv from 'keyv';
3
3
  import type { Knex } from 'knex';
4
4
  export declare class ItemsService<Item extends AnyItem = AnyItem, Collection extends string = string> implements AbstractService<Item> {