directus 9.5.1 → 9.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app.js +3 -1
- package/dist/auth/drivers/ldap.d.ts +0 -1
- package/dist/auth/drivers/ldap.js +54 -60
- package/dist/auth/drivers/oauth2.js +3 -0
- package/dist/auth/drivers/openid.js +9 -6
- package/dist/cache.d.ts +4 -1
- package/dist/cache.js +27 -5
- package/dist/cli/commands/init/index.js +8 -0
- package/dist/cli/commands/schema/apply.d.ts +1 -0
- package/dist/cli/commands/schema/apply.js +9 -5
- package/dist/cli/index.js +1 -0
- package/dist/cli/utils/create-env/env-stub.liquid +1 -0
- package/dist/controllers/assets.js +9 -1
- package/dist/controllers/utils.js +18 -1
- package/dist/database/migrations/20220303A-remove-default-project-color.d.ts +3 -0
- package/dist/database/migrations/20220303A-remove-default-project-color.js +22 -0
- package/dist/database/run-ast.d.ts +1 -1
- package/dist/database/run-ast.js +48 -35
- package/dist/database/system-data/app-access-permissions/app-access-permissions.yaml +8 -2
- package/dist/database/system-data/fields/collections.yaml +2 -0
- package/dist/database/system-data/fields/settings.yaml +19 -3
- package/dist/env.js +8 -2
- package/dist/exceptions/database/dialects/mysql.js +23 -17
- package/dist/extensions.js +0 -2
- package/dist/middleware/authenticate.d.ts +5 -3
- package/dist/middleware/authenticate.js +22 -39
- package/dist/middleware/respond.js +7 -28
- package/dist/server.js +5 -2
- package/dist/services/authorization.js +2 -1
- package/dist/services/collections.js +9 -7
- package/dist/services/fields.js +8 -8
- package/dist/services/files.js +69 -3
- package/dist/services/graphql.js +2 -2
- package/dist/services/import-export.d.ts +34 -0
- package/dist/services/import-export.js +270 -0
- package/dist/services/index.d.ts +2 -1
- package/dist/services/index.js +2 -1
- package/dist/services/mail/templates/base.liquid +2 -2
- package/dist/services/permissions.js +10 -10
- package/dist/services/relations.js +11 -4
- package/dist/services/utils.js +10 -0
- package/dist/utils/apply-query.js +2 -24
- package/dist/utils/get-date-formatted.d.ts +1 -0
- package/dist/utils/get-date-formatted.js +14 -0
- package/dist/utils/get-permissions.js +1 -1
- package/dist/utils/get-schema.js +1 -1
- package/dist/utils/is-directus-jwt.js +4 -21
- package/dist/utils/jwt.d.ts +2 -0
- package/dist/utils/jwt.js +49 -0
- package/dist/utils/merge-permissions.js +18 -6
- package/example.env +1 -0
- package/package.json +19 -20
- package/dist/__mocks__/cache.d.ts +0 -6
- package/dist/__mocks__/cache.js +0 -8
- package/dist/services/import.d.ts +0 -13
- package/dist/services/import.js +0 -118
|
@@ -129,7 +129,7 @@ class CollectionsService {
|
|
|
129
129
|
if (this.cache && env_1.default.CACHE_AUTO_PURGE && (opts === null || opts === void 0 ? void 0 : opts.autoPurgeCache) !== false) {
|
|
130
130
|
await this.cache.clear();
|
|
131
131
|
}
|
|
132
|
-
await
|
|
132
|
+
await (0, cache_1.clearSystemCache)();
|
|
133
133
|
return payload.collection;
|
|
134
134
|
}
|
|
135
135
|
/**
|
|
@@ -152,7 +152,7 @@ class CollectionsService {
|
|
|
152
152
|
if (this.cache && env_1.default.CACHE_AUTO_PURGE && (opts === null || opts === void 0 ? void 0 : opts.autoPurgeCache) !== false) {
|
|
153
153
|
await this.cache.clear();
|
|
154
154
|
}
|
|
155
|
-
await
|
|
155
|
+
await (0, cache_1.clearSystemCache)();
|
|
156
156
|
return collections;
|
|
157
157
|
}
|
|
158
158
|
/**
|
|
@@ -275,7 +275,7 @@ class CollectionsService {
|
|
|
275
275
|
if (this.cache && env_1.default.CACHE_AUTO_PURGE && (opts === null || opts === void 0 ? void 0 : opts.autoPurgeCache) !== false) {
|
|
276
276
|
await this.cache.clear();
|
|
277
277
|
}
|
|
278
|
-
await
|
|
278
|
+
await (0, cache_1.clearSystemCache)();
|
|
279
279
|
return collectionKey;
|
|
280
280
|
}
|
|
281
281
|
/**
|
|
@@ -298,7 +298,7 @@ class CollectionsService {
|
|
|
298
298
|
if (this.cache && env_1.default.CACHE_AUTO_PURGE && (opts === null || opts === void 0 ? void 0 : opts.autoPurgeCache) !== false) {
|
|
299
299
|
await this.cache.clear();
|
|
300
300
|
}
|
|
301
|
-
await
|
|
301
|
+
await (0, cache_1.clearSystemCache)();
|
|
302
302
|
return collectionKeys;
|
|
303
303
|
}
|
|
304
304
|
/**
|
|
@@ -316,7 +316,9 @@ class CollectionsService {
|
|
|
316
316
|
}
|
|
317
317
|
await this.knex.transaction(async (trx) => {
|
|
318
318
|
var _a;
|
|
319
|
-
|
|
319
|
+
if (collectionToBeDeleted.schema) {
|
|
320
|
+
await trx.schema.dropTable(collectionKey);
|
|
321
|
+
}
|
|
320
322
|
// Make sure this collection isn't used as a group in any other collections
|
|
321
323
|
await trx('directus_collections').update({ group: null }).where({ group: collectionKey });
|
|
322
324
|
if (collectionToBeDeleted.meta) {
|
|
@@ -377,7 +379,7 @@ class CollectionsService {
|
|
|
377
379
|
if (this.cache && env_1.default.CACHE_AUTO_PURGE && (opts === null || opts === void 0 ? void 0 : opts.autoPurgeCache) !== false) {
|
|
378
380
|
await this.cache.clear();
|
|
379
381
|
}
|
|
380
|
-
await
|
|
382
|
+
await (0, cache_1.clearSystemCache)();
|
|
381
383
|
return collectionKey;
|
|
382
384
|
}
|
|
383
385
|
/**
|
|
@@ -400,7 +402,7 @@ class CollectionsService {
|
|
|
400
402
|
if (this.cache && env_1.default.CACHE_AUTO_PURGE && (opts === null || opts === void 0 ? void 0 : opts.autoPurgeCache) !== false) {
|
|
401
403
|
await this.cache.clear();
|
|
402
404
|
}
|
|
403
|
-
await
|
|
405
|
+
await (0, cache_1.clearSystemCache)();
|
|
404
406
|
return collectionKeys;
|
|
405
407
|
}
|
|
406
408
|
}
|
package/dist/services/fields.js
CHANGED
|
@@ -249,7 +249,7 @@ class FieldsService {
|
|
|
249
249
|
if (this.cache && env_1.default.CACHE_AUTO_PURGE) {
|
|
250
250
|
await this.cache.clear();
|
|
251
251
|
}
|
|
252
|
-
await
|
|
252
|
+
await (0, cache_1.clearSystemCache)();
|
|
253
253
|
}
|
|
254
254
|
async updateField(collection, field) {
|
|
255
255
|
if (this.accountability && this.accountability.admin !== true) {
|
|
@@ -300,7 +300,7 @@ class FieldsService {
|
|
|
300
300
|
if (this.cache && env_1.default.CACHE_AUTO_PURGE) {
|
|
301
301
|
await this.cache.clear();
|
|
302
302
|
}
|
|
303
|
-
await
|
|
303
|
+
await (0, cache_1.clearSystemCache)();
|
|
304
304
|
emitter_1.default.emitAction(`fields.update`, {
|
|
305
305
|
payload: hookAdjustedField,
|
|
306
306
|
keys: [hookAdjustedField.field],
|
|
@@ -395,7 +395,7 @@ class FieldsService {
|
|
|
395
395
|
if (this.cache && env_1.default.CACHE_AUTO_PURGE) {
|
|
396
396
|
await this.cache.clear();
|
|
397
397
|
}
|
|
398
|
-
await
|
|
398
|
+
await (0, cache_1.clearSystemCache)();
|
|
399
399
|
emitter_1.default.emitAction('fields.delete', {
|
|
400
400
|
payload: [field],
|
|
401
401
|
collection: collection,
|
|
@@ -406,7 +406,7 @@ class FieldsService {
|
|
|
406
406
|
});
|
|
407
407
|
}
|
|
408
408
|
addColumnToTable(table, field, alter = null) {
|
|
409
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k
|
|
409
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
|
|
410
410
|
let column;
|
|
411
411
|
// Don't attempt to add a DB column for alias / corrupt fields
|
|
412
412
|
if (field.type === 'alias' || field.type === 'unknown')
|
|
@@ -457,20 +457,20 @@ class FieldsService {
|
|
|
457
457
|
column.notNullable();
|
|
458
458
|
}
|
|
459
459
|
}
|
|
460
|
-
else
|
|
460
|
+
else {
|
|
461
461
|
if (!alter || alter.is_nullable === false) {
|
|
462
462
|
column.nullable();
|
|
463
463
|
}
|
|
464
464
|
}
|
|
465
|
-
if ((
|
|
465
|
+
if ((_h = field.schema) === null || _h === void 0 ? void 0 : _h.is_primary_key) {
|
|
466
466
|
column.primary().notNullable();
|
|
467
467
|
}
|
|
468
|
-
else if (((
|
|
468
|
+
else if (((_j = field.schema) === null || _j === void 0 ? void 0 : _j.is_unique) === true) {
|
|
469
469
|
if (!alter || alter.is_unique === false) {
|
|
470
470
|
column.unique();
|
|
471
471
|
}
|
|
472
472
|
}
|
|
473
|
-
else if (((
|
|
473
|
+
else if (((_k = field.schema) === null || _k === void 0 ? void 0 : _k.is_unique) === false) {
|
|
474
474
|
if (alter && alter.is_unique === true) {
|
|
475
475
|
table.dropUnique([field.field]);
|
|
476
476
|
}
|
package/dist/services/files.js
CHANGED
|
@@ -1,4 +1,23 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
|
5
|
+
}) : (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
o[k2] = m[k];
|
|
8
|
+
}));
|
|
9
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
10
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
11
|
+
}) : function(o, v) {
|
|
12
|
+
o["default"] = v;
|
|
13
|
+
});
|
|
14
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
15
|
+
if (mod && mod.__esModule) return mod;
|
|
16
|
+
var result = {};
|
|
17
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
18
|
+
__setModuleDefault(result, mod);
|
|
19
|
+
return result;
|
|
20
|
+
};
|
|
2
21
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
22
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
23
|
};
|
|
@@ -11,7 +30,9 @@ const lodash_1 = require("lodash");
|
|
|
11
30
|
const mime_types_1 = require("mime-types");
|
|
12
31
|
const path_1 = __importDefault(require("path"));
|
|
13
32
|
const sharp_1 = __importDefault(require("sharp"));
|
|
14
|
-
const url_1 =
|
|
33
|
+
const url_1 = __importStar(require("url"));
|
|
34
|
+
const util_1 = require("util");
|
|
35
|
+
const dns_1 = require("dns");
|
|
15
36
|
const emitter_1 = __importDefault(require("../emitter"));
|
|
16
37
|
const env_1 = __importDefault(require("../env"));
|
|
17
38
|
const exceptions_1 = require("../exceptions");
|
|
@@ -19,6 +40,9 @@ const logger_1 = __importDefault(require("../logger"));
|
|
|
19
40
|
const storage_1 = __importDefault(require("../storage"));
|
|
20
41
|
const utils_1 = require("@directus/shared/utils");
|
|
21
42
|
const items_1 = require("./items");
|
|
43
|
+
const net_1 = __importDefault(require("net"));
|
|
44
|
+
const os_1 = __importDefault(require("os"));
|
|
45
|
+
const lookupDNS = (0, util_1.promisify)(dns_1.lookup);
|
|
22
46
|
class FilesService extends items_1.ItemsService {
|
|
23
47
|
constructor(options) {
|
|
24
48
|
super('directus_files', options);
|
|
@@ -137,6 +161,49 @@ class FilesService extends items_1.ItemsService {
|
|
|
137
161
|
if (this.accountability && ((_c = this.accountability) === null || _c === void 0 ? void 0 : _c.admin) !== true && !fileCreatePermissions) {
|
|
138
162
|
throw new exceptions_1.ForbiddenException();
|
|
139
163
|
}
|
|
164
|
+
let resolvedUrl;
|
|
165
|
+
try {
|
|
166
|
+
resolvedUrl = new url_1.URL(importURL);
|
|
167
|
+
}
|
|
168
|
+
catch (err) {
|
|
169
|
+
logger_1.default.warn(err, `Requested URL ${importURL} isn't a valid URL`);
|
|
170
|
+
throw new exceptions_1.ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
|
|
171
|
+
service: 'external-file',
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
let ip = resolvedUrl.hostname;
|
|
175
|
+
if (net_1.default.isIP(ip) === 0) {
|
|
176
|
+
try {
|
|
177
|
+
ip = (await lookupDNS(ip)).address;
|
|
178
|
+
}
|
|
179
|
+
catch (err) {
|
|
180
|
+
logger_1.default.warn(err, `Couldn't lookup the DNS for url ${importURL}`);
|
|
181
|
+
throw new exceptions_1.ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
|
|
182
|
+
service: 'external-file',
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
if (env_1.default.IMPORT_IP_DENY_LIST.includes('0.0.0.0')) {
|
|
187
|
+
const networkInterfaces = os_1.default.networkInterfaces();
|
|
188
|
+
for (const networkInfo of Object.values(networkInterfaces)) {
|
|
189
|
+
if (!networkInfo)
|
|
190
|
+
continue;
|
|
191
|
+
for (const info of networkInfo) {
|
|
192
|
+
if (info.address === ip) {
|
|
193
|
+
logger_1.default.warn(`Requested URL ${importURL} resolves to localhost.`);
|
|
194
|
+
throw new exceptions_1.ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
|
|
195
|
+
service: 'external-file',
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
if (env_1.default.IMPORT_IP_DENY_LIST.includes(ip)) {
|
|
202
|
+
logger_1.default.warn(`Requested URL ${importURL} resolves to a denied IP address.`);
|
|
203
|
+
throw new exceptions_1.ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
|
|
204
|
+
service: 'external-file',
|
|
205
|
+
});
|
|
206
|
+
}
|
|
140
207
|
let fileResponse;
|
|
141
208
|
try {
|
|
142
209
|
fileResponse = await axios_1.default.get(importURL, {
|
|
@@ -144,8 +211,7 @@ class FilesService extends items_1.ItemsService {
|
|
|
144
211
|
});
|
|
145
212
|
}
|
|
146
213
|
catch (err) {
|
|
147
|
-
logger_1.default.warn(`Couldn't fetch file from url "${importURL}"`);
|
|
148
|
-
logger_1.default.warn(err);
|
|
214
|
+
logger_1.default.warn(err, `Couldn't fetch file from url "${importURL}"`);
|
|
149
215
|
throw new exceptions_1.ServiceUnavailableException(`Couldn't fetch file from url "${importURL}"`, {
|
|
150
216
|
service: 'external-file',
|
|
151
217
|
});
|
package/dist/services/graphql.js
CHANGED
|
@@ -1732,9 +1732,9 @@ class GraphQLService {
|
|
|
1732
1732
|
if (((_a = this.accountability) === null || _a === void 0 ? void 0 : _a.admin) !== true) {
|
|
1733
1733
|
throw new exceptions_1.ForbiddenException();
|
|
1734
1734
|
}
|
|
1735
|
-
const { cache
|
|
1735
|
+
const { cache } = (0, cache_1.getCache)();
|
|
1736
1736
|
await (cache === null || cache === void 0 ? void 0 : cache.clear());
|
|
1737
|
-
await
|
|
1737
|
+
await (0, cache_1.clearSystemCache)();
|
|
1738
1738
|
return;
|
|
1739
1739
|
},
|
|
1740
1740
|
},
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { Knex } from 'knex';
|
|
3
|
+
import { AbstractServiceOptions, File } from '../types';
|
|
4
|
+
import { Accountability, Query, SchemaOverview } from '@directus/shared/types';
|
|
5
|
+
export declare class ImportService {
|
|
6
|
+
knex: Knex;
|
|
7
|
+
accountability: Accountability | null;
|
|
8
|
+
schema: SchemaOverview;
|
|
9
|
+
constructor(options: AbstractServiceOptions);
|
|
10
|
+
import(collection: string, mimetype: string, stream: NodeJS.ReadableStream): Promise<void>;
|
|
11
|
+
importJSON(collection: string, stream: NodeJS.ReadableStream): Promise<void>;
|
|
12
|
+
importCSV(collection: string, stream: NodeJS.ReadableStream): Promise<void>;
|
|
13
|
+
}
|
|
14
|
+
export declare class ExportService {
|
|
15
|
+
knex: Knex;
|
|
16
|
+
accountability: Accountability | null;
|
|
17
|
+
schema: SchemaOverview;
|
|
18
|
+
constructor(options: AbstractServiceOptions);
|
|
19
|
+
/**
|
|
20
|
+
* Export the query results as a named file. Will query in batches, and keep appending a tmp file
|
|
21
|
+
* until all the data is retrieved. Uploads the result as a new file using the regular
|
|
22
|
+
* FilesService upload method.
|
|
23
|
+
*/
|
|
24
|
+
exportToFile(collection: string, query: Partial<Query>, format: 'xml' | 'csv' | 'json', options?: {
|
|
25
|
+
file?: Partial<File>;
|
|
26
|
+
}): Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* Transform a given input object / array to the given type
|
|
29
|
+
*/
|
|
30
|
+
transform(input: Record<string, any>[], format: 'xml' | 'csv' | 'json', options?: {
|
|
31
|
+
includeHeader?: boolean;
|
|
32
|
+
includeFooter?: boolean;
|
|
33
|
+
}): string;
|
|
34
|
+
}
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.ExportService = exports.ImportService = void 0;
|
|
7
|
+
const database_1 = __importDefault(require("../database"));
|
|
8
|
+
const exceptions_1 = require("../exceptions");
|
|
9
|
+
const StreamArray_1 = __importDefault(require("stream-json/streamers/StreamArray"));
|
|
10
|
+
const items_1 = require("./items");
|
|
11
|
+
const async_1 = require("async");
|
|
12
|
+
const destroy_1 = __importDefault(require("destroy"));
|
|
13
|
+
const csv_parser_1 = __importDefault(require("csv-parser"));
|
|
14
|
+
const lodash_1 = require("lodash");
|
|
15
|
+
const js2xmlparser_1 = require("js2xmlparser");
|
|
16
|
+
const json2csv_1 = require("json2csv");
|
|
17
|
+
const fs_extra_1 = require("fs-extra");
|
|
18
|
+
const tmp_promise_1 = require("tmp-promise");
|
|
19
|
+
const env_1 = __importDefault(require("../env"));
|
|
20
|
+
const files_1 = require("./files");
|
|
21
|
+
const get_date_formatted_1 = require("../utils/get-date-formatted");
|
|
22
|
+
const utils_1 = require("@directus/shared/utils");
|
|
23
|
+
const notifications_1 = require("./notifications");
|
|
24
|
+
const logger_1 = __importDefault(require("../logger"));
|
|
25
|
+
class ImportService {
|
|
26
|
+
constructor(options) {
|
|
27
|
+
this.knex = options.knex || (0, database_1.default)();
|
|
28
|
+
this.accountability = options.accountability || null;
|
|
29
|
+
this.schema = options.schema;
|
|
30
|
+
}
|
|
31
|
+
async import(collection, mimetype, stream) {
|
|
32
|
+
var _a, _b, _c, _d, _e;
|
|
33
|
+
if (collection.startsWith('directus_'))
|
|
34
|
+
throw new exceptions_1.ForbiddenException();
|
|
35
|
+
const createPermissions = (_b = (_a = this.accountability) === null || _a === void 0 ? void 0 : _a.permissions) === null || _b === void 0 ? void 0 : _b.find((permission) => permission.collection === collection && permission.action === 'create');
|
|
36
|
+
const updatePermissions = (_d = (_c = this.accountability) === null || _c === void 0 ? void 0 : _c.permissions) === null || _d === void 0 ? void 0 : _d.find((permission) => permission.collection === collection && permission.action === 'update');
|
|
37
|
+
if (((_e = this.accountability) === null || _e === void 0 ? void 0 : _e.admin) !== true && (!createPermissions || !updatePermissions)) {
|
|
38
|
+
throw new exceptions_1.ForbiddenException();
|
|
39
|
+
}
|
|
40
|
+
switch (mimetype) {
|
|
41
|
+
case 'application/json':
|
|
42
|
+
return await this.importJSON(collection, stream);
|
|
43
|
+
case 'text/csv':
|
|
44
|
+
case 'application/vnd.ms-excel':
|
|
45
|
+
return await this.importCSV(collection, stream);
|
|
46
|
+
default:
|
|
47
|
+
throw new exceptions_1.UnsupportedMediaTypeException(`Can't import files of type "${mimetype}"`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
importJSON(collection, stream) {
|
|
51
|
+
const extractJSON = StreamArray_1.default.withParser();
|
|
52
|
+
return this.knex.transaction((trx) => {
|
|
53
|
+
const service = new items_1.ItemsService(collection, {
|
|
54
|
+
knex: trx,
|
|
55
|
+
schema: this.schema,
|
|
56
|
+
accountability: this.accountability,
|
|
57
|
+
});
|
|
58
|
+
const saveQueue = (0, async_1.queue)(async (value) => {
|
|
59
|
+
return await service.upsertOne(value);
|
|
60
|
+
});
|
|
61
|
+
return new Promise((resolve, reject) => {
|
|
62
|
+
stream.pipe(extractJSON);
|
|
63
|
+
extractJSON.on('data', ({ value }) => {
|
|
64
|
+
saveQueue.push(value);
|
|
65
|
+
});
|
|
66
|
+
extractJSON.on('error', (err) => {
|
|
67
|
+
(0, destroy_1.default)(stream);
|
|
68
|
+
(0, destroy_1.default)(extractJSON);
|
|
69
|
+
reject(new exceptions_1.InvalidPayloadException(err.message));
|
|
70
|
+
});
|
|
71
|
+
saveQueue.error((err) => {
|
|
72
|
+
reject(err);
|
|
73
|
+
});
|
|
74
|
+
extractJSON.on('end', () => {
|
|
75
|
+
saveQueue.drain(() => {
|
|
76
|
+
return resolve();
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
importCSV(collection, stream) {
|
|
83
|
+
return this.knex.transaction((trx) => {
|
|
84
|
+
const service = new items_1.ItemsService(collection, {
|
|
85
|
+
knex: trx,
|
|
86
|
+
schema: this.schema,
|
|
87
|
+
accountability: this.accountability,
|
|
88
|
+
});
|
|
89
|
+
const saveQueue = (0, async_1.queue)(async (value) => {
|
|
90
|
+
return await service.upsertOne(value);
|
|
91
|
+
});
|
|
92
|
+
return new Promise((resolve, reject) => {
|
|
93
|
+
stream
|
|
94
|
+
.pipe((0, csv_parser_1.default)())
|
|
95
|
+
.on('data', (value) => {
|
|
96
|
+
const obj = (0, lodash_1.transform)(value, (result, value, key) => {
|
|
97
|
+
if (value.length === 0) {
|
|
98
|
+
delete result[key];
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
try {
|
|
102
|
+
const parsedJson = JSON.parse(value);
|
|
103
|
+
(0, lodash_1.set)(result, key, parsedJson);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
(0, lodash_1.set)(result, key, value);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
saveQueue.push(obj);
|
|
111
|
+
})
|
|
112
|
+
.on('error', (err) => {
|
|
113
|
+
(0, destroy_1.default)(stream);
|
|
114
|
+
reject(new exceptions_1.InvalidPayloadException(err.message));
|
|
115
|
+
})
|
|
116
|
+
.on('end', () => {
|
|
117
|
+
saveQueue.drain(() => {
|
|
118
|
+
return resolve();
|
|
119
|
+
});
|
|
120
|
+
});
|
|
121
|
+
saveQueue.error((err) => {
|
|
122
|
+
reject(err);
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
exports.ImportService = ImportService;
|
|
129
|
+
class ExportService {
|
|
130
|
+
constructor(options) {
|
|
131
|
+
this.knex = options.knex || (0, database_1.default)();
|
|
132
|
+
this.accountability = options.accountability || null;
|
|
133
|
+
this.schema = options.schema;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Export the query results as a named file. Will query in batches, and keep appending a tmp file
|
|
137
|
+
* until all the data is retrieved. Uploads the result as a new file using the regular
|
|
138
|
+
* FilesService upload method.
|
|
139
|
+
*/
|
|
140
|
+
async exportToFile(collection, query, format, options) {
|
|
141
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j;
|
|
142
|
+
try {
|
|
143
|
+
const mimeTypes = {
|
|
144
|
+
xml: 'text/xml',
|
|
145
|
+
csv: 'text/csv',
|
|
146
|
+
json: 'application/json',
|
|
147
|
+
};
|
|
148
|
+
const database = (0, database_1.default)();
|
|
149
|
+
const { path, cleanup } = await (0, tmp_promise_1.file)();
|
|
150
|
+
await database.transaction(async (trx) => {
|
|
151
|
+
var _a;
|
|
152
|
+
const service = new items_1.ItemsService(collection, {
|
|
153
|
+
accountability: this.accountability,
|
|
154
|
+
schema: this.schema,
|
|
155
|
+
knex: trx,
|
|
156
|
+
});
|
|
157
|
+
const totalCount = await service
|
|
158
|
+
.readByQuery({
|
|
159
|
+
...query,
|
|
160
|
+
aggregate: {
|
|
161
|
+
count: ['*'],
|
|
162
|
+
},
|
|
163
|
+
})
|
|
164
|
+
.then((result) => { var _a, _b; return Number((_b = (_a = result === null || result === void 0 ? void 0 : result[0]) === null || _a === void 0 ? void 0 : _a.count) !== null && _b !== void 0 ? _b : 0); });
|
|
165
|
+
const count = query.limit ? Math.min(totalCount, query.limit) : totalCount;
|
|
166
|
+
const requestedLimit = (_a = query.limit) !== null && _a !== void 0 ? _a : -1;
|
|
167
|
+
const batchesRequired = Math.ceil(count / env_1.default.EXPORT_BATCH_SIZE);
|
|
168
|
+
let readCount = 0;
|
|
169
|
+
for (let batch = 0; batch <= batchesRequired; batch++) {
|
|
170
|
+
let limit = env_1.default.EXPORT_BATCH_SIZE;
|
|
171
|
+
if (requestedLimit > 0 && env_1.default.EXPORT_BATCH_SIZE > requestedLimit - readCount) {
|
|
172
|
+
limit = requestedLimit - readCount;
|
|
173
|
+
}
|
|
174
|
+
const result = await service.readByQuery({
|
|
175
|
+
...query,
|
|
176
|
+
limit,
|
|
177
|
+
page: batch,
|
|
178
|
+
});
|
|
179
|
+
readCount += result.length;
|
|
180
|
+
if (result.length) {
|
|
181
|
+
await (0, fs_extra_1.appendFile)(path, this.transform(result, format, {
|
|
182
|
+
includeHeader: batch === 0,
|
|
183
|
+
includeFooter: batch + 1 === batchesRequired,
|
|
184
|
+
}));
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
const filesService = new files_1.FilesService({
|
|
189
|
+
accountability: this.accountability,
|
|
190
|
+
schema: this.schema,
|
|
191
|
+
});
|
|
192
|
+
const storage = (0, utils_1.toArray)(env_1.default.STORAGE_LOCATIONS)[0];
|
|
193
|
+
const title = `export-${collection}-${(0, get_date_formatted_1.getDateFormatted)()}`;
|
|
194
|
+
const filename = `${title}.${format}`;
|
|
195
|
+
const fileWithDefaults = {
|
|
196
|
+
...((_a = options === null || options === void 0 ? void 0 : options.file) !== null && _a !== void 0 ? _a : {}),
|
|
197
|
+
title: (_c = (_b = options === null || options === void 0 ? void 0 : options.file) === null || _b === void 0 ? void 0 : _b.title) !== null && _c !== void 0 ? _c : title,
|
|
198
|
+
filename_download: (_e = (_d = options === null || options === void 0 ? void 0 : options.file) === null || _d === void 0 ? void 0 : _d.filename_download) !== null && _e !== void 0 ? _e : filename,
|
|
199
|
+
storage: (_g = (_f = options === null || options === void 0 ? void 0 : options.file) === null || _f === void 0 ? void 0 : _f.storage) !== null && _g !== void 0 ? _g : storage,
|
|
200
|
+
type: mimeTypes[format],
|
|
201
|
+
};
|
|
202
|
+
const savedFile = await filesService.uploadOne((0, fs_extra_1.createReadStream)(path), fileWithDefaults);
|
|
203
|
+
if ((_h = this.accountability) === null || _h === void 0 ? void 0 : _h.user) {
|
|
204
|
+
const notificationsService = new notifications_1.NotificationsService({
|
|
205
|
+
accountability: this.accountability,
|
|
206
|
+
schema: this.schema,
|
|
207
|
+
});
|
|
208
|
+
await notificationsService.createOne({
|
|
209
|
+
recipient: this.accountability.user,
|
|
210
|
+
sender: this.accountability.user,
|
|
211
|
+
subject: `Your export of ${collection} is ready`,
|
|
212
|
+
collection: `directus_files`,
|
|
213
|
+
item: savedFile,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
await cleanup();
|
|
217
|
+
}
|
|
218
|
+
catch (err) {
|
|
219
|
+
logger_1.default.error(err, `Couldn't export ${collection}: ${err.message}`);
|
|
220
|
+
if ((_j = this.accountability) === null || _j === void 0 ? void 0 : _j.user) {
|
|
221
|
+
const notificationsService = new notifications_1.NotificationsService({
|
|
222
|
+
accountability: this.accountability,
|
|
223
|
+
schema: this.schema,
|
|
224
|
+
});
|
|
225
|
+
await notificationsService.createOne({
|
|
226
|
+
recipient: this.accountability.user,
|
|
227
|
+
sender: this.accountability.user,
|
|
228
|
+
subject: `Your export of ${collection} failed`,
|
|
229
|
+
message: `Please contact your system administrator for more information.`,
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Transform a given input object / array to the given type
|
|
236
|
+
*/
|
|
237
|
+
transform(input, format, options) {
|
|
238
|
+
if (format === 'json') {
|
|
239
|
+
let string = JSON.stringify(input || null, null, '\t');
|
|
240
|
+
if ((options === null || options === void 0 ? void 0 : options.includeHeader) === false)
|
|
241
|
+
string = string.split('\n').slice(1).join('\n');
|
|
242
|
+
if ((options === null || options === void 0 ? void 0 : options.includeFooter) === false) {
|
|
243
|
+
const lines = string.split('\n');
|
|
244
|
+
string = lines.slice(0, lines.length - 1).join('\n');
|
|
245
|
+
string += ',\n';
|
|
246
|
+
}
|
|
247
|
+
return string;
|
|
248
|
+
}
|
|
249
|
+
if (format === 'xml') {
|
|
250
|
+
let string = (0, js2xmlparser_1.parse)('data', input);
|
|
251
|
+
if ((options === null || options === void 0 ? void 0 : options.includeHeader) === false)
|
|
252
|
+
string = string.split('\n').slice(2).join('\n');
|
|
253
|
+
if ((options === null || options === void 0 ? void 0 : options.includeFooter) === false) {
|
|
254
|
+
const lines = string.split('\n');
|
|
255
|
+
string = lines.slice(0, lines.length - 1).join('\n');
|
|
256
|
+
string += '\n';
|
|
257
|
+
}
|
|
258
|
+
return string;
|
|
259
|
+
}
|
|
260
|
+
if (format === 'csv') {
|
|
261
|
+
const parser = new json2csv_1.Parser({
|
|
262
|
+
transforms: [json2csv_1.transforms.flatten({ separator: '.' })],
|
|
263
|
+
header: (options === null || options === void 0 ? void 0 : options.includeHeader) !== false,
|
|
264
|
+
});
|
|
265
|
+
return parser.parse(input);
|
|
266
|
+
}
|
|
267
|
+
throw new exceptions_1.ServiceUnavailableException(`Illegal export type used: "${format}"`, { service: 'export' });
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
exports.ExportService = ExportService;
|
package/dist/services/index.d.ts
CHANGED
|
@@ -2,13 +2,14 @@ export * from './items';
|
|
|
2
2
|
export * from './activity';
|
|
3
3
|
export * from './assets';
|
|
4
4
|
export * from './authentication';
|
|
5
|
+
export * from './authorization';
|
|
5
6
|
export * from './collections';
|
|
6
7
|
export * from './dashboards';
|
|
7
8
|
export * from './fields';
|
|
8
9
|
export * from './files';
|
|
9
10
|
export * from './folders';
|
|
10
11
|
export * from './graphql';
|
|
11
|
-
export * from './import';
|
|
12
|
+
export * from './import-export';
|
|
12
13
|
export * from './mail';
|
|
13
14
|
export * from './meta';
|
|
14
15
|
export * from './notifications';
|
package/dist/services/index.js
CHANGED
|
@@ -15,13 +15,14 @@ __exportStar(require("./items"), exports);
|
|
|
15
15
|
__exportStar(require("./activity"), exports);
|
|
16
16
|
__exportStar(require("./assets"), exports);
|
|
17
17
|
__exportStar(require("./authentication"), exports);
|
|
18
|
+
__exportStar(require("./authorization"), exports);
|
|
18
19
|
__exportStar(require("./collections"), exports);
|
|
19
20
|
__exportStar(require("./dashboards"), exports);
|
|
20
21
|
__exportStar(require("./fields"), exports);
|
|
21
22
|
__exportStar(require("./files"), exports);
|
|
22
23
|
__exportStar(require("./folders"), exports);
|
|
23
24
|
__exportStar(require("./graphql"), exports);
|
|
24
|
-
__exportStar(require("./import"), exports);
|
|
25
|
+
__exportStar(require("./import-export"), exports);
|
|
25
26
|
__exportStar(require("./mail"), exports);
|
|
26
27
|
__exportStar(require("./meta"), exports);
|
|
27
28
|
__exportStar(require("./notifications"), exports);
|
|
@@ -35,7 +35,7 @@ a[x-apple-data-detectors] {
|
|
|
35
35
|
line-height: inherit !important;
|
|
36
36
|
}
|
|
37
37
|
body a {
|
|
38
|
-
color: #
|
|
38
|
+
color: #6644ff;
|
|
39
39
|
text-decoration: none;
|
|
40
40
|
}
|
|
41
41
|
hr {
|
|
@@ -74,7 +74,7 @@ hr {
|
|
|
74
74
|
color: #FFFFFF !important;
|
|
75
75
|
}
|
|
76
76
|
.link {
|
|
77
|
-
color: #
|
|
77
|
+
color: #6644ff !important;
|
|
78
78
|
}
|
|
79
79
|
.button {
|
|
80
80
|
background-color:#0BA582 !important;
|
|
@@ -52,52 +52,52 @@ class PermissionsService extends items_1.ItemsService {
|
|
|
52
52
|
}
|
|
53
53
|
async createOne(data, opts) {
|
|
54
54
|
const res = await super.createOne(data, opts);
|
|
55
|
-
await
|
|
55
|
+
await (0, cache_1.clearSystemCache)();
|
|
56
56
|
return res;
|
|
57
57
|
}
|
|
58
58
|
async createMany(data, opts) {
|
|
59
59
|
const res = await super.createMany(data, opts);
|
|
60
|
-
await
|
|
60
|
+
await (0, cache_1.clearSystemCache)();
|
|
61
61
|
return res;
|
|
62
62
|
}
|
|
63
63
|
async updateByQuery(query, data, opts) {
|
|
64
64
|
const res = await super.updateByQuery(query, data, opts);
|
|
65
|
-
await
|
|
65
|
+
await (0, cache_1.clearSystemCache)();
|
|
66
66
|
return res;
|
|
67
67
|
}
|
|
68
68
|
async updateOne(key, data, opts) {
|
|
69
69
|
const res = await super.updateOne(key, data, opts);
|
|
70
|
-
await
|
|
70
|
+
await (0, cache_1.clearSystemCache)();
|
|
71
71
|
return res;
|
|
72
72
|
}
|
|
73
73
|
async updateMany(keys, data, opts) {
|
|
74
74
|
const res = await super.updateMany(keys, data, opts);
|
|
75
|
-
await
|
|
75
|
+
await (0, cache_1.clearSystemCache)();
|
|
76
76
|
return res;
|
|
77
77
|
}
|
|
78
78
|
async upsertOne(payload, opts) {
|
|
79
79
|
const res = await super.upsertOne(payload, opts);
|
|
80
|
-
await
|
|
80
|
+
await (0, cache_1.clearSystemCache)();
|
|
81
81
|
return res;
|
|
82
82
|
}
|
|
83
83
|
async upsertMany(payloads, opts) {
|
|
84
84
|
const res = await super.upsertMany(payloads, opts);
|
|
85
|
-
await
|
|
85
|
+
await (0, cache_1.clearSystemCache)();
|
|
86
86
|
return res;
|
|
87
87
|
}
|
|
88
88
|
async deleteByQuery(query, opts) {
|
|
89
89
|
const res = await super.deleteByQuery(query, opts);
|
|
90
|
-
await
|
|
90
|
+
await (0, cache_1.clearSystemCache)();
|
|
91
91
|
return res;
|
|
92
92
|
}
|
|
93
93
|
async deleteOne(key, opts) {
|
|
94
94
|
const res = await super.deleteOne(key, opts);
|
|
95
|
-
await
|
|
95
|
+
await (0, cache_1.clearSystemCache)();
|
|
96
96
|
return res;
|
|
97
97
|
}
|
|
98
98
|
async deleteMany(keys, opts) {
|
|
99
99
|
const res = await super.deleteMany(keys, opts);
|
|
100
|
-
await
|
|
100
|
+
await (0, cache_1.clearSystemCache)();
|
|
101
101
|
return res;
|
|
102
102
|
}
|
|
103
103
|
}
|