@webexatech/plugin-collection-transfer 0.0.0-stage → 0.1.3-nocobase.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/LICENSE +201 -0
- package/README.md +158 -3
- package/client-v2.d.ts +2 -0
- package/client-v2.js +1 -0
- package/client.d.ts +2 -0
- package/client.js +1 -0
- package/dist/client/949.72b7b97eaf5473f5.js +10 -0
- package/dist/client/SettingsPage.d.ts +3 -0
- package/dist/client/index.d.ts +1 -0
- package/dist/client/index.js +10 -0
- package/dist/client/plugin.d.ts +5 -0
- package/dist/client-v2/907.b7c707602a2b4e6c.js +10 -0
- package/dist/client-v2/CollectionTransferSettings.d.ts +22 -0
- package/dist/client-v2/SettingsPage.d.ts +4 -0
- package/dist/client-v2/index.d.ts +1 -0
- package/dist/client-v2/index.js +10 -0
- package/dist/client-v2/locale.d.ts +7 -0
- package/dist/client-v2/plugin.d.ts +7 -0
- package/dist/externalVersion.js +24 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +48 -0
- package/dist/locale/en-US.json +70 -0
- package/dist/locale/zh-CN.json +70 -0
- package/dist/node_modules/archiver/LICENSE +22 -0
- package/dist/node_modules/archiver/index.js +68 -0
- package/dist/node_modules/archiver/lib/core.js +974 -0
- package/dist/node_modules/archiver/lib/error.js +40 -0
- package/dist/node_modules/archiver/lib/plugins/json.js +110 -0
- package/dist/node_modules/archiver/lib/plugins/tar.js +167 -0
- package/dist/node_modules/archiver/lib/plugins/zip.js +120 -0
- package/dist/node_modules/archiver/node_modules/async/all.js +119 -0
- package/dist/node_modules/archiver/node_modules/async/allLimit.js +46 -0
- package/dist/node_modules/archiver/node_modules/async/allSeries.js +45 -0
- package/dist/node_modules/archiver/node_modules/async/any.js +122 -0
- package/dist/node_modules/archiver/node_modules/async/anyLimit.js +47 -0
- package/dist/node_modules/archiver/node_modules/async/anySeries.js +46 -0
- package/dist/node_modules/archiver/node_modules/async/apply.js +11 -0
- package/dist/node_modules/archiver/node_modules/async/applyEach.js +57 -0
- package/dist/node_modules/archiver/node_modules/async/applyEachSeries.js +37 -0
- package/dist/node_modules/archiver/node_modules/async/asyncify.js +118 -0
- package/dist/node_modules/archiver/node_modules/async/auto.js +333 -0
- package/dist/node_modules/archiver/node_modules/async/autoInject.js +182 -0
- package/dist/node_modules/archiver/node_modules/async/bower.json +17 -0
- package/dist/node_modules/archiver/node_modules/async/cargo.js +63 -0
- package/dist/node_modules/archiver/node_modules/async/cargoQueue.js +71 -0
- package/dist/node_modules/archiver/node_modules/async/compose.js +55 -0
- package/dist/node_modules/archiver/node_modules/async/concat.js +115 -0
- package/dist/node_modules/archiver/node_modules/async/concatLimit.js +60 -0
- package/dist/node_modules/archiver/node_modules/async/concatSeries.js +41 -0
- package/dist/node_modules/archiver/node_modules/async/constant.js +14 -0
- package/dist/node_modules/archiver/node_modules/async/detect.js +96 -0
- package/dist/node_modules/archiver/node_modules/async/detectLimit.js +48 -0
- package/dist/node_modules/archiver/node_modules/async/detectSeries.js +47 -0
- package/dist/node_modules/archiver/node_modules/async/dir.js +43 -0
- package/dist/node_modules/archiver/node_modules/async/dist/async.js +6062 -0
- package/dist/node_modules/archiver/node_modules/async/dist/async.min.js +1 -0
- package/dist/node_modules/archiver/node_modules/async/doDuring.js +68 -0
- package/dist/node_modules/archiver/node_modules/async/doUntil.js +46 -0
- package/dist/node_modules/archiver/node_modules/async/doWhilst.js +68 -0
- package/dist/node_modules/archiver/node_modules/async/during.js +78 -0
- package/dist/node_modules/archiver/node_modules/async/each.js +129 -0
- package/dist/node_modules/archiver/node_modules/async/eachLimit.js +50 -0
- package/dist/node_modules/archiver/node_modules/async/eachOf.js +185 -0
- package/dist/node_modules/archiver/node_modules/async/eachOfLimit.js +47 -0
- package/dist/node_modules/archiver/node_modules/async/eachOfSeries.js +39 -0
- package/dist/node_modules/archiver/node_modules/async/eachSeries.js +44 -0
- package/dist/node_modules/archiver/node_modules/async/ensureAsync.js +67 -0
- package/dist/node_modules/archiver/node_modules/async/every.js +119 -0
- package/dist/node_modules/archiver/node_modules/async/everyLimit.js +46 -0
- package/dist/node_modules/archiver/node_modules/async/everySeries.js +45 -0
- package/dist/node_modules/archiver/node_modules/async/filter.js +93 -0
- package/dist/node_modules/archiver/node_modules/async/filterLimit.js +45 -0
- package/dist/node_modules/archiver/node_modules/async/filterSeries.js +43 -0
- package/dist/node_modules/archiver/node_modules/async/find.js +96 -0
- package/dist/node_modules/archiver/node_modules/async/findLimit.js +48 -0
- package/dist/node_modules/archiver/node_modules/async/findSeries.js +47 -0
- package/dist/node_modules/archiver/node_modules/async/flatMap.js +115 -0
- package/dist/node_modules/archiver/node_modules/async/flatMapLimit.js +60 -0
- package/dist/node_modules/archiver/node_modules/async/flatMapSeries.js +41 -0
- package/dist/node_modules/archiver/node_modules/async/foldl.js +153 -0
- package/dist/node_modules/archiver/node_modules/async/foldr.js +41 -0
- package/dist/node_modules/archiver/node_modules/async/forEach.js +129 -0
- package/dist/node_modules/archiver/node_modules/async/forEachLimit.js +50 -0
- package/dist/node_modules/archiver/node_modules/async/forEachOf.js +185 -0
- package/dist/node_modules/archiver/node_modules/async/forEachOfLimit.js +47 -0
- package/dist/node_modules/archiver/node_modules/async/forEachOfSeries.js +39 -0
- package/dist/node_modules/archiver/node_modules/async/forEachSeries.js +44 -0
- package/dist/node_modules/archiver/node_modules/async/forever.js +68 -0
- package/dist/node_modules/archiver/node_modules/async/groupBy.js +108 -0
- package/dist/node_modules/archiver/node_modules/async/groupByLimit.js +71 -0
- package/dist/node_modules/archiver/node_modules/async/groupBySeries.js +36 -0
- package/dist/node_modules/archiver/node_modules/async/index.js +588 -0
- package/dist/node_modules/archiver/node_modules/async/inject.js +153 -0
- package/dist/node_modules/archiver/node_modules/async/internal/DoublyLinkedList.js +92 -0
- package/dist/node_modules/archiver/node_modules/async/internal/Heap.js +120 -0
- package/dist/node_modules/archiver/node_modules/async/internal/applyEach.js +29 -0
- package/dist/node_modules/archiver/node_modules/async/internal/asyncEachOfLimit.js +75 -0
- package/dist/node_modules/archiver/node_modules/async/internal/awaitify.js +28 -0
- package/dist/node_modules/archiver/node_modules/async/internal/breakLoop.js +10 -0
- package/dist/node_modules/archiver/node_modules/async/internal/consoleFunc.js +31 -0
- package/dist/node_modules/archiver/node_modules/async/internal/createTester.js +40 -0
- package/dist/node_modules/archiver/node_modules/async/internal/eachOfLimit.js +90 -0
- package/dist/node_modules/archiver/node_modules/async/internal/filter.js +55 -0
- package/dist/node_modules/archiver/node_modules/async/internal/getIterator.js +11 -0
- package/dist/node_modules/archiver/node_modules/async/internal/initialParams.js +14 -0
- package/dist/node_modules/archiver/node_modules/async/internal/isArrayLike.js +10 -0
- package/dist/node_modules/archiver/node_modules/async/internal/iterator.js +57 -0
- package/dist/node_modules/archiver/node_modules/async/internal/map.js +30 -0
- package/dist/node_modules/archiver/node_modules/async/internal/once.js +17 -0
- package/dist/node_modules/archiver/node_modules/async/internal/onlyOnce.js +15 -0
- package/dist/node_modules/archiver/node_modules/async/internal/parallel.js +34 -0
- package/dist/node_modules/archiver/node_modules/async/internal/promiseCallback.js +23 -0
- package/dist/node_modules/archiver/node_modules/async/internal/queue.js +294 -0
- package/dist/node_modules/archiver/node_modules/async/internal/range.js +14 -0
- package/dist/node_modules/archiver/node_modules/async/internal/reject.js +26 -0
- package/dist/node_modules/archiver/node_modules/async/internal/setImmediate.js +34 -0
- package/dist/node_modules/archiver/node_modules/async/internal/withoutIndex.js +10 -0
- package/dist/node_modules/archiver/node_modules/async/internal/wrapAsync.js +34 -0
- package/dist/node_modules/archiver/node_modules/async/log.js +41 -0
- package/dist/node_modules/archiver/node_modules/async/map.js +142 -0
- package/dist/node_modules/archiver/node_modules/async/mapLimit.js +45 -0
- package/dist/node_modules/archiver/node_modules/async/mapSeries.js +44 -0
- package/dist/node_modules/archiver/node_modules/async/mapValues.js +152 -0
- package/dist/node_modules/archiver/node_modules/async/mapValuesLimit.js +61 -0
- package/dist/node_modules/archiver/node_modules/async/mapValuesSeries.js +37 -0
- package/dist/node_modules/archiver/node_modules/async/memoize.js +91 -0
- package/dist/node_modules/archiver/node_modules/async/nextTick.js +52 -0
- package/dist/node_modules/archiver/node_modules/async/package.json +75 -0
- package/dist/node_modules/archiver/node_modules/async/parallel.js +180 -0
- package/dist/node_modules/archiver/node_modules/async/parallelLimit.js +41 -0
- package/dist/node_modules/archiver/node_modules/async/priorityQueue.js +60 -0
- package/dist/node_modules/archiver/node_modules/async/queue.js +24 -0
- package/dist/node_modules/archiver/node_modules/async/race.js +67 -0
- package/dist/node_modules/archiver/node_modules/async/reduce.js +153 -0
- package/dist/node_modules/archiver/node_modules/async/reduceRight.js +41 -0
- package/dist/node_modules/archiver/node_modules/async/reflect.js +78 -0
- package/dist/node_modules/archiver/node_modules/async/reflectAll.js +93 -0
- package/dist/node_modules/archiver/node_modules/async/reject.js +87 -0
- package/dist/node_modules/archiver/node_modules/async/rejectLimit.js +45 -0
- package/dist/node_modules/archiver/node_modules/async/rejectSeries.js +43 -0
- package/dist/node_modules/archiver/node_modules/async/retry.js +159 -0
- package/dist/node_modules/archiver/node_modules/async/retryable.js +77 -0
- package/dist/node_modules/archiver/node_modules/async/select.js +93 -0
- package/dist/node_modules/archiver/node_modules/async/selectLimit.js +45 -0
- package/dist/node_modules/archiver/node_modules/async/selectSeries.js +43 -0
- package/dist/node_modules/archiver/node_modules/async/seq.js +79 -0
- package/dist/node_modules/archiver/node_modules/async/series.js +186 -0
- package/dist/node_modules/archiver/node_modules/async/setImmediate.js +45 -0
- package/dist/node_modules/archiver/node_modules/async/some.js +122 -0
- package/dist/node_modules/archiver/node_modules/async/someLimit.js +47 -0
- package/dist/node_modules/archiver/node_modules/async/someSeries.js +46 -0
- package/dist/node_modules/archiver/node_modules/async/sortBy.js +190 -0
- package/dist/node_modules/archiver/node_modules/async/timeout.js +89 -0
- package/dist/node_modules/archiver/node_modules/async/times.js +50 -0
- package/dist/node_modules/archiver/node_modules/async/timesLimit.js +43 -0
- package/dist/node_modules/archiver/node_modules/async/timesSeries.js +32 -0
- package/dist/node_modules/archiver/node_modules/async/transform.js +173 -0
- package/dist/node_modules/archiver/node_modules/async/tryEach.js +78 -0
- package/dist/node_modules/archiver/node_modules/async/unmemoize.js +25 -0
- package/dist/node_modules/archiver/node_modules/async/until.js +61 -0
- package/dist/node_modules/archiver/node_modules/async/waterfall.js +105 -0
- package/dist/node_modules/archiver/node_modules/async/whilst.js +78 -0
- package/dist/node_modules/archiver/node_modules/async/wrapSync.js +118 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/errors-browser.js +127 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/errors.js +116 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/experimentalWarning.js +17 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/_stream_duplex.js +126 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/_stream_passthrough.js +37 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/_stream_readable.js +1027 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/_stream_transform.js +190 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/_stream_writable.js +641 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/internal/streams/async_iterator.js +180 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/internal/streams/buffer_list.js +183 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/internal/streams/destroy.js +96 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/internal/streams/end-of-stream.js +86 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/internal/streams/from-browser.js +3 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/internal/streams/from.js +52 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/internal/streams/pipeline.js +86 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/internal/streams/state.js +22 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/internal/streams/stream-browser.js +1 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/lib/internal/streams/stream.js +1 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/package.json +68 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/readable-browser.js +9 -0
- package/dist/node_modules/archiver/node_modules/readable-stream/readable.js +16 -0
- package/dist/node_modules/archiver/node_modules/tar-stream/extract.js +257 -0
- package/dist/node_modules/archiver/node_modules/tar-stream/headers.js +295 -0
- package/dist/node_modules/archiver/node_modules/tar-stream/index.js +2 -0
- package/dist/node_modules/archiver/node_modules/tar-stream/pack.js +255 -0
- package/dist/node_modules/archiver/node_modules/tar-stream/package.json +58 -0
- package/dist/node_modules/archiver/node_modules/tar-stream/sandbox.js +11 -0
- package/dist/node_modules/archiver/package.json +1 -0
- package/dist/node_modules/yauzl/LICENSE +21 -0
- package/dist/node_modules/yauzl/index.js +1 -0
- package/dist/node_modules/yauzl/package.json +1 -0
- package/dist/server/exporter.d.ts +42 -0
- package/dist/server/exporter.js +311 -0
- package/dist/server/field-value-writer.d.ts +9 -0
- package/dist/server/field-value-writer.js +107 -0
- package/dist/server/importer.d.ts +138 -0
- package/dist/server/importer.js +842 -0
- package/dist/server/index.d.ts +1 -0
- package/dist/server/index.js +42 -0
- package/dist/server/package-format.d.ts +50 -0
- package/dist/server/package-format.js +115 -0
- package/dist/server/plugin.d.ts +6 -0
- package/dist/server/plugin.js +61 -0
- package/dist/server/resourcer.d.ts +16 -0
- package/dist/server/resourcer.js +187 -0
- package/package.json +54 -4
- package/server.d.ts +2 -0
- package/server.js +1 -0
|
@@ -0,0 +1,842 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This file is part of the NocoBase (R) project.
|
|
3
|
+
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
|
|
4
|
+
* Authors: NocoBase Team.
|
|
5
|
+
*
|
|
6
|
+
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
|
|
7
|
+
* For more information, please refer to: https://www.nocobase.com/agreement.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
var __create = Object.create;
|
|
11
|
+
var __defProp = Object.defineProperty;
|
|
12
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
13
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
14
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
15
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
16
|
+
var __export = (target, all) => {
|
|
17
|
+
for (var name in all)
|
|
18
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
19
|
+
};
|
|
20
|
+
var __copyProps = (to, from, except, desc) => {
|
|
21
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
22
|
+
for (let key of __getOwnPropNames(from))
|
|
23
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
24
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
25
|
+
}
|
|
26
|
+
return to;
|
|
27
|
+
};
|
|
28
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
29
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
30
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
31
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
32
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
33
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
34
|
+
mod
|
|
35
|
+
));
|
|
36
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
37
|
+
var importer_exports = {};
|
|
38
|
+
__export(importer_exports, {
|
|
39
|
+
CONFLICT_MODES: () => CONFLICT_MODES,
|
|
40
|
+
CollectionImporter: () => CollectionImporter,
|
|
41
|
+
DEFAULT_CONFLICT_MODE: () => DEFAULT_CONFLICT_MODE,
|
|
42
|
+
ImportBlockedError: () => ImportBlockedError
|
|
43
|
+
});
|
|
44
|
+
module.exports = __toCommonJS(importer_exports);
|
|
45
|
+
var import_database = require("@nocobase/database");
|
|
46
|
+
var import_utils = require("@nocobase/utils");
|
|
47
|
+
var import_crypto = __toESM(require("crypto"));
|
|
48
|
+
var import_fs = __toESM(require("fs"));
|
|
49
|
+
var import_promises = __toESM(require("fs/promises"));
|
|
50
|
+
var import_lodash = __toESM(require("lodash"));
|
|
51
|
+
var import_path = __toESM(require("path"));
|
|
52
|
+
var import_sequelize = require("sequelize");
|
|
53
|
+
var import_yauzl = __toESM(require("yauzl"));
|
|
54
|
+
var import_exporter = require("./exporter");
|
|
55
|
+
var import_field_value_writer = require("./field-value-writer");
|
|
56
|
+
var import_package_format = require("./package-format");
|
|
57
|
+
const TOKEN_PATTERN = /^[0-9a-f-]{36}$/;
|
|
58
|
+
const PLACEMENT_OPTIONS = [
|
|
59
|
+
"tableName",
|
|
60
|
+
"schema",
|
|
61
|
+
"viewName",
|
|
62
|
+
"view",
|
|
63
|
+
"sql",
|
|
64
|
+
"from",
|
|
65
|
+
"isThrough",
|
|
66
|
+
"autoCreate",
|
|
67
|
+
"dataSourceKey"
|
|
68
|
+
];
|
|
69
|
+
const NESTED_FIELD_STRIP = ["key", "sort", "collectionName", "parentKey", "reverseKey", "reverseField", "children"];
|
|
70
|
+
const ASSOCIATION_STRIP = ["key", "sort", "parentKey", "reverseKey", "children"];
|
|
71
|
+
const UPLOAD_TTL_MS = 60 * 60 * 1e3;
|
|
72
|
+
const INSERT_CHUNK_SIZE = 500;
|
|
73
|
+
const ASSOCIATION_TYPES_FOR_REFERENCES = ["belongsTo", "hasOne", "hasMany", "belongsToMany"];
|
|
74
|
+
const ALLOWED_ENTRY = /^(manifest\.json|schema\/(collections|associations|through)\.json|data\/[A-Za-z_][A-Za-z0-9_]*\/(meta|data))$/;
|
|
75
|
+
const CONFLICT_MODES = ["skipCollectionImportData", "skipAll", "overrideSchema", "overrideAll"];
|
|
76
|
+
const DEFAULT_CONFLICT_MODE = "overrideAll";
|
|
77
|
+
async function* readRows(filePath, maxRowBytes) {
|
|
78
|
+
const rowTooLarge = () => new Error("A data row is larger than the allowed row size");
|
|
79
|
+
let pieces = [];
|
|
80
|
+
let pendingBytes = 0;
|
|
81
|
+
for await (const value of import_fs.default.createReadStream(filePath, { encoding: "utf8" })) {
|
|
82
|
+
const chunk = value;
|
|
83
|
+
let start = 0;
|
|
84
|
+
let index = chunk.indexOf("\n");
|
|
85
|
+
while (index >= 0) {
|
|
86
|
+
const piece = chunk.slice(start, index);
|
|
87
|
+
if (pendingBytes + Buffer.byteLength(piece) > maxRowBytes) throw rowTooLarge();
|
|
88
|
+
const line = (pieces.length ? pieces.join("") + piece : piece).replace(/\r$/, "");
|
|
89
|
+
pieces = [];
|
|
90
|
+
pendingBytes = 0;
|
|
91
|
+
if (line) yield line;
|
|
92
|
+
start = index + 1;
|
|
93
|
+
index = chunk.indexOf("\n", start);
|
|
94
|
+
}
|
|
95
|
+
const rest = chunk.slice(start);
|
|
96
|
+
if (rest) {
|
|
97
|
+
pendingBytes += Buffer.byteLength(rest);
|
|
98
|
+
if (pendingBytes > maxRowBytes) throw rowTooLarge();
|
|
99
|
+
pieces.push(rest);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const last = pieces.join("");
|
|
103
|
+
if (last.trim()) yield last;
|
|
104
|
+
}
|
|
105
|
+
class ImportBlockedError extends Error {
|
|
106
|
+
constructor(problems) {
|
|
107
|
+
super(`Import blocked by ${problems.length} problem(s): ${problems.map((p) => p.code).join(", ")}`);
|
|
108
|
+
this.problems = problems;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
class CollectionImporter {
|
|
112
|
+
constructor(app) {
|
|
113
|
+
this.app = app;
|
|
114
|
+
}
|
|
115
|
+
get db() {
|
|
116
|
+
return this.app.db;
|
|
117
|
+
}
|
|
118
|
+
static uploadDir() {
|
|
119
|
+
return (0, import_utils.storagePathJoin)("tmp", "collection-transfer");
|
|
120
|
+
}
|
|
121
|
+
packageDir(token) {
|
|
122
|
+
if (!TOKEN_PATTERN.test(token)) {
|
|
123
|
+
throw new Error("Invalid upload token");
|
|
124
|
+
}
|
|
125
|
+
return import_path.default.join(CollectionImporter.uploadDir(), token);
|
|
126
|
+
}
|
|
127
|
+
/** Extracts an uploaded zip into its own directory and returns the token that identifies it. */
|
|
128
|
+
async receive(zipPath) {
|
|
129
|
+
await this.removeExpiredUploads();
|
|
130
|
+
await CollectionImporter.inspectArchive(zipPath);
|
|
131
|
+
const token = import_crypto.default.randomUUID();
|
|
132
|
+
const dir = this.packageDir(token);
|
|
133
|
+
await import_promises.default.mkdir(dir, { recursive: true });
|
|
134
|
+
try {
|
|
135
|
+
await CollectionImporter.extractArchive(zipPath, dir);
|
|
136
|
+
await import_promises.default.access(import_path.default.join(dir, import_package_format.MANIFEST_FILE));
|
|
137
|
+
} catch (error) {
|
|
138
|
+
await import_promises.default.rm(dir, { recursive: true, force: true });
|
|
139
|
+
const message = error instanceof Error ? error.message : "";
|
|
140
|
+
throw new Error(message.startsWith("Package refused") ? message : "Not a collection transfer package");
|
|
141
|
+
}
|
|
142
|
+
return token;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Extracts only the entries the package format defines, streaming each one to disk while counting bytes, and stops as
|
|
146
|
+
* soon as an entry or the whole package passes the limits. Nothing is buffered in memory.
|
|
147
|
+
*/
|
|
148
|
+
static async extractArchive(zipPath, dir) {
|
|
149
|
+
const limits = CollectionImporter.limits;
|
|
150
|
+
let total = 0;
|
|
151
|
+
await new Promise((resolve, reject) => {
|
|
152
|
+
import_yauzl.default.open(zipPath, { lazyEntries: true, autoClose: true }, (openError, zipfile) => {
|
|
153
|
+
if (openError || !zipfile) {
|
|
154
|
+
reject(openError || new Error("Not a collection transfer package"));
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
const fail = (error) => {
|
|
158
|
+
zipfile.close();
|
|
159
|
+
reject(error);
|
|
160
|
+
};
|
|
161
|
+
zipfile.on("error", fail);
|
|
162
|
+
zipfile.on("end", () => resolve());
|
|
163
|
+
zipfile.on("entry", (entry) => {
|
|
164
|
+
const name = entry.fileName.replace(/\\/g, "/");
|
|
165
|
+
if (name.endsWith("/") || !ALLOWED_ENTRY.test(name)) {
|
|
166
|
+
zipfile.readEntry();
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
zipfile.openReadStream(entry, (streamError, input) => {
|
|
170
|
+
if (streamError || !input) {
|
|
171
|
+
fail(streamError || new Error("Not a collection transfer package"));
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
const target = import_path.default.join(dir, name);
|
|
175
|
+
let written = 0;
|
|
176
|
+
import_promises.default.mkdir(import_path.default.dirname(target), { recursive: true }).then(() => {
|
|
177
|
+
const output = import_fs.default.createWriteStream(target);
|
|
178
|
+
input.on("data", (chunk) => {
|
|
179
|
+
written += chunk.length;
|
|
180
|
+
total += chunk.length;
|
|
181
|
+
if (written > limits.maxEntryBytes || total > limits.maxTotalBytes) {
|
|
182
|
+
input.destroy();
|
|
183
|
+
output.destroy();
|
|
184
|
+
fail(new Error(`Package refused: ${name} expands beyond the size limit`));
|
|
185
|
+
}
|
|
186
|
+
});
|
|
187
|
+
input.on("error", fail);
|
|
188
|
+
output.on("error", fail);
|
|
189
|
+
output.on("finish", () => zipfile.readEntry());
|
|
190
|
+
input.pipe(output);
|
|
191
|
+
}).catch(fail);
|
|
192
|
+
});
|
|
193
|
+
});
|
|
194
|
+
zipfile.readEntry();
|
|
195
|
+
});
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
async discard(token) {
|
|
199
|
+
await import_promises.default.rm(this.packageDir(token), { recursive: true, force: true });
|
|
200
|
+
}
|
|
201
|
+
/** Upper bounds for an uploaded package (static so tests can lower them). */
|
|
202
|
+
static limits = {
|
|
203
|
+
maxEntries: 1e4,
|
|
204
|
+
maxEntryBytes: 1024 * 1024 * 1024,
|
|
205
|
+
maxTotalBytes: 2 * 1024 * 1024 * 1024,
|
|
206
|
+
// Only checked for entries above `ratioCheckBytes`; text compresses well, bombs compress absurdly well.
|
|
207
|
+
maxCompressionRatio: 200,
|
|
208
|
+
ratioCheckBytes: 10 * 1024 * 1024,
|
|
209
|
+
maxSchemaFileBytes: 50 * 1024 * 1024,
|
|
210
|
+
// One data row (one line of NDJSON).
|
|
211
|
+
maxRowBytes: 16 * 1024 * 1024
|
|
212
|
+
};
|
|
213
|
+
/**
|
|
214
|
+
* Reads the zip's central directory before anything is extracted and refuses archives that would expand beyond the
|
|
215
|
+
* limits (zip bombs). yauzl validates the declared sizes while extracting, so the declared values cannot be undercut.
|
|
216
|
+
*/
|
|
217
|
+
static async inspectArchive(zipPath) {
|
|
218
|
+
const limits = CollectionImporter.limits;
|
|
219
|
+
await new Promise((resolve, reject) => {
|
|
220
|
+
import_yauzl.default.open(zipPath, { lazyEntries: true, autoClose: true }, (openError, zipfile) => {
|
|
221
|
+
if (openError || !zipfile) {
|
|
222
|
+
reject(new Error("Not a collection transfer package"));
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
if (zipfile.entryCount > limits.maxEntries) {
|
|
226
|
+
zipfile.close();
|
|
227
|
+
reject(new Error(`Package has too many entries (${zipfile.entryCount})`));
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
let total = 0;
|
|
231
|
+
zipfile.on("entry", (entry) => {
|
|
232
|
+
const size = entry.uncompressedSize;
|
|
233
|
+
const ratio = entry.compressedSize > 0 ? size / entry.compressedSize : size > 0 ? Infinity : 1;
|
|
234
|
+
total += size;
|
|
235
|
+
let reason;
|
|
236
|
+
if (size > limits.maxEntryBytes) reason = `entry ${entry.fileName} is too large`;
|
|
237
|
+
else if (total > limits.maxTotalBytes) reason = "package expands beyond the size limit";
|
|
238
|
+
else if (size > limits.ratioCheckBytes && ratio > limits.maxCompressionRatio) {
|
|
239
|
+
reason = `entry ${entry.fileName} is suspiciously compressed`;
|
|
240
|
+
}
|
|
241
|
+
if (reason) {
|
|
242
|
+
zipfile.close();
|
|
243
|
+
reject(new Error(`Package refused: ${reason}`));
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
zipfile.readEntry();
|
|
247
|
+
});
|
|
248
|
+
zipfile.on("end", () => resolve());
|
|
249
|
+
zipfile.on("error", () => reject(new Error("Not a collection transfer package")));
|
|
250
|
+
zipfile.readEntry();
|
|
251
|
+
});
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
async preview(token) {
|
|
255
|
+
const pkg = await this.load(token);
|
|
256
|
+
const problems = await this.check(pkg);
|
|
257
|
+
const fieldCounts = import_lodash.default.countBy(pkg.associations, (a) => a.collectionName);
|
|
258
|
+
const collections = [
|
|
259
|
+
...pkg.collections.map((c) => {
|
|
260
|
+
var _a;
|
|
261
|
+
return {
|
|
262
|
+
name: c.name,
|
|
263
|
+
title: c.title || c.name,
|
|
264
|
+
fieldCount: c.fields.length + (fieldCounts[c.name] || 0),
|
|
265
|
+
rowCount: ((_a = pkg.metas[c.name]) == null ? void 0 : _a.count) || 0,
|
|
266
|
+
isThrough: false
|
|
267
|
+
};
|
|
268
|
+
}),
|
|
269
|
+
...pkg.through.map((name) => {
|
|
270
|
+
var _a;
|
|
271
|
+
return {
|
|
272
|
+
name,
|
|
273
|
+
title: name,
|
|
274
|
+
fieldCount: 0,
|
|
275
|
+
rowCount: ((_a = pkg.metas[name]) == null ? void 0 : _a.count) || 0,
|
|
276
|
+
isThrough: true
|
|
277
|
+
};
|
|
278
|
+
})
|
|
279
|
+
];
|
|
280
|
+
return {
|
|
281
|
+
token,
|
|
282
|
+
manifest: pkg.manifest,
|
|
283
|
+
collections,
|
|
284
|
+
problems,
|
|
285
|
+
interfaces: this.interfacesOf(pkg),
|
|
286
|
+
missingPlugins: this.missingPlugins(pkg.manifest),
|
|
287
|
+
conflicts: await this.conflicts(pkg)
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* @param options.userId the importing user. "Created by" / "Updated by" of every imported row is set to this user,
|
|
292
|
+
* because user ids from the source environment do not identify the same people here.
|
|
293
|
+
*/
|
|
294
|
+
async import(token, options = {}) {
|
|
295
|
+
const pkg = await this.load(token);
|
|
296
|
+
const { manifest } = pkg;
|
|
297
|
+
const problems = await this.check(pkg);
|
|
298
|
+
const conflicts = await this.conflicts(pkg);
|
|
299
|
+
const resolutions = options.resolutions || {};
|
|
300
|
+
for (const conflict of conflicts) {
|
|
301
|
+
const mode = resolutions[conflict.name];
|
|
302
|
+
if (!CONFLICT_MODES.includes(mode)) {
|
|
303
|
+
problems.push({ code: "unresolvedConflict", collection: conflict.name });
|
|
304
|
+
} else if (mode === "skipCollectionImportData" && manifest.includeData && conflict.missingColumns.length) {
|
|
305
|
+
problems.push({
|
|
306
|
+
code: "columnsMismatch",
|
|
307
|
+
collection: conflict.name,
|
|
308
|
+
detail: conflict.missingColumns.join(", ")
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
if (problems.length) {
|
|
313
|
+
throw new ImportBlockedError(problems);
|
|
314
|
+
}
|
|
315
|
+
const existing = new Set(conflicts.map((conflict) => conflict.name));
|
|
316
|
+
const modeOf = (name) => existing.has(name) ? resolutions[name] : "new";
|
|
317
|
+
const isOverride = (name) => ["overrideSchema", "overrideAll"].includes(modeOf(name));
|
|
318
|
+
const create = new Set(manifest.collections.filter((name) => modeOf(name) === "new" || isOverride(name)));
|
|
319
|
+
const overridden = manifest.collections.filter(isOverride);
|
|
320
|
+
const skipped = manifest.collections.filter((name) => existing.has(name) && !isOverride(name));
|
|
321
|
+
const wantsData = (name) => manifest.includeData && ["new", "overrideAll", "skipCollectionImportData"].includes(modeOf(name));
|
|
322
|
+
const owners = this.throughOwners(pkg);
|
|
323
|
+
const throughCreate = manifest.through.filter((name) => create.has(owners[name]));
|
|
324
|
+
const throughExisting = new Set(manifest.through.filter((name) => this.db.hasCollection(name)));
|
|
325
|
+
const throughAppend = manifest.through.filter(
|
|
326
|
+
(name) => !create.has(owners[name]) && modeOf(owners[name]) === "skipCollectionImportData" && throughExisting.has(name)
|
|
327
|
+
);
|
|
328
|
+
const reloadOnRollback = /* @__PURE__ */ new Set([...overridden, ...throughCreate.filter((n) => throughExisting.has(n))]);
|
|
329
|
+
for (const conflict of conflicts) {
|
|
330
|
+
if (isOverride(conflict.name)) conflict.referencedBy.forEach((ref) => reloadOnRollback.add(ref.collection));
|
|
331
|
+
}
|
|
332
|
+
const collectionsRepository = this.db.getRepository("collections");
|
|
333
|
+
const skippedFieldsBefore = /* @__PURE__ */ new Set();
|
|
334
|
+
for (const association of pkg.associations) {
|
|
335
|
+
const sides = [[association.collectionName, association.name]];
|
|
336
|
+
if (association.reverseField) sides.push([association.target, association.reverseField.name]);
|
|
337
|
+
for (const [collectionName, name] of sides) {
|
|
338
|
+
if (!skipped.includes(collectionName)) continue;
|
|
339
|
+
if (await this.db.getRepository("fields").count({ filter: { collectionName, name } }) > 0) {
|
|
340
|
+
skippedFieldsBefore.add(`${collectionName}.${name}`);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
const transaction = await this.db.sequelize.transaction();
|
|
345
|
+
const rowCounts = {};
|
|
346
|
+
try {
|
|
347
|
+
for (const name of [...overridden, ...throughCreate.filter((n) => throughExisting.has(n))]) {
|
|
348
|
+
await collectionsRepository.destroy({ filter: { name }, cascade: true, context: {}, transaction });
|
|
349
|
+
}
|
|
350
|
+
for (const collection of pkg.collections) {
|
|
351
|
+
if (!create.has(collection.name)) continue;
|
|
352
|
+
const values = {
|
|
353
|
+
...import_lodash.default.omit(collection, ["key", "sort", "category"]),
|
|
354
|
+
fields: collection.fields.map((field) => import_lodash.default.omit(field, NESTED_FIELD_STRIP))
|
|
355
|
+
};
|
|
356
|
+
await collectionsRepository.create({ values, context: {}, transaction });
|
|
357
|
+
}
|
|
358
|
+
const fieldExists = async (collectionName, name) => await this.db.getRepository("fields").count({ filter: { collectionName, name }, transaction }) > 0;
|
|
359
|
+
const needsSide = async (collectionName, name) => create.has(collectionName) || skippedFieldsBefore.has(`${collectionName}.${name}`) && !await fieldExists(collectionName, name);
|
|
360
|
+
const createdAssociations = [];
|
|
361
|
+
for (const association of pkg.associations) {
|
|
362
|
+
const ownerNeeded = await needsSide(association.collectionName, association.name);
|
|
363
|
+
const reverseNeeded = !!association.reverseField && await needsSide(association.target, association.reverseField.name);
|
|
364
|
+
let values = null;
|
|
365
|
+
if (ownerNeeded) {
|
|
366
|
+
values = {
|
|
367
|
+
...import_lodash.default.omit(association, [...ASSOCIATION_STRIP, "reverseField"]),
|
|
368
|
+
...reverseNeeded ? { reverseField: import_lodash.default.omit(association.reverseField, NESTED_FIELD_STRIP) } : {}
|
|
369
|
+
};
|
|
370
|
+
} else if (reverseNeeded) {
|
|
371
|
+
values = {
|
|
372
|
+
...import_lodash.default.omit(association.reverseField, NESTED_FIELD_STRIP),
|
|
373
|
+
collectionName: association.target,
|
|
374
|
+
target: association.collectionName
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
if (!values) continue;
|
|
378
|
+
await this.db.getRepository("fields").create({
|
|
379
|
+
values,
|
|
380
|
+
updateAssociationValues: ["reverseField"],
|
|
381
|
+
context: {},
|
|
382
|
+
transaction
|
|
383
|
+
});
|
|
384
|
+
createdAssociations.push(values);
|
|
385
|
+
}
|
|
386
|
+
await this.assertHookResults(createdAssociations, throughCreate, transaction);
|
|
387
|
+
await this.assertColumns(pkg, [...create, ...throughCreate], transaction);
|
|
388
|
+
const userId = options.userId ?? null;
|
|
389
|
+
for (const name of create) {
|
|
390
|
+
if (wantsData(name)) rowCounts[name] = await this.insertRows(token, name, pkg.metas[name], transaction, userId);
|
|
391
|
+
}
|
|
392
|
+
for (const name of skipped) {
|
|
393
|
+
if (wantsData(name)) {
|
|
394
|
+
rowCounts[name] = await this.insertRows(token, name, pkg.metas[name], transaction, userId, true);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
for (const name of throughCreate) {
|
|
398
|
+
if (wantsData(owners[name]))
|
|
399
|
+
rowCounts[name] = await this.insertRows(token, name, pkg.metas[name], transaction, userId);
|
|
400
|
+
}
|
|
401
|
+
for (const name of throughAppend) {
|
|
402
|
+
if (manifest.includeData) {
|
|
403
|
+
await this.assertColumnsSubset(name, pkg.metas[name], transaction);
|
|
404
|
+
rowCounts[name] = await this.insertRows(token, name, pkg.metas[name], transaction, userId, true);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
await transaction.commit();
|
|
408
|
+
} catch (error) {
|
|
409
|
+
await transaction.rollback();
|
|
410
|
+
for (const name of [...create, ...throughCreate]) {
|
|
411
|
+
if (this.db.hasCollection(name)) this.db.removeCollection(name);
|
|
412
|
+
}
|
|
413
|
+
if (reloadOnRollback.size) {
|
|
414
|
+
await collectionsRepository.load({
|
|
415
|
+
filter: { name: { $in: [...reloadOnRollback] } }
|
|
416
|
+
});
|
|
417
|
+
}
|
|
418
|
+
throw error;
|
|
419
|
+
}
|
|
420
|
+
await this.discard(token);
|
|
421
|
+
return {
|
|
422
|
+
collections: [...create],
|
|
423
|
+
overridden,
|
|
424
|
+
skipped,
|
|
425
|
+
through: [...throughCreate, ...throughAppend],
|
|
426
|
+
rowCounts
|
|
427
|
+
};
|
|
428
|
+
}
|
|
429
|
+
/** Collections of the package that already exist here, with what an override would remove. */
|
|
430
|
+
async conflicts(pkg) {
|
|
431
|
+
const result = [];
|
|
432
|
+
const packageFields = /* @__PURE__ */ new Set();
|
|
433
|
+
for (const association of pkg.associations) {
|
|
434
|
+
packageFields.add(`${association.collectionName}.${association.name}`);
|
|
435
|
+
if (association.reverseField) packageFields.add(`${association.target}.${association.reverseField.name}`);
|
|
436
|
+
}
|
|
437
|
+
for (const name of pkg.manifest.collections) {
|
|
438
|
+
if (!(0, import_package_format.isValidName)(name)) continue;
|
|
439
|
+
const row = await this.db.getRepository("collections").findOne({ filter: { name } });
|
|
440
|
+
if (!row && !this.db.hasCollection(name)) continue;
|
|
441
|
+
if (!await this.isUserCollection(name)) continue;
|
|
442
|
+
const references = await this.db.getRepository("fields").find({
|
|
443
|
+
filter: { type: { $in: ASSOCIATION_TYPES_FOR_REFERENCES }, collectionName: { $ne: name } }
|
|
444
|
+
});
|
|
445
|
+
const referencedBy = references.filter((field) => field.get("target") === name || field.get("through") === name).map((field) => {
|
|
446
|
+
const collection2 = field.get("collectionName");
|
|
447
|
+
const fieldName = field.get("name");
|
|
448
|
+
return { collection: collection2, field: fieldName, restored: packageFields.has(`${collection2}.${fieldName}`) };
|
|
449
|
+
});
|
|
450
|
+
let missingColumns = [];
|
|
451
|
+
const collection = this.db.getCollection(name);
|
|
452
|
+
if (collection && pkg.metas[name]) {
|
|
453
|
+
const live = await this.db.sequelize.getQueryInterface().describeTable(collection.getTableNameWithSchema());
|
|
454
|
+
missingColumns = pkg.metas[name].columns.filter((column) => !(column in live));
|
|
455
|
+
}
|
|
456
|
+
result.push({ name, title: (row == null ? void 0 : row.get("title")) || name, referencedBy, missingColumns });
|
|
457
|
+
}
|
|
458
|
+
return result;
|
|
459
|
+
}
|
|
460
|
+
/** A collection created in the collection manager: has a row, is not hidden, not a junction, not mirrored from code. */
|
|
461
|
+
async isUserCollection(name) {
|
|
462
|
+
const row = await this.db.getRepository("collections").findOne({ filter: { name } });
|
|
463
|
+
if (!row) return false;
|
|
464
|
+
const values = row.get();
|
|
465
|
+
return !values.hidden && !values.isThrough && !import_package_format.MIRRORED_ORIGINS.includes(values.from);
|
|
466
|
+
}
|
|
467
|
+
/** An auto-created junction table whose owner is a user collection with a belongsToMany field using it. */
|
|
468
|
+
async isOwnedJunction(name, owner) {
|
|
469
|
+
if (!owner || !await this.isUserCollection(owner)) return false;
|
|
470
|
+
const row = await this.db.getRepository("collections").findOne({ filter: { name } });
|
|
471
|
+
const values = (row == null ? void 0 : row.get()) || {};
|
|
472
|
+
if (!values.isThrough || !values.autoCreate) return false;
|
|
473
|
+
const fields = await this.db.getRepository("fields").find({ filter: { collectionName: owner, type: "belongsToMany" } });
|
|
474
|
+
return fields.some((field) => field.get("through") === name);
|
|
475
|
+
}
|
|
476
|
+
/** Junction table name → collection that owns the belongsToMany field using it. */
|
|
477
|
+
throughOwners(pkg) {
|
|
478
|
+
const owners = {};
|
|
479
|
+
for (const association of pkg.associations) {
|
|
480
|
+
if (association.type === "belongsToMany" && typeof association.through === "string") {
|
|
481
|
+
owners[association.through] ??= association.collectionName;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
return owners;
|
|
485
|
+
}
|
|
486
|
+
async load(token) {
|
|
487
|
+
const dir = this.packageDir(token);
|
|
488
|
+
const readJson = async (relativePath) => {
|
|
489
|
+
const filePath = import_path.default.join(dir, relativePath);
|
|
490
|
+
if ((await import_promises.default.stat(filePath)).size > CollectionImporter.limits.maxSchemaFileBytes) {
|
|
491
|
+
throw new Error(`${relativePath} is too large`);
|
|
492
|
+
}
|
|
493
|
+
return JSON.parse(await import_promises.default.readFile(filePath, "utf8"));
|
|
494
|
+
};
|
|
495
|
+
let manifest;
|
|
496
|
+
try {
|
|
497
|
+
manifest = await readJson(import_package_format.MANIFEST_FILE);
|
|
498
|
+
} catch (error) {
|
|
499
|
+
throw new Error("Not a collection transfer package (manifest.json missing or unreadable)");
|
|
500
|
+
}
|
|
501
|
+
const collections = await readJson(import_package_format.COLLECTIONS_FILE);
|
|
502
|
+
const associations = await readJson(import_package_format.ASSOCIATIONS_FILE);
|
|
503
|
+
const through = await readJson(import_package_format.THROUGH_FILE);
|
|
504
|
+
const metas = {};
|
|
505
|
+
for (const name of [...manifest.collections || [], ...manifest.through || []]) {
|
|
506
|
+
if (!(0, import_package_format.isValidName)(name)) continue;
|
|
507
|
+
try {
|
|
508
|
+
const meta = await readJson(`${(0, import_package_format.dataDir)(name)}/meta`);
|
|
509
|
+
const valid = (meta == null ? void 0 : meta.name) === name && Array.isArray(meta.columns) && meta.columns.every((column) => (0, import_package_format.isValidName)(column)) && Number.isInteger(meta.count) && meta.count >= 0;
|
|
510
|
+
if (valid) metas[name] = meta;
|
|
511
|
+
} catch (error) {
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
return { manifest, collections, associations, through, metas };
|
|
515
|
+
}
|
|
516
|
+
async check(pkg) {
|
|
517
|
+
const { manifest } = pkg;
|
|
518
|
+
const problems = [];
|
|
519
|
+
if (manifest.formatVersion !== import_package_format.FORMAT_VERSION) {
|
|
520
|
+
problems.push({ code: "formatVersion", detail: String(manifest.formatVersion) });
|
|
521
|
+
return problems;
|
|
522
|
+
}
|
|
523
|
+
const dialect = this.db.sequelize.getDialect();
|
|
524
|
+
if (manifest.dialect !== dialect) {
|
|
525
|
+
problems.push({ code: "dialect", detail: `${manifest.dialect} \u2192 ${dialect}` });
|
|
526
|
+
}
|
|
527
|
+
const names = [...manifest.collections, ...manifest.through];
|
|
528
|
+
const payloadNames = pkg.collections.map((c) => c.name);
|
|
529
|
+
if (!import_lodash.default.isEqual(payloadNames, manifest.collections) || !import_lodash.default.isEqual(pkg.through, manifest.through)) {
|
|
530
|
+
problems.push({ code: "invalidPackage", detail: "schema files do not match manifest" });
|
|
531
|
+
}
|
|
532
|
+
const existingTables = await this.existingTableNames();
|
|
533
|
+
const owners = this.throughOwners(pkg);
|
|
534
|
+
const exists = async (name) => this.db.hasCollection(name) || await this.collectionRowExists(name);
|
|
535
|
+
for (const name of names) {
|
|
536
|
+
if (!(0, import_package_format.isValidName)(name)) {
|
|
537
|
+
problems.push({ code: "invalidName", collection: String(name) });
|
|
538
|
+
continue;
|
|
539
|
+
}
|
|
540
|
+
if (!pkg.metas[name]) {
|
|
541
|
+
problems.push({ code: "invalidPackage", collection: name, detail: "meta missing" });
|
|
542
|
+
}
|
|
543
|
+
if (name.length > import_package_format.MAX_IDENTIFIER_LENGTH) {
|
|
544
|
+
problems.push({ code: "identifierTooLong", collection: name });
|
|
545
|
+
}
|
|
546
|
+
if (await exists(name)) {
|
|
547
|
+
if (manifest.collections.includes(name)) {
|
|
548
|
+
if (!await this.isUserCollection(name)) problems.push({ code: "systemCollection", collection: name });
|
|
549
|
+
} else if (!await this.isOwnedJunction(name, owners[name])) {
|
|
550
|
+
problems.push({ code: "collectionExists", collection: name });
|
|
551
|
+
}
|
|
552
|
+
} else if (existingTables.has(this.expectedTableName(name))) {
|
|
553
|
+
problems.push({ code: "tableExists", collection: name });
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
for (const collection of pkg.collections) {
|
|
557
|
+
const template = collection.template || "general";
|
|
558
|
+
if (!import_package_format.SUPPORTED_TEMPLATES.includes(template) || !import_lodash.default.isEmpty(collection.inherits)) {
|
|
559
|
+
problems.push({ code: "unsupportedTemplate", collection: collection.name, detail: template });
|
|
560
|
+
}
|
|
561
|
+
for (const field of collection.fields) {
|
|
562
|
+
this.checkField(field, collection.name, problems);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
for (const association of pkg.associations) {
|
|
566
|
+
this.checkField(association, association.collectionName, problems);
|
|
567
|
+
if (association.reverseField) {
|
|
568
|
+
this.checkField(association.reverseField, association.target, problems);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
this.checkContainment(pkg, problems);
|
|
572
|
+
for (const payload of [...pkg.collections, ...pkg.associations]) {
|
|
573
|
+
const unsafe = (0, import_package_format.findUnsafeExpression)(payload);
|
|
574
|
+
if (unsafe) {
|
|
575
|
+
const collection = "collectionName" in payload ? payload.collectionName : payload.name;
|
|
576
|
+
problems.push({ code: "unsafeExpression", collection, detail: unsafe.slice(0, 120) });
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
for (const dependency of manifest.dependencies) {
|
|
580
|
+
if (!this.db.hasCollection(dependency)) {
|
|
581
|
+
problems.push({ code: "missingDependency", collection: dependency });
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
return import_lodash.default.uniqWith(problems, import_lodash.default.isEqual);
|
|
585
|
+
}
|
|
586
|
+
interfacesOf(pkg) {
|
|
587
|
+
const result = [];
|
|
588
|
+
const add = (field, collection) => {
|
|
589
|
+
if (typeof field.interface === "string" && field.interface) {
|
|
590
|
+
result.push({ interface: field.interface, collection, field: field.name });
|
|
591
|
+
}
|
|
592
|
+
};
|
|
593
|
+
for (const collection of pkg.collections) {
|
|
594
|
+
for (const field of collection.fields) add(field, collection.name);
|
|
595
|
+
}
|
|
596
|
+
for (const association of pkg.associations) {
|
|
597
|
+
add(association, association.collectionName);
|
|
598
|
+
if (association.reverseField) add(association.reverseField, association.target);
|
|
599
|
+
}
|
|
600
|
+
return result;
|
|
601
|
+
}
|
|
602
|
+
missingPlugins(manifest) {
|
|
603
|
+
var _a;
|
|
604
|
+
if (!((_a = manifest.plugins) == null ? void 0 : _a.length)) return [];
|
|
605
|
+
const here = new Set((0, import_exporter.enabledPluginNames)(this.app));
|
|
606
|
+
return manifest.plugins.filter((name) => !here.has(name));
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
609
|
+
* Everything an import creates (collections, fields, FK columns, through tables) must belong to the package.
|
|
610
|
+
* Exports only produce such packages; this refuses crafted ones that point at existing collections or tables.
|
|
611
|
+
*/
|
|
612
|
+
checkContainment(pkg, problems) {
|
|
613
|
+
const { manifest } = pkg;
|
|
614
|
+
const own = new Set(manifest.collections);
|
|
615
|
+
const ownOrThrough = /* @__PURE__ */ new Set([...manifest.collections, ...manifest.through]);
|
|
616
|
+
const dependencies = new Set(manifest.dependencies);
|
|
617
|
+
const outside = (collection, detail, field) => problems.push({ code: "outsidePackage", collection, field, detail });
|
|
618
|
+
for (const name of manifest.dependencies) {
|
|
619
|
+
if (ownOrThrough.has(name)) outside(name, "listed both as a dependency and as part of the package");
|
|
620
|
+
}
|
|
621
|
+
for (const collection of pkg.collections) {
|
|
622
|
+
for (const option of PLACEMENT_OPTIONS) {
|
|
623
|
+
const value = collection[option];
|
|
624
|
+
if (value === void 0 || value === null || value === false) continue;
|
|
625
|
+
if (option === "tableName" && value === collection.name) continue;
|
|
626
|
+
outside(collection.name, `option "${option}" is not allowed`);
|
|
627
|
+
}
|
|
628
|
+
for (const field of collection.fields || []) {
|
|
629
|
+
const fieldClass = this.db.fieldTypes.get(field.type);
|
|
630
|
+
const isRelation = !!(fieldClass == null ? void 0 : fieldClass.prototype) && fieldClass.prototype instanceof import_database.RelationField;
|
|
631
|
+
const relationKeys = ["target", "through", "foreignKey", "otherKey", "sourceKey", "targetKey"];
|
|
632
|
+
if (isRelation || relationKeys.some((key) => field[key] !== void 0 && field[key] !== null)) {
|
|
633
|
+
outside(collection.name, "relation fields are only allowed in associations.json", field.name);
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
const hasRawOptions = (value) => !!value && Object.prototype.hasOwnProperty.call(value, "options");
|
|
638
|
+
for (const collection of pkg.collections) {
|
|
639
|
+
if (hasRawOptions(collection)) outside(collection.name, 'raw "options" is not allowed');
|
|
640
|
+
for (const field of collection.fields || []) {
|
|
641
|
+
if (hasRawOptions(field)) outside(collection.name, 'raw "options" is not allowed', field.name);
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
for (const association of pkg.associations) {
|
|
645
|
+
if (hasRawOptions(association) || hasRawOptions(association.reverseField)) {
|
|
646
|
+
outside(String(association.collectionName), 'raw "options" is not allowed', association.name);
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
for (const association of pkg.associations) {
|
|
650
|
+
const { collectionName, name, type, target, through, reverseField } = association;
|
|
651
|
+
if (!own.has(collectionName)) {
|
|
652
|
+
outside(String(collectionName), "adds a field to a collection that is not in the package", name);
|
|
653
|
+
continue;
|
|
654
|
+
}
|
|
655
|
+
if (!ownOrThrough.has(target) && !dependencies.has(target)) {
|
|
656
|
+
outside(collectionName, `target "${target}" is neither in the package nor a declared dependency`, name);
|
|
657
|
+
}
|
|
658
|
+
if ((type === "hasMany" || type === "hasOne") && !own.has(target)) {
|
|
659
|
+
outside(collectionName, `would add a key column to "${target}"`, name);
|
|
660
|
+
}
|
|
661
|
+
if (reverseField && !own.has(target)) {
|
|
662
|
+
outside(collectionName, `would add a field to "${target}"`, name);
|
|
663
|
+
}
|
|
664
|
+
if (type === "belongsToMany" && (typeof through !== "string" || !ownOrThrough.has(through))) {
|
|
665
|
+
outside(collectionName, `junction table "${String(through)}" is not in the package`, name);
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
checkField(field, collectionName, problems) {
|
|
670
|
+
const name = field.name;
|
|
671
|
+
if (!this.db.fieldTypes.get(field.type)) {
|
|
672
|
+
problems.push({ code: "unknownFieldType", collection: collectionName, field: name, detail: String(field.type) });
|
|
673
|
+
}
|
|
674
|
+
for (const identifier of [name, field.foreignKey, field.otherKey, field.through]) {
|
|
675
|
+
if (typeof identifier === "string" && identifier.length > import_package_format.MAX_IDENTIFIER_LENGTH) {
|
|
676
|
+
problems.push({ code: "identifierTooLong", collection: collectionName, field: name, detail: identifier });
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
async collectionRowExists(name) {
|
|
681
|
+
return await this.db.getRepository("collections").count({ filter: { name } }) > 0;
|
|
682
|
+
}
|
|
683
|
+
expectedTableName(name) {
|
|
684
|
+
const options = this.db.options;
|
|
685
|
+
const tableName = options.underscored ? import_lodash.default.snakeCase(name) : name;
|
|
686
|
+
return `${options.tablePrefix || ""}${tableName}`;
|
|
687
|
+
}
|
|
688
|
+
async existingTableNames() {
|
|
689
|
+
const tables = await this.db.sequelize.getQueryInterface().showAllTables();
|
|
690
|
+
return new Set(
|
|
691
|
+
tables.map(
|
|
692
|
+
(table) => typeof table === "string" ? table : String(table.tableName ?? "")
|
|
693
|
+
)
|
|
694
|
+
);
|
|
695
|
+
}
|
|
696
|
+
/**
|
|
697
|
+
* afterCreateForForeignKeyField catches and only logs its own errors, so a through collection or FK field that
|
|
698
|
+
* failed to be created would not abort the transaction. Verify every expected object explicitly.
|
|
699
|
+
*/
|
|
700
|
+
async assertHookResults(associations, through, transaction) {
|
|
701
|
+
const missing = [];
|
|
702
|
+
for (const name of through) {
|
|
703
|
+
const exists = await this.db.getRepository("collections").count({ filter: { name }, transaction });
|
|
704
|
+
if (!exists || !this.db.hasCollection(name)) missing.push(`collection ${name}`);
|
|
705
|
+
}
|
|
706
|
+
const expectFk = async (collectionName, fieldName) => {
|
|
707
|
+
if (typeof fieldName !== "string") return;
|
|
708
|
+
const collection = this.db.getCollection(collectionName);
|
|
709
|
+
if (!collection) return;
|
|
710
|
+
const exists = await this.db.getRepository("fields").count({ filter: { collectionName, name: fieldName }, transaction });
|
|
711
|
+
if (!exists && !collection.getField(fieldName)) missing.push(`field ${collectionName}.${fieldName}`);
|
|
712
|
+
};
|
|
713
|
+
for (const association of associations) {
|
|
714
|
+
const { type, collectionName, target, foreignKey, otherKey, through: through2 } = association;
|
|
715
|
+
if (type === "belongsTo") await expectFk(collectionName, foreignKey);
|
|
716
|
+
if (type === "hasMany" || type === "hasOne") await expectFk(target, foreignKey);
|
|
717
|
+
if (type === "belongsToMany" && typeof through2 === "string") {
|
|
718
|
+
await expectFk(through2, foreignKey);
|
|
719
|
+
await expectFk(through2, otherKey);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
if (missing.length) {
|
|
723
|
+
throw new Error(`NocoBase did not create: ${missing.join(", ")}`);
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
/** Rows are appended by column name into an existing table, so every package column must exist there. */
|
|
727
|
+
async assertColumnsSubset(name, meta, transaction) {
|
|
728
|
+
const collection = this.db.getCollection(name);
|
|
729
|
+
const describeOptions = { transaction };
|
|
730
|
+
const live = await this.db.sequelize.getQueryInterface().describeTable(collection.getTableNameWithSchema(), describeOptions);
|
|
731
|
+
const missing = meta.columns.filter((column) => !(column in live));
|
|
732
|
+
if (missing.length) {
|
|
733
|
+
throw new Error(`Columns missing in ${name}: ${missing.join(", ")}`);
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
/** The dumped rows are column arrays, so the tables created here must have exactly the source columns. */
|
|
737
|
+
async assertColumns(pkg, names, transaction) {
|
|
738
|
+
for (const name of names) {
|
|
739
|
+
const collection = this.db.getCollection(name);
|
|
740
|
+
const describeOptions = { transaction };
|
|
741
|
+
const live = Object.keys(
|
|
742
|
+
await this.db.sequelize.getQueryInterface().describeTable(collection.getTableNameWithSchema(), describeOptions)
|
|
743
|
+
).sort();
|
|
744
|
+
const expected = [...pkg.metas[name].columns].sort();
|
|
745
|
+
if (!import_lodash.default.isEqual(live, expected)) {
|
|
746
|
+
throw new Error(
|
|
747
|
+
`Columns of ${name} differ from the package: expected [${expected.join(", ")}], got [${live.join(", ")}]`
|
|
748
|
+
);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
/**
|
|
753
|
+
* @param intoExisting rows go into a table that existed before (skip mode): rows whose primary key already exists
|
|
754
|
+
* are left alone, and the returned count is the number of rows actually added.
|
|
755
|
+
*/
|
|
756
|
+
async insertRows(token, name, meta, transaction, importingUserId, intoExisting = false) {
|
|
757
|
+
var _a;
|
|
758
|
+
const dataPath = import_path.default.join(this.packageDir(token), (0, import_package_format.dataDir)(name), "data");
|
|
759
|
+
const exists = await import_promises.default.stat(dataPath).then(() => true).catch(() => false);
|
|
760
|
+
if (!exists || !meta.count) return 0;
|
|
761
|
+
const collection = this.db.getCollection(name);
|
|
762
|
+
const attributes = collection.model.getAttributes();
|
|
763
|
+
const attributeByColumn = new Map(
|
|
764
|
+
Object.entries(attributes).map(([attributeName, attribute]) => [attribute.field || attributeName, attributeName])
|
|
765
|
+
);
|
|
766
|
+
const columnAttributes = Object.fromEntries(
|
|
767
|
+
Object.entries(attributes).map(([attributeName, attribute]) => [attribute.field || attributeName, attribute])
|
|
768
|
+
);
|
|
769
|
+
const queryInterface = this.db.sequelize.getQueryInterface();
|
|
770
|
+
const tableName = collection.getTableNameWithSchema();
|
|
771
|
+
const userColumns = /* @__PURE__ */ new Set();
|
|
772
|
+
for (const field of collection.fields.values()) {
|
|
773
|
+
const fieldOptions = field.options;
|
|
774
|
+
if (field.type !== "belongsTo" || !["createdBy", "updatedBy"].includes(fieldOptions.interface)) continue;
|
|
775
|
+
const foreignKey = fieldOptions.foreignKey;
|
|
776
|
+
if (foreignKey) userColumns.add(((_a = attributes[foreignKey]) == null ? void 0 : _a.field) || foreignKey);
|
|
777
|
+
}
|
|
778
|
+
const countRows = () => collection.repository.count({ transaction });
|
|
779
|
+
const before = intoExisting ? await countRows() : 0;
|
|
780
|
+
const insertOptions = { transaction, ignoreDuplicates: intoExisting };
|
|
781
|
+
let batch = [];
|
|
782
|
+
let count = 0;
|
|
783
|
+
const flush = async () => {
|
|
784
|
+
if (!batch.length) return;
|
|
785
|
+
await queryInterface.bulkInsert(tableName, batch, insertOptions, columnAttributes);
|
|
786
|
+
count += batch.length;
|
|
787
|
+
batch = [];
|
|
788
|
+
};
|
|
789
|
+
for await (const line of readRows(dataPath, CollectionImporter.limits.maxRowBytes)) {
|
|
790
|
+
const values = JSON.parse(line);
|
|
791
|
+
if (!Array.isArray(values)) {
|
|
792
|
+
throw new Error(`Invalid row in ${name}`);
|
|
793
|
+
}
|
|
794
|
+
const record = {};
|
|
795
|
+
meta.columns.forEach((column, index) => {
|
|
796
|
+
const field = collection.getField(attributeByColumn.get(column));
|
|
797
|
+
if (userColumns.has(column)) {
|
|
798
|
+
record[column] = importingUserId;
|
|
799
|
+
return;
|
|
800
|
+
}
|
|
801
|
+
record[column] = field ? import_field_value_writer.FieldValueWriter.write(field, values[index], this.db) : values[index];
|
|
802
|
+
});
|
|
803
|
+
batch.push(record);
|
|
804
|
+
if (batch.length >= INSERT_CHUNK_SIZE) await flush();
|
|
805
|
+
}
|
|
806
|
+
await flush();
|
|
807
|
+
await this.resetSequence(collection, transaction);
|
|
808
|
+
return intoExisting ? await countRows() - before : count;
|
|
809
|
+
}
|
|
810
|
+
/** Serial PKs only: move the sequence past the imported ids. Snowflake/uid PKs have no sequence. */
|
|
811
|
+
async resetSequence(collection, transaction) {
|
|
812
|
+
if (!this.db.inDialect("postgres")) return;
|
|
813
|
+
const entry = Object.entries(collection.model.getAttributes()).find(([, attribute]) => attribute.autoIncrement);
|
|
814
|
+
if (!entry) return;
|
|
815
|
+
const column = entry[1].field || entry[0];
|
|
816
|
+
const table = collection.quotedTableName();
|
|
817
|
+
const quotedColumn = this.db.sequelize.getQueryInterface().quoteIdentifier(column);
|
|
818
|
+
await this.db.sequelize.query(
|
|
819
|
+
`SELECT setval(pg_get_serial_sequence(:table, :column), COALESCE(MAX(${quotedColumn}), 0) + 1, false) FROM ${table}`,
|
|
820
|
+
{ replacements: { table, column }, type: import_sequelize.QueryTypes.SELECT, transaction }
|
|
821
|
+
);
|
|
822
|
+
}
|
|
823
|
+
async removeExpiredUploads() {
|
|
824
|
+
const root = CollectionImporter.uploadDir();
|
|
825
|
+
const entries = await import_promises.default.readdir(root, { withFileTypes: true }).catch(() => []);
|
|
826
|
+
const now = Date.now();
|
|
827
|
+
for (const entry of entries) {
|
|
828
|
+
const entryPath = import_path.default.join(root, entry.name);
|
|
829
|
+
const stat = await import_promises.default.stat(entryPath).catch(() => null);
|
|
830
|
+
if (stat && now - stat.mtimeMs > UPLOAD_TTL_MS) {
|
|
831
|
+
await import_promises.default.rm(entryPath, { recursive: true, force: true });
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
837
|
+
0 && (module.exports = {
|
|
838
|
+
CONFLICT_MODES,
|
|
839
|
+
CollectionImporter,
|
|
840
|
+
DEFAULT_CONFLICT_MODE,
|
|
841
|
+
ImportBlockedError
|
|
842
|
+
});
|