@saltcorn/admin-models 1.6.0 → 1.7.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cjs/index.cjs +18 -0
- package/cjs/models/backup.cjs +18 -0
- package/cjs/models/config-check.cjs +18 -0
- package/cjs/models/pack.cjs +18 -0
- package/cjs/models/snapshot.cjs +18 -0
- package/cjs/models/tenant.cjs +18 -0
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -18
- package/dist/index.js.map +1 -1
- package/dist/models/backup.d.ts +3 -2
- package/dist/models/backup.d.ts.map +1 -1
- package/dist/models/backup.js +175 -167
- package/dist/models/backup.js.map +1 -1
- package/dist/models/config-check.d.ts +2 -1
- package/dist/models/config-check.d.ts.map +1 -1
- package/dist/models/config-check.js +16 -23
- package/dist/models/config-check.js.map +1 -1
- package/dist/models/pack.d.ts +2 -1
- package/dist/models/pack.d.ts.map +1 -1
- package/dist/models/pack.js +121 -150
- package/dist/models/pack.js.map +1 -1
- package/dist/models/snapshot.d.ts +1 -1
- package/dist/models/snapshot.d.ts.map +1 -1
- package/dist/models/snapshot.js +26 -29
- package/dist/models/snapshot.js.map +1 -1
- package/dist/models/tenant.d.ts +1 -1
- package/dist/models/tenant.d.ts.map +1 -1
- package/dist/models/tenant.js +43 -48
- package/dist/models/tenant.js.map +1 -1
- package/dist/tests/backup.test.d.ts +2 -0
- package/dist/tests/backup.test.d.ts.map +1 -0
- package/dist/tests/backup.test.js +201 -0
- package/dist/tests/backup.test.js.map +1 -0
- package/dist/tests/cfgcheck.test.d.ts +2 -0
- package/dist/tests/cfgcheck.test.d.ts.map +1 -0
- package/dist/tests/cfgcheck.test.js +32 -0
- package/dist/tests/cfgcheck.test.js.map +1 -0
- package/dist/tests/pack.test.d.ts +2 -0
- package/dist/tests/pack.test.d.ts.map +1 -0
- package/dist/tests/pack.test.js +769 -0
- package/dist/tests/pack.test.js.map +1 -0
- package/dist/tests/random.test.d.ts +2 -0
- package/dist/tests/random.test.d.ts.map +1 -0
- package/dist/tests/random.test.js +150 -0
- package/dist/tests/random.test.js.map +1 -0
- package/dist/tests/tenant.test.d.ts +2 -0
- package/dist/tests/tenant.test.d.ts.map +1 -0
- package/dist/tests/tenant.test.js +106 -0
- package/dist/tests/tenant.test.js.map +1 -0
- package/gen-cjs-shims.cjs +71 -0
- package/package.json +31 -27
package/dist/models/backup.js
CHANGED
|
@@ -1,88 +1,93 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
const
|
|
1
|
+
import { createRequire } from "module";
|
|
2
|
+
const require = createRequire(import.meta.url);
|
|
3
|
+
import { getState } from "@saltcorn/data/db/state";
|
|
4
|
+
import db from "@saltcorn/data/db/index";
|
|
5
|
+
import Table from "@saltcorn/data/models/table";
|
|
6
|
+
import { instanceOfErrorMsg } from "@saltcorn/types/common_types";
|
|
7
|
+
import View from "@saltcorn/data/models/view";
|
|
8
|
+
import File from "@saltcorn/data/models/file";
|
|
9
|
+
import Crash from "@saltcorn/data/models/crash";
|
|
10
|
+
import Field from "@saltcorn/data/models/field";
|
|
11
|
+
import Role from "@saltcorn/data/models/role";
|
|
12
|
+
import Page from "@saltcorn/data/models/page";
|
|
13
|
+
import PageGroup from "@saltcorn/data/models/page_group";
|
|
14
|
+
import Plugin from "@saltcorn/data/models/plugin";
|
|
15
|
+
import { getMigrationsInDB } from "@saltcorn/data/migrate";
|
|
16
|
+
import Zip from "adm-zip";
|
|
17
|
+
import { dir } from "tmp-promise";
|
|
18
|
+
import { writeFile, mkdir, copyFile, rename, readFile, unlink, readdir, stat, } from "fs/promises";
|
|
19
|
+
import { existsSync, readdirSync, statSync, createReadStream } from "fs";
|
|
20
|
+
import { join, basename } from "path";
|
|
21
|
+
import dateFormatLib from "dateformat";
|
|
22
|
+
const dateFormat = dateFormatLib; // NodeNext default-import interop for dateformat
|
|
23
|
+
import { stringify } from "csv-stringify/sync";
|
|
24
|
+
import csvtojson from "csvtojson";
|
|
25
|
+
import pack from "./pack.js";
|
|
26
|
+
const { table_pack, view_pack, plugin_pack, page_pack, page_group_pack, tag_pack, model_instance_pack, event_log_pack, install_pack, can_install_pack, trigger_pack, } = pack;
|
|
27
|
+
import * as config from "@saltcorn/data/models/config";
|
|
28
|
+
const { configTypes } = config;
|
|
29
|
+
import { asyncMap, isTest } from "@saltcorn/data/utils";
|
|
30
|
+
import Trigger from "@saltcorn/data/models/trigger";
|
|
31
|
+
import Library from "@saltcorn/data/models/library";
|
|
32
|
+
import Tag from "@saltcorn/data/models/tag";
|
|
33
|
+
import Model from "@saltcorn/data/models/model";
|
|
34
|
+
import ModelInstance from "@saltcorn/data/models/model_instance";
|
|
35
|
+
import EventLog from "@saltcorn/data/models/eventlog";
|
|
36
|
+
import path from "path";
|
|
37
|
+
import { execSync, spawn } from "child_process";
|
|
38
|
+
import SftpClient from "ssh2-sftp-client";
|
|
39
|
+
import os from "os";
|
|
40
|
+
import semver from "semver";
|
|
41
|
+
// @aws-sdk is heavy to load (~50ms) and only used when backing up to S3,
|
|
42
|
+
// so require it lazily rather than at module load time.
|
|
43
|
+
let _awsS3;
|
|
44
|
+
const awsS3 = () => _awsS3 || (_awsS3 = require("@aws-sdk/client-s3"));
|
|
45
|
+
let _awsLibStorage;
|
|
46
|
+
const awsLibStorage = () => _awsLibStorage || (_awsLibStorage = require("@aws-sdk/lib-storage"));
|
|
47
|
+
import MetaData from "@saltcorn/data/models/metadata";
|
|
48
|
+
import lodash from "lodash";
|
|
49
|
+
const { orderBy } = lodash;
|
|
45
50
|
/**
|
|
46
51
|
* @param [withEventLog] - include event log
|
|
47
52
|
*/
|
|
48
53
|
const create_pack_json = async (withEventLog = false, forSnapshot = false) => {
|
|
49
54
|
const state = getState();
|
|
50
55
|
// tables
|
|
51
|
-
const tables = await asyncMap(await
|
|
56
|
+
const tables = await asyncMap(await Table.find({}, { orderBy: "id" }), async (t) => await table_pack(t) // find already done before
|
|
52
57
|
);
|
|
53
58
|
// views
|
|
54
|
-
const views = await asyncMap((await
|
|
59
|
+
const views = await asyncMap((await View.find({})).filter((v) => v.id), async (v) => await view_pack(v.name));
|
|
55
60
|
// plugins
|
|
56
|
-
const plugins = await asyncMap(await
|
|
61
|
+
const plugins = await asyncMap(await Plugin.find({}, { orderBy: "id" }), async (v) => await plugin_pack(v.name));
|
|
57
62
|
// pages
|
|
58
|
-
const pages = await asyncMap(await
|
|
63
|
+
const pages = await asyncMap(await Page.find({}), async (v) => await page_pack(v.name));
|
|
59
64
|
// page groups
|
|
60
|
-
const page_groups = await asyncMap(await
|
|
65
|
+
const page_groups = await asyncMap(await PageGroup.find({}), async (v) => await page_group_pack(v.name));
|
|
61
66
|
// triggers
|
|
62
|
-
const triggers = await asyncMap(
|
|
67
|
+
const triggers = await asyncMap(Trigger.find({}), trigger_pack);
|
|
63
68
|
// roles
|
|
64
|
-
const roles = await
|
|
69
|
+
const roles = await Role.find({});
|
|
65
70
|
// library
|
|
66
|
-
const library = (await
|
|
71
|
+
const library = (await Library.find({})).map((l) => l.toJson);
|
|
67
72
|
// tags
|
|
68
|
-
const tags = await asyncMap(await
|
|
73
|
+
const tags = await asyncMap(await Tag.find({}), async (v) => await tag_pack(v.name));
|
|
69
74
|
// models
|
|
70
|
-
const models = (await
|
|
75
|
+
const models = (await Model.find({})).map((m) => m.toJson);
|
|
71
76
|
// model instances
|
|
72
77
|
const model_instances = forSnapshot
|
|
73
78
|
? []
|
|
74
|
-
: await asyncMap(await
|
|
75
|
-
const model = await
|
|
79
|
+
: await asyncMap(await ModelInstance.find({}), async (modelinst) => {
|
|
80
|
+
const model = await Model.findOne({ id: modelinst.model_id });
|
|
76
81
|
if (!model)
|
|
77
82
|
throw new Error(`Model of instance '${modelinst.name}' not found`);
|
|
78
|
-
const table = await
|
|
83
|
+
const table = await Table.findOne({ id: model.table_id });
|
|
79
84
|
if (!table)
|
|
80
85
|
throw new Error(`Table of model '${model.name}' not found`);
|
|
81
86
|
return await model_instance_pack(modelinst.name, model.name, table.name);
|
|
82
87
|
});
|
|
83
88
|
// optional event log
|
|
84
89
|
const event_logs = withEventLog
|
|
85
|
-
? await asyncMap(await
|
|
90
|
+
? await asyncMap(await EventLog.find({}), async (e) => await event_log_pack(e))
|
|
86
91
|
: [];
|
|
87
92
|
const function_code_pages = state.getConfigCopy("function_code_pages", {});
|
|
88
93
|
const function_code_pages_tags = state.getConfigCopy("function_code_pages_tags", {});
|
|
@@ -110,7 +115,7 @@ const create_pack_json = async (withEventLog = false, forSnapshot = false) => {
|
|
|
110
115
|
code_pages,
|
|
111
116
|
};
|
|
112
117
|
if (forSnapshot) {
|
|
113
|
-
const cfgs = await
|
|
118
|
+
const cfgs = await db.select("_sc_config");
|
|
114
119
|
const config = {};
|
|
115
120
|
for (const cfg of cfgs) {
|
|
116
121
|
//exclude base url, multitenancy, ssl/lets encrypt,
|
|
@@ -118,7 +123,7 @@ const create_pack_json = async (withEventLog = false, forSnapshot = false) => {
|
|
|
118
123
|
continue;
|
|
119
124
|
if (state.isFixedConfig(cfg.key))
|
|
120
125
|
continue;
|
|
121
|
-
config[cfg.key] = (
|
|
126
|
+
config[cfg.key] = (db.isSQLite ? JSON.parse(cfg.value) : cfg.value)?.v;
|
|
122
127
|
}
|
|
123
128
|
pack.config = config;
|
|
124
129
|
}
|
|
@@ -129,7 +134,7 @@ const create_pack_json = async (withEventLog = false, forSnapshot = false) => {
|
|
|
129
134
|
*/
|
|
130
135
|
const create_pack = async (dirpath) => {
|
|
131
136
|
const pack = await create_pack_json(getState().getConfig("backup_with_event_log", false));
|
|
132
|
-
await
|
|
137
|
+
await writeFile(join(dirpath, "pack.json"), JSON.stringify(pack));
|
|
133
138
|
};
|
|
134
139
|
/**
|
|
135
140
|
*
|
|
@@ -139,13 +144,13 @@ const create_pack = async (dirpath) => {
|
|
|
139
144
|
const create_csv_from_rows = async (rows, fnm) => {
|
|
140
145
|
if (rows.length === 0)
|
|
141
146
|
return;
|
|
142
|
-
const s =
|
|
147
|
+
const s = stringify(rows, {
|
|
143
148
|
header: true,
|
|
144
149
|
cast: {
|
|
145
150
|
date: (value) => value.toISOString(),
|
|
146
151
|
},
|
|
147
152
|
});
|
|
148
|
-
await
|
|
153
|
+
await writeFile(fnm, s);
|
|
149
154
|
};
|
|
150
155
|
const sanitiseTableName = (nm) => nm.replaceAll("/", "");
|
|
151
156
|
/**
|
|
@@ -155,7 +160,7 @@ const sanitiseTableName = (nm) => nm.replaceAll("/", "");
|
|
|
155
160
|
* @returns {Promise<void>}
|
|
156
161
|
*/
|
|
157
162
|
const create_table_json = async (table, dirpath) => {
|
|
158
|
-
await table.dump_to_json(
|
|
163
|
+
await table.dump_to_json(join(dirpath, sanitiseTableName(table.name) + ".json"));
|
|
159
164
|
};
|
|
160
165
|
/**
|
|
161
166
|
* @function
|
|
@@ -163,15 +168,15 @@ const create_table_json = async (table, dirpath) => {
|
|
|
163
168
|
* @return {Promise<void>}
|
|
164
169
|
*/
|
|
165
170
|
const create_table_jsons = async (root_dirpath) => {
|
|
166
|
-
const dirpath =
|
|
167
|
-
await
|
|
168
|
-
const tables = await
|
|
171
|
+
const dirpath = join(root_dirpath, "tables");
|
|
172
|
+
await mkdir(dirpath, { recursive: true });
|
|
173
|
+
const tables = await Table.find({});
|
|
169
174
|
const backup_history = getState().getConfig("backup_history", true);
|
|
170
175
|
for (const t of tables) {
|
|
171
176
|
if (!t.external && !t.provider_name) {
|
|
172
177
|
await create_table_json(t, dirpath);
|
|
173
178
|
if (t.versioned && backup_history) {
|
|
174
|
-
await t.dump_history_to_json(
|
|
179
|
+
await t.dump_history_to_json(join(dirpath, sanitiseTableName(t.name) + "__history.json"));
|
|
175
180
|
}
|
|
176
181
|
}
|
|
177
182
|
}
|
|
@@ -183,17 +188,17 @@ const create_table_jsons = async (root_dirpath) => {
|
|
|
183
188
|
*/
|
|
184
189
|
const backup_files = async (root_dirpath) => {
|
|
185
190
|
const backup_file_prefix = getState().getConfig("backup_file_prefix");
|
|
186
|
-
const dirpath =
|
|
187
|
-
await
|
|
191
|
+
const dirpath = join(root_dirpath, "files");
|
|
192
|
+
await mkdir(dirpath);
|
|
188
193
|
const allFiles = [];
|
|
189
194
|
const iterFolder = async (folder) => {
|
|
190
|
-
const files = await
|
|
195
|
+
const files = await File.find(folder ? { folder } : {});
|
|
191
196
|
for (const f of files) {
|
|
192
|
-
const base =
|
|
197
|
+
const base = basename(f.location);
|
|
193
198
|
if (f.isDirectory && (await f.is_symlink()))
|
|
194
199
|
continue;
|
|
195
200
|
else if (f.isDirectory && !(await f.is_symlink())) {
|
|
196
|
-
await
|
|
201
|
+
await mkdir(join(dirpath, f.path_to_serve));
|
|
197
202
|
await iterFolder(f.path_to_serve);
|
|
198
203
|
}
|
|
199
204
|
else {
|
|
@@ -202,22 +207,22 @@ const backup_files = async (root_dirpath) => {
|
|
|
202
207
|
f.mime_sub === "zip" &&
|
|
203
208
|
!f.user_id)
|
|
204
209
|
continue;
|
|
205
|
-
await
|
|
210
|
+
await copyFile(f.location, join(dirpath, folder || "", base));
|
|
206
211
|
}
|
|
207
|
-
f.location =
|
|
212
|
+
f.location = path.join(folder || "", base);
|
|
208
213
|
allFiles.push(f);
|
|
209
214
|
}
|
|
210
215
|
};
|
|
211
216
|
await iterFolder();
|
|
212
|
-
await create_csv_from_rows(allFiles,
|
|
217
|
+
await create_csv_from_rows(allFiles, join(root_dirpath, "files.csv"));
|
|
213
218
|
};
|
|
214
219
|
const backup_migrations = async (root_dirpath) => {
|
|
215
|
-
const migrations = await
|
|
216
|
-
await
|
|
220
|
+
const migrations = await getMigrationsInDB();
|
|
221
|
+
await writeFile(join(root_dirpath, "migrations.json"), JSON.stringify(migrations));
|
|
217
222
|
};
|
|
218
223
|
const backup_metadata = async (root_dirpath) => {
|
|
219
|
-
const metadata = await
|
|
220
|
-
await
|
|
224
|
+
const metadata = await MetaData.find({});
|
|
225
|
+
await writeFile(join(root_dirpath, "metadata.json"), JSON.stringify(metadata));
|
|
221
226
|
};
|
|
222
227
|
/**
|
|
223
228
|
* @function
|
|
@@ -225,26 +230,26 @@ const backup_metadata = async (root_dirpath) => {
|
|
|
225
230
|
* @returns {Promise<void>}
|
|
226
231
|
*/
|
|
227
232
|
const backup_config = async (root_dirpath) => {
|
|
228
|
-
const dirpath =
|
|
229
|
-
await
|
|
230
|
-
const cfgs = await
|
|
233
|
+
const dirpath = join(root_dirpath, "config");
|
|
234
|
+
await mkdir(dirpath);
|
|
235
|
+
const cfgs = await db.select("_sc_config");
|
|
231
236
|
const state = getState();
|
|
232
237
|
for (const cfg of cfgs) {
|
|
233
238
|
if (!state.isFixedConfig(cfg.key))
|
|
234
|
-
await
|
|
239
|
+
await writeFile(join(dirpath, cfg.key), JSON.stringify(db.isSQLite ? JSON.parse(cfg.value) : cfg.value));
|
|
235
240
|
}
|
|
236
241
|
};
|
|
237
242
|
const backup_info_file = async (root_dirpath) => {
|
|
238
243
|
const state = getState();
|
|
239
|
-
const migrations_run = await
|
|
240
|
-
const dbversion = await
|
|
241
|
-
const saltcorn_version =
|
|
242
|
-
await
|
|
244
|
+
const migrations_run = await getMigrationsInDB();
|
|
245
|
+
const dbversion = await db.getVersion(true);
|
|
246
|
+
const saltcorn_version = db.connectObj.sc_version;
|
|
247
|
+
await writeFile(join(root_dirpath, "backup-info.json"), JSON.stringify({
|
|
243
248
|
saltcorn_version,
|
|
244
249
|
migrations_run,
|
|
245
250
|
backup_date: new Date().toISOString(),
|
|
246
251
|
node_version: process.version,
|
|
247
|
-
database_type:
|
|
252
|
+
database_type: db.isSQLite ? "SQLite" : "PostgreSQL",
|
|
248
253
|
database_version: dbversion,
|
|
249
254
|
os: {
|
|
250
255
|
platform: os.platform(),
|
|
@@ -261,7 +266,7 @@ const zipFolder = async (folder, zipFileName) => {
|
|
|
261
266
|
const backup_password = getState().getConfig("backup_password", "");
|
|
262
267
|
if (backup_with_system_zip) {
|
|
263
268
|
return await new Promise((resolve, reject) => {
|
|
264
|
-
const absZipPath =
|
|
269
|
+
const absZipPath = path.join(process.cwd(), zipFileName);
|
|
265
270
|
const args = [
|
|
266
271
|
"-5",
|
|
267
272
|
"-rq",
|
|
@@ -285,7 +290,7 @@ const zipFolder = async (folder, zipFileName) => {
|
|
|
285
290
|
});
|
|
286
291
|
}
|
|
287
292
|
else {
|
|
288
|
-
const zip = new
|
|
293
|
+
const zip = new Zip();
|
|
289
294
|
zip.addLocalFolder(folder);
|
|
290
295
|
zip.writeZip(zipFileName);
|
|
291
296
|
}
|
|
@@ -295,7 +300,7 @@ const zipFolder = async (folder, zipFileName) => {
|
|
|
295
300
|
* @param fnm
|
|
296
301
|
*/
|
|
297
302
|
const create_backup = async (fnm) => {
|
|
298
|
-
const tmpDir = await
|
|
303
|
+
const tmpDir = await dir({ unsafeCleanup: true });
|
|
299
304
|
await create_pack(tmpDir.path);
|
|
300
305
|
await create_table_jsons(tmpDir.path);
|
|
301
306
|
await backup_files(tmpDir.path);
|
|
@@ -303,9 +308,9 @@ const create_backup = async (fnm) => {
|
|
|
303
308
|
await backup_migrations(tmpDir.path);
|
|
304
309
|
await backup_info_file(tmpDir.path);
|
|
305
310
|
await backup_metadata(tmpDir.path);
|
|
306
|
-
const day = (
|
|
307
|
-
const ten =
|
|
308
|
-
const tens = ten ===
|
|
311
|
+
const day = dateFormat(new Date(), "yyyy-mm-dd-HH-MM");
|
|
312
|
+
const ten = db.getTenantSchema();
|
|
313
|
+
const tens = ten === db.connectObj.default_schema
|
|
309
314
|
? getState().getConfig("site_name", "Saltcorn")
|
|
310
315
|
: ten;
|
|
311
316
|
const backup_file_prefix = getState().getConfig("backup_file_prefix");
|
|
@@ -338,7 +343,7 @@ const extract = async (fnm, dir, password) => {
|
|
|
338
343
|
return await new Promise((resolve, reject) => {
|
|
339
344
|
const args = [
|
|
340
345
|
...(password ? [`-P${password}`] : []),
|
|
341
|
-
|
|
346
|
+
File.normalise(fnm),
|
|
342
347
|
"-d",
|
|
343
348
|
dir,
|
|
344
349
|
];
|
|
@@ -362,7 +367,7 @@ const extract = async (fnm, dir, password) => {
|
|
|
362
367
|
});
|
|
363
368
|
}
|
|
364
369
|
else {
|
|
365
|
-
const zip = new
|
|
370
|
+
const zip = new Zip(fnm);
|
|
366
371
|
try {
|
|
367
372
|
if (password) {
|
|
368
373
|
zip.extractAllTo(dir, true, false, password);
|
|
@@ -386,39 +391,39 @@ const extract = async (fnm, dir, password) => {
|
|
|
386
391
|
* @returns {Promise<object>}
|
|
387
392
|
*/
|
|
388
393
|
const restore_files = async (dirpath) => {
|
|
389
|
-
const fnm =
|
|
394
|
+
const fnm = join(dirpath, "files.csv");
|
|
390
395
|
const file_users = {};
|
|
391
396
|
const newLocations = {};
|
|
392
397
|
const state = getState();
|
|
393
|
-
if (
|
|
394
|
-
const file_rows = await (
|
|
398
|
+
if (existsSync(fnm)) {
|
|
399
|
+
const file_rows = await csvtojson().fromFile(fnm);
|
|
395
400
|
for (const file of file_rows) {
|
|
396
401
|
if (file.isDirectory)
|
|
397
|
-
await
|
|
402
|
+
await mkdir(File.get_new_path(file.location), { recursive: true });
|
|
398
403
|
}
|
|
399
404
|
if (!isTest())
|
|
400
405
|
state.log(2, `Restoring ${file_rows.length} files...`);
|
|
401
406
|
for (const file of file_rows) {
|
|
402
407
|
try {
|
|
403
|
-
const newPath =
|
|
408
|
+
const newPath = File.get_new_path(file.id ? file.filename : file.location);
|
|
404
409
|
//copy file
|
|
405
410
|
if (!file.isDirectory)
|
|
406
|
-
await
|
|
411
|
+
await copyFile(join(dirpath, "files", file.location), newPath);
|
|
407
412
|
file_users[file.location] = file.user_id;
|
|
408
413
|
//set location
|
|
409
414
|
if (file.id)
|
|
410
415
|
newLocations[file.id] = file.id ? file.filename : file.location;
|
|
411
416
|
file.location = newPath;
|
|
412
417
|
//insert in db
|
|
413
|
-
await
|
|
418
|
+
await File.create(file);
|
|
414
419
|
//const id = await db.insert("_sc_files", file_row);}
|
|
415
420
|
}
|
|
416
421
|
catch (e) {
|
|
417
422
|
state.log(1, `Error restoring file ${JSON.stringify(file)}: ${e.message}`);
|
|
418
423
|
}
|
|
419
424
|
}
|
|
420
|
-
if (
|
|
421
|
-
await
|
|
425
|
+
if (db.reset_sequence)
|
|
426
|
+
await db.reset_sequence("_sc_files");
|
|
422
427
|
}
|
|
423
428
|
return { file_users, newLocations };
|
|
424
429
|
};
|
|
@@ -427,10 +432,10 @@ const restore_files = async (dirpath) => {
|
|
|
427
432
|
* @param newLocations
|
|
428
433
|
*/
|
|
429
434
|
const correct_fileid_references_to_location = async (newLocations) => {
|
|
430
|
-
const fileFields = await
|
|
435
|
+
const fileFields = await Field.find({ type: "File" });
|
|
431
436
|
getState().log(2, `Correcting file id references to locations`);
|
|
432
437
|
for (const field of fileFields) {
|
|
433
|
-
const table =
|
|
438
|
+
const table = Table.findOne({ id: field.table_id });
|
|
434
439
|
const rows = await table.getRows({});
|
|
435
440
|
for (const row of rows) {
|
|
436
441
|
if (row[field.name] && newLocations[row[field.name]]) {
|
|
@@ -449,7 +454,7 @@ const restore_file_users = async (file_users) => {
|
|
|
449
454
|
getState().log(2, `Restoring file users`);
|
|
450
455
|
for (const [id, user_id] of Object.entries(file_users)) {
|
|
451
456
|
if (user_id) {
|
|
452
|
-
const file = await
|
|
457
|
+
const file = await File.findOne(id);
|
|
453
458
|
if (file)
|
|
454
459
|
await file.set_user(user_id);
|
|
455
460
|
//await db.update("_sc_files", { user_id }, id);
|
|
@@ -464,34 +469,34 @@ const restore_file_users = async (file_users) => {
|
|
|
464
469
|
*/
|
|
465
470
|
const restore_tables = async (dirpath, restore_first_user) => {
|
|
466
471
|
let err;
|
|
467
|
-
const tables = await
|
|
472
|
+
const tables = await Table.find();
|
|
468
473
|
const restore_history = getState().getConfig("restore_history", true);
|
|
469
474
|
for (const table of tables) {
|
|
470
475
|
if (!isTest())
|
|
471
476
|
getState().log(2, `restoring table ${table.name}`);
|
|
472
477
|
const fnm_csv = table.name === "users"
|
|
473
|
-
?
|
|
474
|
-
:
|
|
475
|
-
const fnm_json =
|
|
476
|
-
if (
|
|
478
|
+
? join(dirpath, "users.csv")
|
|
479
|
+
: join(dirpath, "tables", sanitiseTableName(table.name) + ".csv");
|
|
480
|
+
const fnm_json = join(dirpath, "tables", sanitiseTableName(table.name) + ".json");
|
|
481
|
+
if (existsSync(fnm_json)) {
|
|
477
482
|
const res = await table.import_json_file(fnm_json, table.name === "users" && !restore_first_user);
|
|
478
|
-
if (
|
|
483
|
+
if (instanceOfErrorMsg(res)) {
|
|
479
484
|
console.error(err);
|
|
480
485
|
err = (err || "") + res.error;
|
|
481
486
|
}
|
|
482
487
|
}
|
|
483
|
-
else if (
|
|
488
|
+
else if (existsSync(fnm_csv)) {
|
|
484
489
|
const res = await table.import_csv_file(fnm_csv, {
|
|
485
490
|
skip_first_data_row: table.name === "users" && !restore_first_user,
|
|
486
491
|
});
|
|
487
|
-
if (
|
|
492
|
+
if (instanceOfErrorMsg(res)) {
|
|
488
493
|
console.error(res);
|
|
489
494
|
err = (err || "") + res.error;
|
|
490
495
|
}
|
|
491
496
|
}
|
|
492
497
|
if (table.versioned && restore_history) {
|
|
493
|
-
const fnm_hist_json =
|
|
494
|
-
if (
|
|
498
|
+
const fnm_hist_json = join(dirpath, "tables", sanitiseTableName(table.name) + "__history.json");
|
|
499
|
+
if (existsSync(fnm_hist_json)) {
|
|
495
500
|
if (!isTest())
|
|
496
501
|
getState().log(2, `restoring table history ${table.name}`);
|
|
497
502
|
await table.import_json_history_file(fnm_hist_json);
|
|
@@ -515,22 +520,22 @@ const restore_tables = async (dirpath, restore_first_user) => {
|
|
|
515
520
|
* @returns {Promise<void>}
|
|
516
521
|
*/
|
|
517
522
|
const restore_config = async (dirpath) => {
|
|
518
|
-
const cfgs =
|
|
523
|
+
const cfgs = readdirSync(join(dirpath, "config"));
|
|
519
524
|
const state = getState();
|
|
520
525
|
for (const cfg of cfgs) {
|
|
521
|
-
const s = await
|
|
526
|
+
const s = await readFile(join(dirpath, "config", cfg));
|
|
522
527
|
await state.setConfig(cfg, JSON.parse(s.toString()).v);
|
|
523
528
|
}
|
|
524
529
|
};
|
|
525
530
|
const restore_metadata = async (dirpath) => {
|
|
526
|
-
const fnm =
|
|
527
|
-
if (!
|
|
531
|
+
const fnm = join(dirpath, "metadata.json");
|
|
532
|
+
if (!existsSync(fnm))
|
|
528
533
|
return;
|
|
529
534
|
if (!isTest())
|
|
530
535
|
getState().log(2, `Restoring metadata`);
|
|
531
|
-
const mds = JSON.parse((await
|
|
536
|
+
const mds = JSON.parse((await readFile(fnm)).toString());
|
|
532
537
|
for (const md of mds) {
|
|
533
|
-
await
|
|
538
|
+
await MetaData.create(md);
|
|
534
539
|
}
|
|
535
540
|
};
|
|
536
541
|
/**
|
|
@@ -541,19 +546,19 @@ const restore_metadata = async (dirpath) => {
|
|
|
541
546
|
*/
|
|
542
547
|
const restore = async (fnm, loadAndSaveNewPlugin, restore_first_user, password) => {
|
|
543
548
|
const state = getState();
|
|
544
|
-
state.log(2, `Starting restore to tenant ${
|
|
545
|
-
const tmpDir = await
|
|
549
|
+
state.log(2, `Starting restore to tenant ${db.getTenantSchema()}`);
|
|
550
|
+
const tmpDir = await dir({ unsafeCleanup: true });
|
|
546
551
|
await extract(fnm, tmpDir.path, password);
|
|
547
552
|
state.log(2, `Unzip done`);
|
|
548
553
|
let basePath = tmpDir.path;
|
|
549
554
|
// safari re-compressed. Safari unpacks zip files on download. If the user
|
|
550
555
|
// chooses compress in finder, the backup dir is nested inside the zip file
|
|
551
|
-
if (!
|
|
552
|
-
const files = await
|
|
556
|
+
if (!existsSync(join(basePath, "pack.json"))) {
|
|
557
|
+
const files = await readdir(basePath);
|
|
553
558
|
let found = false;
|
|
554
559
|
for (const file of files) {
|
|
555
|
-
if (
|
|
556
|
-
basePath =
|
|
560
|
+
if (existsSync(join(basePath, file, "pack.json"))) {
|
|
561
|
+
basePath = join(basePath, file);
|
|
557
562
|
found = true;
|
|
558
563
|
break;
|
|
559
564
|
}
|
|
@@ -562,9 +567,9 @@ const restore = async (fnm, loadAndSaveNewPlugin, restore_first_user, password)
|
|
|
562
567
|
return "Not a valid backup file";
|
|
563
568
|
}
|
|
564
569
|
let err;
|
|
565
|
-
if (
|
|
566
|
-
const info = JSON.parse((await
|
|
567
|
-
const saltcorn_version =
|
|
570
|
+
if (existsSync(join(basePath, "backup-info.json"))) {
|
|
571
|
+
const info = JSON.parse((await readFile(join(basePath, "backup-info.json"))).toString());
|
|
572
|
+
const saltcorn_version = db.connectObj.sc_version;
|
|
568
573
|
if (info.saltcorn_version &&
|
|
569
574
|
semver.gt(info.saltcorn_version, saltcorn_version)) {
|
|
570
575
|
err = `Warning: backup is from a more recent version (${info.saltcorn_version}) than the installed version (${saltcorn_version}). `;
|
|
@@ -573,7 +578,7 @@ const restore = async (fnm, loadAndSaveNewPlugin, restore_first_user, password)
|
|
|
573
578
|
//install pack
|
|
574
579
|
if (!isTest())
|
|
575
580
|
state.log(2, `Reading pack`);
|
|
576
|
-
const pack = JSON.parse((await
|
|
581
|
+
const pack = JSON.parse((await readFile(join(basePath, "pack.json"))).toString());
|
|
577
582
|
const can_restore = await can_install_pack(pack);
|
|
578
583
|
if (typeof can_restore !== "boolean" && can_restore.error) {
|
|
579
584
|
return `Cannot restore backup, clashing entities:
|
|
@@ -605,7 +610,7 @@ const restore = async (fnm, loadAndSaveNewPlugin, restore_first_user, password)
|
|
|
605
610
|
await correct_fileid_references_to_location(newLocations);
|
|
606
611
|
await restore_file_users(file_users);
|
|
607
612
|
await tmpDir.cleanup();
|
|
608
|
-
state.log(2, `Completed restore to tenant ${
|
|
613
|
+
state.log(2, `Completed restore to tenant ${db.getTenantSchema()}${err ? ` with errors ${err}` : " successfully"}`);
|
|
609
614
|
return err;
|
|
610
615
|
};
|
|
611
616
|
/**
|
|
@@ -619,18 +624,18 @@ const delete_old_backups = async () => {
|
|
|
619
624
|
if (!expire_days || expire_days < 0)
|
|
620
625
|
return;
|
|
621
626
|
if (destination === "Local directory") {
|
|
622
|
-
const files = await
|
|
627
|
+
const files = await readdir(directory);
|
|
623
628
|
for (const file of files) {
|
|
624
629
|
if (!file.startsWith(backup_file_prefix))
|
|
625
630
|
continue;
|
|
626
|
-
const stats = await
|
|
631
|
+
const stats = await stat(path.join(directory, file));
|
|
627
632
|
const fileTime = stats.birthtime?.getTime
|
|
628
633
|
? stats.birthtime.getTime()
|
|
629
634
|
: stats.mtime.getTime();
|
|
630
635
|
const ageMs = new Date().getTime() - fileTime;
|
|
631
636
|
const ageDays = ageMs / (1000 * 3600 * 24);
|
|
632
637
|
if (ageDays > expire_days)
|
|
633
|
-
await
|
|
638
|
+
await unlink(path.join(directory, file));
|
|
634
639
|
}
|
|
635
640
|
}
|
|
636
641
|
else if (destination === "S3") {
|
|
@@ -641,7 +646,7 @@ const delete_old_backups = async () => {
|
|
|
641
646
|
? s3EndpointCfg
|
|
642
647
|
: `${s3Secure ? "https" : "http"}://${s3EndpointCfg}`
|
|
643
648
|
: undefined;
|
|
644
|
-
const s3 = new
|
|
649
|
+
const s3 = new (awsS3().S3Client)({
|
|
645
650
|
credentials: {
|
|
646
651
|
accessKeyId: getState().getConfig("backup_s3_access_key"),
|
|
647
652
|
secretAccessKey: getState().getConfig("backup_s3_access_secret"),
|
|
@@ -663,7 +668,7 @@ const delete_old_backups = async () => {
|
|
|
663
668
|
try {
|
|
664
669
|
let continuationToken = undefined;
|
|
665
670
|
do {
|
|
666
|
-
const listedObjects = await s3.send(new
|
|
671
|
+
const listedObjects = await s3.send(new (awsS3().ListObjectsV2Command)({
|
|
667
672
|
...listParams,
|
|
668
673
|
ContinuationToken: continuationToken,
|
|
669
674
|
}));
|
|
@@ -674,7 +679,10 @@ const delete_old_backups = async () => {
|
|
|
674
679
|
const ageMs = new Date().getTime() - new Date(obj.LastModified).getTime();
|
|
675
680
|
const ageDays = ageMs / (1000 * 3600 * 24);
|
|
676
681
|
if (ageDays > expire_days) {
|
|
677
|
-
await s3.send(new
|
|
682
|
+
await s3.send(new (awsS3().DeleteObjectCommand)({
|
|
683
|
+
Bucket: bucket,
|
|
684
|
+
Key: obj.Key,
|
|
685
|
+
}));
|
|
678
686
|
}
|
|
679
687
|
}
|
|
680
688
|
}
|
|
@@ -703,12 +711,12 @@ const auto_backup_now_tenant = async (state) => {
|
|
|
703
711
|
switch (destination) {
|
|
704
712
|
case "Saltcorn files":
|
|
705
713
|
if (directory.length > 0) {
|
|
706
|
-
await
|
|
714
|
+
await File.new_folder(directory);
|
|
707
715
|
}
|
|
708
|
-
const newPath =
|
|
709
|
-
const stats =
|
|
710
|
-
await
|
|
711
|
-
await
|
|
716
|
+
const newPath = File.get_new_path(join(directory, fileName));
|
|
717
|
+
const stats = statSync(fileName);
|
|
718
|
+
await copyFile(fileName, newPath);
|
|
719
|
+
await File.create({
|
|
712
720
|
filename: fileName,
|
|
713
721
|
location: newPath,
|
|
714
722
|
uploaded_at: new Date(),
|
|
@@ -717,35 +725,35 @@ const auto_backup_now_tenant = async (state) => {
|
|
|
717
725
|
mime_sub: "zip",
|
|
718
726
|
min_role_read: 1,
|
|
719
727
|
});
|
|
720
|
-
await
|
|
728
|
+
await unlink(fileName);
|
|
721
729
|
break;
|
|
722
730
|
case "Local directory":
|
|
723
731
|
//const directory = state.getConfig("auto_backup_directory");
|
|
724
732
|
if (directory.length > 0) {
|
|
725
|
-
await
|
|
733
|
+
await mkdir(directory, { recursive: true });
|
|
726
734
|
}
|
|
727
|
-
await
|
|
735
|
+
await rename(fileName, join(directory, fileName));
|
|
728
736
|
await delete_old_backups();
|
|
729
737
|
break;
|
|
730
738
|
case "SFTP server":
|
|
731
|
-
let sftp = new
|
|
739
|
+
let sftp = new SftpClient();
|
|
732
740
|
await sftp.connect({
|
|
733
741
|
host: state.getConfig("auto_backup_server"),
|
|
734
742
|
port: state.getConfig("auto_backup_port"),
|
|
735
743
|
username: state.getConfig("auto_backup_username"),
|
|
736
744
|
password: state.getConfig("auto_backup_password"),
|
|
737
745
|
});
|
|
738
|
-
let data =
|
|
739
|
-
let remote =
|
|
746
|
+
let data = createReadStream(fileName);
|
|
747
|
+
let remote = join(state.getConfig("auto_backup_directory", ""), basename(fileName));
|
|
740
748
|
const putres = await sftp.put(data, remote);
|
|
741
749
|
state.log(6, `SFTP Put response: ${putres}`);
|
|
742
750
|
await sftp.end();
|
|
743
751
|
const retain_dir = state.getConfig("auto_backup_retain_local_directory");
|
|
744
752
|
if (retain_dir) {
|
|
745
|
-
await
|
|
746
|
-
await
|
|
753
|
+
await mkdir(retain_dir, { recursive: true });
|
|
754
|
+
await copyFile(fileName, join(retain_dir, fileName));
|
|
747
755
|
}
|
|
748
|
-
await
|
|
756
|
+
await unlink(fileName);
|
|
749
757
|
break;
|
|
750
758
|
case "S3":
|
|
751
759
|
const bEndpointCfg = state.getConfig("backup_s3_endpoint");
|
|
@@ -755,7 +763,7 @@ const auto_backup_now_tenant = async (state) => {
|
|
|
755
763
|
? bEndpointCfg
|
|
756
764
|
: `${bSecure ? "https" : "http"}://${bEndpointCfg}`
|
|
757
765
|
: undefined;
|
|
758
|
-
const s3 = new
|
|
766
|
+
const s3 = new (awsS3().S3)({
|
|
759
767
|
credentials: {
|
|
760
768
|
accessKeyId: state.getConfig("backup_s3_access_key"),
|
|
761
769
|
secretAccessKey: state.getConfig("backup_s3_access_secret"),
|
|
@@ -767,12 +775,12 @@ const auto_backup_now_tenant = async (state) => {
|
|
|
767
775
|
const pathPrefix = (state.getConfig("backup_s3_path_prefix", "") || "")
|
|
768
776
|
.toString()
|
|
769
777
|
.replace(/^\/+|\/+$/g, "");
|
|
770
|
-
const s3Key = [pathPrefix,
|
|
778
|
+
const s3Key = [pathPrefix, basename(fileName)]
|
|
771
779
|
.filter((s) => s && s.length)
|
|
772
780
|
.join("/");
|
|
773
|
-
const fileStream = () =>
|
|
781
|
+
const fileStream = () => createReadStream(fileName);
|
|
774
782
|
try {
|
|
775
|
-
const uploadResult = await new
|
|
783
|
+
const uploadResult = await new (awsLibStorage().Upload)({
|
|
776
784
|
client: s3,
|
|
777
785
|
params: {
|
|
778
786
|
Bucket: bucket,
|
|
@@ -797,14 +805,14 @@ const auto_backup_now_tenant = async (state) => {
|
|
|
797
805
|
}
|
|
798
806
|
};
|
|
799
807
|
const auto_backup_now = async () => {
|
|
800
|
-
const isRoot =
|
|
808
|
+
const isRoot = db.getTenantSchema() === db.connectObj.default_schema;
|
|
801
809
|
const state = getState();
|
|
802
|
-
const tenantModule =
|
|
810
|
+
const tenantModule = (await import("./tenant.js")).default;
|
|
803
811
|
if (isRoot && state.getConfig("auto_backup_tenants"))
|
|
804
812
|
await tenantModule.eachTenant(async () => {
|
|
805
813
|
try {
|
|
806
814
|
await auto_backup_now_tenant(state);
|
|
807
|
-
await
|
|
815
|
+
await MetaData.create({
|
|
808
816
|
type: "Backup",
|
|
809
817
|
name: "Success",
|
|
810
818
|
body: {},
|
|
@@ -813,7 +821,7 @@ const auto_backup_now = async () => {
|
|
|
813
821
|
}
|
|
814
822
|
catch (e) {
|
|
815
823
|
console.error(e);
|
|
816
|
-
await
|
|
824
|
+
await Crash.create(e, {
|
|
817
825
|
url: `Scheduler auto backup for tenant`,
|
|
818
826
|
headers: {},
|
|
819
827
|
});
|
|
@@ -823,7 +831,7 @@ const auto_backup_now = async () => {
|
|
|
823
831
|
else
|
|
824
832
|
try {
|
|
825
833
|
await auto_backup_now_tenant(state);
|
|
826
|
-
await
|
|
834
|
+
await MetaData.create({
|
|
827
835
|
type: "Backup",
|
|
828
836
|
name: "Success",
|
|
829
837
|
body: {},
|
|
@@ -832,14 +840,14 @@ const auto_backup_now = async () => {
|
|
|
832
840
|
}
|
|
833
841
|
catch (e) {
|
|
834
842
|
console.error(e);
|
|
835
|
-
await
|
|
843
|
+
await Crash.create(e, {
|
|
836
844
|
url: `Scheduler auto backup`,
|
|
837
845
|
headers: {},
|
|
838
846
|
});
|
|
839
847
|
throw new Error(e);
|
|
840
848
|
}
|
|
841
849
|
};
|
|
842
|
-
|
|
850
|
+
export default {
|
|
843
851
|
create_backup,
|
|
844
852
|
restore,
|
|
845
853
|
create_csv_from_rows,
|