@aglyn/tenant-data-admin 1.0.0-beta.221 → 1.0.0-beta.222
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -7
- package/src/lib/server/api-keys.js +3 -3
- package/src/lib/server/api-keys.js.map +1 -1
- package/src/lib/server/api-v1-resources.d.ts +44 -3
- package/src/lib/server/api-v1-resources.js +82 -5
- package/src/lib/server/api-v1-resources.js.map +1 -1
- package/src/lib/server/collection-live-pages.d.ts +2 -2
- package/src/lib/server/collection-live-pages.js +1 -1
- package/src/lib/server/collection-live-pages.js.map +1 -1
- package/src/lib/server/email-delivery-log.d.ts +17 -73
- package/src/lib/server/email-delivery-log.js +44 -222
- package/src/lib/server/email-delivery-log.js.map +1 -1
- package/src/lib/server/email-flow-gate.js +27 -38
- package/src/lib/server/email-flow-gate.js.map +1 -1
- package/src/lib/server/erase-person.d.ts +34 -54
- package/src/lib/server/erase-person.js +44 -239
- package/src/lib/server/erase-person.js.map +1 -1
- package/src/lib/server/host-visitor-records.d.ts +12 -4
- package/src/lib/server/host-visitor-records.js +20 -10
- package/src/lib/server/host-visitor-records.js.map +1 -1
- package/src/lib/server/hosted-page-review.js +46 -33
- package/src/lib/server/hosted-page-review.js.map +1 -1
- package/src/lib/server/link-reputation-review.d.ts +6 -2
- package/src/lib/server/link-reputation-review.js +61 -20
- package/src/lib/server/link-reputation-review.js.map +1 -1
- package/src/lib/server/live-page-drops.d.ts +67 -0
- package/src/lib/server/live-page-drops.js +57 -0
- package/src/lib/server/live-page-drops.js.map +1 -0
- package/src/lib/server/media-bandwidth-cap.d.ts +78 -0
- package/src/lib/server/media-bandwidth-cap.js +180 -0
- package/src/lib/server/media-bandwidth-cap.js.map +1 -0
- package/src/lib/server/media-cdn-icon.d.ts +77 -0
- package/src/lib/server/media-cdn-icon.js +140 -0
- package/src/lib/server/media-cdn-icon.js.map +1 -0
- package/src/lib/server/media-delivery-regeneration.d.ts +132 -0
- package/src/lib/server/media-delivery-regeneration.js +216 -0
- package/src/lib/server/media-delivery-regeneration.js.map +1 -0
- package/src/lib/server/media-delivery.d.ts +2 -0
- package/src/lib/server/media-delivery.js +2 -1
- package/src/lib/server/media-delivery.js.map +1 -1
- package/src/lib/server/media-serve-count.d.ts +50 -0
- package/src/lib/server/media-serve-count.js +62 -0
- package/src/lib/server/media-serve-count.js.map +1 -0
- package/src/lib/server/media-signing.d.ts +13 -2
- package/src/lib/server/media-signing.js +12 -8
- package/src/lib/server/media-signing.js.map +1 -1
- package/src/lib/server/media-tombstone.js +7 -5
- package/src/lib/server/media-tombstone.js.map +1 -1
- package/src/lib/server/media-variants.d.ts +151 -20
- package/src/lib/server/media-variants.js +204 -27
- package/src/lib/server/media-variants.js.map +1 -1
- package/src/lib/server/organizations.d.ts +102 -2
- package/src/lib/server/organizations.js +388 -58
- package/src/lib/server/organizations.js.map +1 -1
- package/src/lib/server/outbound-send-review.js +5 -4
- package/src/lib/server/outbound-send-review.js.map +1 -1
- package/src/lib/server/paged-sweeps.d.ts +51 -0
- package/src/lib/server/paged-sweeps.js +79 -0
- package/src/lib/server/paged-sweeps.js.map +1 -0
- package/src/lib/server/platform-marketing-consent.d.ts +0 -49
- package/src/lib/server/platform-marketing-consent.js +27 -11
- package/src/lib/server/platform-marketing-consent.js.map +1 -1
- package/src/lib/server/serve-media-cdn.d.ts +69 -1
- package/src/lib/server/serve-media-cdn.js +424 -70
- package/src/lib/server/serve-media-cdn.js.map +1 -1
- package/src/lib/server/site-icon-render.d.ts +68 -0
- package/src/lib/server/site-icon-render.js +157 -0
- package/src/lib/server/site-icon-render.js.map +1 -0
- package/src/lib/server/staff-seat.d.ts +50 -0
- package/src/lib/server/staff-seat.js +118 -0
- package/src/lib/server/staff-seat.js.map +1 -0
- package/src/lib/server/transfer-export.d.ts +46 -0
- package/src/lib/server/transfer-export.js +221 -0
- package/src/lib/server/transfer-export.js.map +1 -0
- package/src/lib/server/transfer-jobs.d.ts +385 -0
- package/src/lib/server/transfer-jobs.js +2368 -0
- package/src/lib/server/transfer-jobs.js.map +1 -0
- package/src/lib/server/transfer-packages.d.ts +127 -0
- package/src/lib/server/transfer-packages.js +1036 -0
- package/src/lib/server/transfer-packages.js.map +1 -0
- package/src/lib/server/upgrade-proposal.d.ts +97 -0
- package/src/lib/server/upgrade-proposal.js +312 -0
- package/src/lib/server/upgrade-proposal.js.map +1 -0
- package/src/lib/server/upsert-contact.d.ts +17 -8
- package/src/lib/server/upsert-contact.js +82 -16
- package/src/lib/server/upsert-contact.js.map +1 -1
- package/src/lib/server/web-risk.d.ts +48 -9
- package/src/lib/server/web-risk.js +281 -143
- package/src/lib/server/web-risk.js.map +1 -1
- package/src/tenancy.d.ts +3 -1
- package/src/tenancy.js +5 -1
- package/src/tenancy.js.map +1 -1
- package/src/lib/server/campaign-attribution-store.d.ts +0 -55
- package/src/lib/server/campaign-attribution-store.js +0 -95
- package/src/lib/server/campaign-attribution-store.js.map +0 -1
- package/src/lib/server/campaign-conversion-attribution.d.ts +0 -283
- package/src/lib/server/campaign-conversion-attribution.js +0 -450
- package/src/lib/server/campaign-conversion-attribution.js.map +0 -1
- package/src/lib/server/dataset-live-pages.d.ts +0 -179
- package/src/lib/server/dataset-live-pages.js +0 -421
- package/src/lib/server/dataset-live-pages.js.map +0 -1
- package/src/lib/server/email-revenue-attribution.d.ts +0 -185
- package/src/lib/server/email-revenue-attribution.js +0 -321
- package/src/lib/server/email-revenue-attribution.js.map +0 -1
|
@@ -0,0 +1,1036 @@
|
|
|
1
|
+
import { _ as _extends } from "@swc/helpers/_/_extends";
|
|
2
|
+
/**
|
|
3
|
+
* @license
|
|
4
|
+
* Copyright 2026 Aglyn LLC
|
|
5
|
+
*
|
|
6
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
* you may not use this file except in compliance with the License.
|
|
8
|
+
* You may obtain a copy of the License at
|
|
9
|
+
*
|
|
10
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
*
|
|
12
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
+
* See the License for the specific language governing permissions and
|
|
16
|
+
* limitations under the License.
|
|
17
|
+
*/ /*==========================================
|
|
18
|
+
* WORKSPACE PACKAGES (AGL-3535) — a workspace's sequences, campaigns,
|
|
19
|
+
* automations and email templates as one file, imported as a job.
|
|
20
|
+
*
|
|
21
|
+
* A row import's file is rows; a package's is items, each owned by the
|
|
22
|
+
* plugin whose `package` transfer resource its kind names. So a package
|
|
23
|
+
* does not go through the row engine's upload, analyze and chunked plan —
|
|
24
|
+
* there is no column to map — but it is the same kind of job, under the
|
|
25
|
+
* same guarantees, in the same place:
|
|
26
|
+
*
|
|
27
|
+
* - the job is `orgs/{orgId}/transferJobs/{jobId}` with `kind: 'package'`
|
|
28
|
+
* and `resource: 'package'`, so the hub's history lists it beside every
|
|
29
|
+
* row import, the sweep resumes and cleans it, and only one package
|
|
30
|
+
* import runs per workspace at a time (the lease's per-resource lock);
|
|
31
|
+
* - the file is stored where a row import's is
|
|
32
|
+
* (`orgs/{orgId}/transfers/{jobId}/source`), and the dry run in
|
|
33
|
+
* `chunks/0` — the plan the person reviewed is the plan applied;
|
|
34
|
+
* - every item's write is recorded in the ledger the moment it lands, so
|
|
35
|
+
* a retried apply never writes an item twice;
|
|
36
|
+
* - the undo snapshot (`undo/0`) is taken BEFORE the first write: each
|
|
37
|
+
* replaced item's content as the workspace held it. Once the writes
|
|
38
|
+
* land, each written item's content hash is added to it, so undo can
|
|
39
|
+
* tell the import's own content from a later edit and asks before it
|
|
40
|
+
* overwrites or deletes one — for seven days;
|
|
41
|
+
* - plan, apply and undo are audited by the route.
|
|
42
|
+
*
|
|
43
|
+
* Why not the row engine's routes: a row import plans 200 rows a chunk
|
|
44
|
+
* against a column mapping; a package is planned whole, against item
|
|
45
|
+
* identities and references (`planTransferPackage`), and its writes are
|
|
46
|
+
* grouped by owning plugin in dependency order. One route
|
|
47
|
+
* (`/api/transfer/package`) speaks its actions; this module is the engine
|
|
48
|
+
* behind it, built on the row engine's job, ledger and lease.
|
|
49
|
+
*
|
|
50
|
+
* The plugin reads, writes and reverts through its own paths; this module
|
|
51
|
+
* decides what to write, in which order, under which id, and what undo
|
|
52
|
+
* does.
|
|
53
|
+
*=========================================*/ import { TRANSFER_PACKAGE_MAX_BYTES, TRANSFER_PACKAGE_MAX_ITEMS, TRANSFER_PACKAGE_RESOURCE, TRANSFER_RESULT_COLUMNS, TRANSFER_SITE_KIND, buildPackageManifest, canApplyTransferPackage, contentHash, missingPackageAcknowledgements, packageIdMap, packageItemKey, planTransferPackage, readPackageManifest, summarizeTransferResults, transferLedgerKey, transferUndoAvailable, transferUndoExpiresAt, TRANSFER_WRITE_CONCURRENCY } from "@aglyn/aglyn/data-transfer";
|
|
54
|
+
import { createResourceUid } from "@aglyn/aglyn/app-utils/create-resource-uid";
|
|
55
|
+
import { escapeCsvCell } from "@aglyn/aglyn/app-utils/csv";
|
|
56
|
+
import { inspectUploadBytes } from "@aglyn/aglyn/app-utils/upload-inspection";
|
|
57
|
+
import { resolveTransferResource, transferPackageHooks, TransferResourceUnavailableError } from "@aglyn/aglyn/plugin-manager/plugin-transfer-resources";
|
|
58
|
+
import { clock, eachLimited, emptyResults, emptyUndoCounts, isAlreadyExists, moveJob, readJob, readJsonDoc, saveJob, stored, takeLease, transferJobsCollection, transferSourcePath, TransferEngineError, TRANSFER_LEASE_GRACE_MS, TRANSFER_MIN_CHUNK_BUDGET_MS, writeJsonDoc } from "./transfer-jobs.js";
|
|
59
|
+
/** The package resources among `keys` that the workspace may move, by key. */ async function packageResources(deps, keys) {
|
|
60
|
+
const found = new Map();
|
|
61
|
+
for (const key of keys){
|
|
62
|
+
try {
|
|
63
|
+
var _deps_resolveResource;
|
|
64
|
+
const resource = await ((_deps_resolveResource = deps.resolveResource) != null ? _deps_resolveResource : resolveTransferResource)(key);
|
|
65
|
+
if (resource.scope !== 'org' || !resource.kinds.includes('package')) continue;
|
|
66
|
+
found.set(resource.key, {
|
|
67
|
+
resource,
|
|
68
|
+
hooks: transferPackageHooks(resource)
|
|
69
|
+
});
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (error instanceof TransferResourceUnavailableError && error.reason === 'undeclared') continue;
|
|
72
|
+
if (error instanceof TransferResourceUnavailableError) {
|
|
73
|
+
throw new TransferEngineError('unavailable', 500, 'Importing packages is unavailable right now. Try again in a minute.');
|
|
74
|
+
}
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return found;
|
|
79
|
+
}
|
|
80
|
+
function contextOf(orgId, resource, actorUid, jobId) {
|
|
81
|
+
return _extends({
|
|
82
|
+
resource,
|
|
83
|
+
orgId,
|
|
84
|
+
hostId: null,
|
|
85
|
+
actorUid
|
|
86
|
+
}, jobId ? {
|
|
87
|
+
jobId
|
|
88
|
+
} : {});
|
|
89
|
+
}
|
|
90
|
+
function rulesOf(entry) {
|
|
91
|
+
var _entry_hooks_rules;
|
|
92
|
+
return ((_entry_hooks_rules = entry.hooks.rules) != null ? _entry_hooks_rules : []).map((rule)=>({
|
|
93
|
+
id: rule.id,
|
|
94
|
+
label: rule.label,
|
|
95
|
+
reason: rule.reason
|
|
96
|
+
}));
|
|
97
|
+
}
|
|
98
|
+
/** The workspace's sites, as references to the `site` kind resolve. */ async function siteTargets(deps, orgId) {
|
|
99
|
+
const snapshot = await deps.firestore.collection('hosts').where('orgId', '==', orgId).get();
|
|
100
|
+
return {
|
|
101
|
+
kind: TRANSFER_SITE_KIND,
|
|
102
|
+
label: 'Site',
|
|
103
|
+
targets: snapshot.docs.filter((doc)=>!doc.data().deletedAt).map((doc)=>{
|
|
104
|
+
var _ref, _data_name;
|
|
105
|
+
const data = doc.data();
|
|
106
|
+
const name = String((_ref = (_data_name = data.name) != null ? _data_name : data.title) != null ? _ref : '').trim();
|
|
107
|
+
return _extends({
|
|
108
|
+
id: doc.id
|
|
109
|
+
}, name ? {
|
|
110
|
+
name
|
|
111
|
+
} : {});
|
|
112
|
+
})
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
/** Every item of each package resource, with its name and what it names, for the export picker. */ export async function listTransferPackageItems(deps, input) {
|
|
116
|
+
var _input_resources;
|
|
117
|
+
const keys = ((_input_resources = input.resources) == null ? void 0 : _input_resources.length) ? input.allowed.filter((key)=>{
|
|
118
|
+
var _input_resources;
|
|
119
|
+
return (_input_resources = input.resources) == null ? void 0 : _input_resources.includes(key);
|
|
120
|
+
}) : input.allowed;
|
|
121
|
+
const resources = await packageResources(deps, keys);
|
|
122
|
+
const lists = [];
|
|
123
|
+
for (const [key, entry] of resources){
|
|
124
|
+
const ctx = contextOf(input.orgId, key, input.actorUid);
|
|
125
|
+
const held = await entry.hooks.items(ctx);
|
|
126
|
+
const contents = held.length ? await entry.hooks.readItems(ctx, held.map((item)=>item.id)) : [];
|
|
127
|
+
const items = contents.map((one)=>_extends({
|
|
128
|
+
key: packageItemKey({
|
|
129
|
+
kind: key,
|
|
130
|
+
id: one.id
|
|
131
|
+
}),
|
|
132
|
+
kind: key,
|
|
133
|
+
id: one.id
|
|
134
|
+
}, one.name ? {
|
|
135
|
+
name: one.name
|
|
136
|
+
} : {}, {
|
|
137
|
+
deps: entry.hooks.dependencies(one.content)
|
|
138
|
+
}));
|
|
139
|
+
lists.push(_extends({
|
|
140
|
+
key,
|
|
141
|
+
label: entry.resource.label,
|
|
142
|
+
pluginId: entry.resource.pluginId
|
|
143
|
+
}, entry.resource.description ? {
|
|
144
|
+
description: entry.resource.description
|
|
145
|
+
} : {}, {
|
|
146
|
+
items,
|
|
147
|
+
rules: rulesOf(entry)
|
|
148
|
+
}));
|
|
149
|
+
}
|
|
150
|
+
return lists;
|
|
151
|
+
}
|
|
152
|
+
/** The package file: the chosen items (and, when asked, what they need), hashed and listed. */ export async function exportTransferPackage(deps, input) {
|
|
153
|
+
var _input_resources, _input_items;
|
|
154
|
+
const resources = await packageResources(deps, input.allowed);
|
|
155
|
+
const wanted = ((_input_resources = input.resources) == null ? void 0 : _input_resources.length) ? new Set(input.resources) : null;
|
|
156
|
+
const contents = new Map();
|
|
157
|
+
const read = async (key, ids)=>{
|
|
158
|
+
const entry = resources.get(key);
|
|
159
|
+
if (!entry || !ids.length) return;
|
|
160
|
+
for (const one of (await entry.hooks.readItems(contextOf(input.orgId, key, input.actorUid), ids))){
|
|
161
|
+
contents.set(packageItemKey({
|
|
162
|
+
kind: key,
|
|
163
|
+
id: one.id
|
|
164
|
+
}), _extends({
|
|
165
|
+
kind: key,
|
|
166
|
+
id: one.id
|
|
167
|
+
}, one.name ? {
|
|
168
|
+
name: one.name
|
|
169
|
+
} : {}, one.slug ? {
|
|
170
|
+
slug: one.slug
|
|
171
|
+
} : {}, {
|
|
172
|
+
content: one.content
|
|
173
|
+
}));
|
|
174
|
+
}
|
|
175
|
+
};
|
|
176
|
+
if ((_input_items = input.items) == null ? void 0 : _input_items.length) {
|
|
177
|
+
const byKind = new Map();
|
|
178
|
+
for (const itemKey of input.items){
|
|
179
|
+
var _byKind_get;
|
|
180
|
+
const at = itemKey.indexOf('/');
|
|
181
|
+
const kind = itemKey.slice(0, at);
|
|
182
|
+
if (at < 1 || wanted && !wanted.has(kind)) continue;
|
|
183
|
+
byKind.set(kind, [
|
|
184
|
+
...(_byKind_get = byKind.get(kind)) != null ? _byKind_get : [],
|
|
185
|
+
itemKey.slice(at + 1)
|
|
186
|
+
]);
|
|
187
|
+
}
|
|
188
|
+
for (const [kind, ids] of byKind)await read(kind, ids);
|
|
189
|
+
} else {
|
|
190
|
+
for (const [key, entry] of resources){
|
|
191
|
+
if (wanted && !wanted.has(key)) continue;
|
|
192
|
+
const held = await entry.hooks.items(contextOf(input.orgId, key, input.actorUid));
|
|
193
|
+
await read(key, held.map((item)=>item.id));
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
if (input.dependencies) {
|
|
197
|
+
// What the chosen items name that is itself a package item here, until nothing new is named.
|
|
198
|
+
for(let pass = 0; pass < TRANSFER_PACKAGE_MAX_ITEMS; pass += 1){
|
|
199
|
+
const missing = new Map();
|
|
200
|
+
for (const item of contents.values()){
|
|
201
|
+
var _ref;
|
|
202
|
+
var _resources_get;
|
|
203
|
+
for (const dep of (_ref = (_resources_get = resources.get(item.kind)) == null ? void 0 : _resources_get.hooks.dependencies(item.content)) != null ? _ref : []){
|
|
204
|
+
var _missing_get;
|
|
205
|
+
if (!resources.has(dep.kind) || contents.has(packageItemKey(dep))) continue;
|
|
206
|
+
missing.set(dep.kind, [
|
|
207
|
+
...new Set([
|
|
208
|
+
...(_missing_get = missing.get(dep.kind)) != null ? _missing_get : [],
|
|
209
|
+
dep.id
|
|
210
|
+
])
|
|
211
|
+
]);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
if (!missing.size) break;
|
|
215
|
+
const before = contents.size;
|
|
216
|
+
for (const [kind, ids] of missing)await read(kind, ids);
|
|
217
|
+
if (contents.size === before) break;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
if (contents.size > TRANSFER_PACKAGE_MAX_ITEMS) {
|
|
221
|
+
throw new TransferEngineError('tooLarge', 413, `A package holds at most ${TRANSFER_PACKAGE_MAX_ITEMS} items. Choose fewer.`);
|
|
222
|
+
}
|
|
223
|
+
return buildPackageManifest([
|
|
224
|
+
...contents.values()
|
|
225
|
+
], (item)=>{
|
|
226
|
+
var _ref;
|
|
227
|
+
var _resources_get;
|
|
228
|
+
const held = contents.get(packageItemKey(item));
|
|
229
|
+
return _extends({}, (held == null ? void 0 : held.name) ? {
|
|
230
|
+
name: held.name
|
|
231
|
+
} : {}, (held == null ? void 0 : held.slug) ? {
|
|
232
|
+
slug: held.slug
|
|
233
|
+
} : {}, {
|
|
234
|
+
deps: (_ref = (_resources_get = resources.get(item.kind)) == null ? void 0 : _resources_get.hooks.dependencies(item.content)) != null ? _ref : []
|
|
235
|
+
});
|
|
236
|
+
}, _extends({
|
|
237
|
+
createdAt: clock(deps)
|
|
238
|
+
}, input.source ? {
|
|
239
|
+
source: input.source
|
|
240
|
+
} : {}));
|
|
241
|
+
}
|
|
242
|
+
/** A package read from untrusted JSON, every item's content present and matching its hash — or why not. */ async function readPackageFile(raw) {
|
|
243
|
+
const source = raw && typeof raw === 'object' ? raw : {};
|
|
244
|
+
const read = readPackageManifest(source.manifest);
|
|
245
|
+
if (read.ok === false) {
|
|
246
|
+
throw new TransferEngineError('rejectedFile', 422, 'This file is not a package this workspace can import.', {
|
|
247
|
+
problems: read.problems
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
const items = source.items && typeof source.items === 'object' ? source.items : {};
|
|
251
|
+
if (read.manifest.items.length > TRANSFER_PACKAGE_MAX_ITEMS) {
|
|
252
|
+
throw new TransferEngineError('tooLarge', 413, `A package holds at most ${TRANSFER_PACKAGE_MAX_ITEMS} items.`);
|
|
253
|
+
}
|
|
254
|
+
const problems = [];
|
|
255
|
+
for (const item of read.manifest.items){
|
|
256
|
+
const key = packageItemKey(item);
|
|
257
|
+
if (!(key in items)) {
|
|
258
|
+
var _item_name;
|
|
259
|
+
problems.push(`${(_item_name = item.name) != null ? _item_name : key} is listed and not carried.`);
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
if (await contentHash(items[key]) !== item.contentHash) {
|
|
263
|
+
var _item_name1;
|
|
264
|
+
problems.push(`${(_item_name1 = item.name) != null ? _item_name1 : key} does not match its content hash: the file was changed after it was made.`);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
if (problems.length) {
|
|
268
|
+
throw new TransferEngineError('rejectedFile', 422, 'This package is damaged.', {
|
|
269
|
+
problems
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
return {
|
|
273
|
+
manifest: read.manifest,
|
|
274
|
+
items
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
async function loadPackage(deps, job) {
|
|
278
|
+
if (!job.sourcePath) throw new TransferEngineError('state', 409, 'The file behind this import is gone.');
|
|
279
|
+
const [bytes] = await deps.bucket.file(job.sourcePath).download();
|
|
280
|
+
return JSON.parse(bytes.toString('utf8'));
|
|
281
|
+
}
|
|
282
|
+
async function readStoredPlan(deps, job) {
|
|
283
|
+
const plan = await readJsonDoc(transferJobsCollection(deps.firestore, job.orgId).doc(job.id).collection('chunks').doc('0'));
|
|
284
|
+
if (!plan) throw new TransferEngineError('state', 409, 'This import has no dry run.');
|
|
285
|
+
return plan;
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* The dry run (see the block header): reads what the workspace holds of
|
|
289
|
+
* every kind the package carries or names, plans with the person's
|
|
290
|
+
* choices, asks each written item's plugin what its own write would
|
|
291
|
+
* refuse, and stores the plan. Writes no item.
|
|
292
|
+
*/ export async function planTransferPackageImport(deps, input) {
|
|
293
|
+
var _ref, _input_decisions, _ref1, _input_dependencyChoices;
|
|
294
|
+
var _job_package, _job_package1;
|
|
295
|
+
const now = clock(deps);
|
|
296
|
+
let job;
|
|
297
|
+
let pkg;
|
|
298
|
+
let created = false;
|
|
299
|
+
if (input.jobId) {
|
|
300
|
+
job = await readJob(deps, input.orgId, input.jobId);
|
|
301
|
+
if (job.kind !== 'package') throw new TransferEngineError('invalid', 400, 'This import is not a package.');
|
|
302
|
+
if (job.applyStartedAt !== undefined) {
|
|
303
|
+
throw new TransferEngineError('state', 409, 'This import has started writing and cannot be planned again.');
|
|
304
|
+
}
|
|
305
|
+
if (job.status !== 'planned' && job.status !== 'failed') {
|
|
306
|
+
throw new TransferEngineError('state', 409, `This import is ${job.status} and cannot be planned again.`);
|
|
307
|
+
}
|
|
308
|
+
pkg = await loadPackage(deps, job);
|
|
309
|
+
} else {
|
|
310
|
+
var _deps_newJobId, _input_fileName, _input_fileName1;
|
|
311
|
+
if (input.package === undefined) throw new TransferEngineError('invalid', 400, 'Choose a package file.');
|
|
312
|
+
const text = JSON.stringify(input.package);
|
|
313
|
+
if (Buffer.byteLength(text, 'utf8') > TRANSFER_PACKAGE_MAX_BYTES) {
|
|
314
|
+
throw new TransferEngineError('tooLarge', 413, 'A package file holds at most 3 MB.');
|
|
315
|
+
}
|
|
316
|
+
pkg = await readPackageFile(input.package);
|
|
317
|
+
const id = ((_deps_newJobId = deps.newJobId) != null ? _deps_newJobId : createResourceUid)();
|
|
318
|
+
const sourcePath = transferSourcePath(input.orgId, id);
|
|
319
|
+
// A member's file, stored beside the workspace's other objects: inspected like every upload first.
|
|
320
|
+
const bytes = Buffer.from(JSON.stringify(pkg), 'utf8');
|
|
321
|
+
const refusal = inspectUploadBytes({
|
|
322
|
+
bytes,
|
|
323
|
+
contentType: 'application/json',
|
|
324
|
+
fileName: String((_input_fileName = input.fileName) != null ? _input_fileName : 'package.json')
|
|
325
|
+
});
|
|
326
|
+
if (refusal) throw new TransferEngineError('rejectedFile', 422, refusal.message, {
|
|
327
|
+
code: refusal.code
|
|
328
|
+
});
|
|
329
|
+
await deps.bucket.file(sourcePath).save(bytes, {
|
|
330
|
+
contentType: 'application/json',
|
|
331
|
+
resumable: false
|
|
332
|
+
});
|
|
333
|
+
created = true;
|
|
334
|
+
job = {
|
|
335
|
+
id,
|
|
336
|
+
resource: TRANSFER_PACKAGE_RESOURCE,
|
|
337
|
+
kind: 'package',
|
|
338
|
+
direction: 'import',
|
|
339
|
+
format: 'json',
|
|
340
|
+
status: 'planned',
|
|
341
|
+
orgId: input.orgId,
|
|
342
|
+
createdBy: input.actorUid,
|
|
343
|
+
createdAt: now,
|
|
344
|
+
updatedAt: now,
|
|
345
|
+
fileName: String((_input_fileName1 = input.fileName) != null ? _input_fileName1 : '').trim().slice(0, 200) || 'package.json',
|
|
346
|
+
sourcePath,
|
|
347
|
+
rowCount: pkg.manifest.items.length,
|
|
348
|
+
chunkCount: 1
|
|
349
|
+
};
|
|
350
|
+
// Kept before planning, so a plan that fails still leaves a job the sweep expires with its file.
|
|
351
|
+
await saveJob(deps, job);
|
|
352
|
+
}
|
|
353
|
+
const resources = await packageResources(deps, input.allowed);
|
|
354
|
+
const owned = new Set(resources.keys());
|
|
355
|
+
const manifest = pkg.manifest;
|
|
356
|
+
const kindsNeeded = new Set();
|
|
357
|
+
const foreignByResource = new Map();
|
|
358
|
+
let needsSites = false;
|
|
359
|
+
for (const item of manifest.items){
|
|
360
|
+
if (!owned.has(item.kind)) continue;
|
|
361
|
+
kindsNeeded.add(item.kind);
|
|
362
|
+
for (const dep of item.deps){
|
|
363
|
+
var _foreignByResource_get;
|
|
364
|
+
if (owned.has(dep.kind)) kindsNeeded.add(dep.kind);
|
|
365
|
+
else if (dep.kind === TRANSFER_SITE_KIND) needsSites = true;
|
|
366
|
+
else foreignByResource.set(item.kind, new Set([
|
|
367
|
+
...(_foreignByResource_get = foreignByResource.get(item.kind)) != null ? _foreignByResource_get : [],
|
|
368
|
+
dep.kind
|
|
369
|
+
]));
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
const existing = [];
|
|
373
|
+
for (const kind of kindsNeeded){
|
|
374
|
+
const entry = resources.get(kind);
|
|
375
|
+
for (const one of (await entry.hooks.items(contextOf(input.orgId, kind, input.actorUid, job.id)))){
|
|
376
|
+
existing.push(_extends({}, one, {
|
|
377
|
+
kind
|
|
378
|
+
}));
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
const references = needsSites ? [
|
|
382
|
+
await siteTargets(deps, input.orgId)
|
|
383
|
+
] : [];
|
|
384
|
+
for (const [key, kinds] of foreignByResource){
|
|
385
|
+
var _resources_get_hooks_referenceTargets, _resources_get;
|
|
386
|
+
const answered = await ((_resources_get = resources.get(key)) == null ? void 0 : (_resources_get_hooks_referenceTargets = _resources_get.hooks.referenceTargets) == null ? void 0 : _resources_get_hooks_referenceTargets.call(_resources_get.hooks, contextOf(input.orgId, key, input.actorUid, job.id), [
|
|
387
|
+
...kinds
|
|
388
|
+
]));
|
|
389
|
+
for (const kind of answered != null ? answered : []){
|
|
390
|
+
if (!references.some((known)=>known.kind === kind.kind)) references.push(kind);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
const kindLabels = Object.fromEntries([
|
|
394
|
+
...resources
|
|
395
|
+
].map(([key, entry])=>{
|
|
396
|
+
var _entry_resource_singularLabel;
|
|
397
|
+
return [
|
|
398
|
+
key,
|
|
399
|
+
(_entry_resource_singularLabel = entry.resource.singularLabel) != null ? _entry_resource_singularLabel : entry.resource.label
|
|
400
|
+
];
|
|
401
|
+
}));
|
|
402
|
+
// Kept-both ids are minted once and reused by the second pass, so the problems asked about are the ids applied.
|
|
403
|
+
const minted = [];
|
|
404
|
+
let mintedAt = 0;
|
|
405
|
+
const newId = ()=>{
|
|
406
|
+
if (mintedAt >= minted.length) minted.push(createResourceUid());
|
|
407
|
+
return minted[mintedAt++];
|
|
408
|
+
};
|
|
409
|
+
const base = {
|
|
410
|
+
manifest,
|
|
411
|
+
ownedKinds: owned,
|
|
412
|
+
existing,
|
|
413
|
+
references,
|
|
414
|
+
kindLabels,
|
|
415
|
+
decisions: (_ref = (_input_decisions = input.decisions) != null ? _input_decisions : (_job_package = job.package) == null ? void 0 : _job_package.decisions) != null ? _ref : {},
|
|
416
|
+
dependencyChoices: (_ref1 = (_input_dependencyChoices = input.dependencyChoices) != null ? _input_dependencyChoices : (_job_package1 = job.package) == null ? void 0 : _job_package1.dependencyChoices) != null ? _ref1 : {}
|
|
417
|
+
};
|
|
418
|
+
const first = planTransferPackage(_extends({}, base, {
|
|
419
|
+
newId
|
|
420
|
+
}));
|
|
421
|
+
const problems = {};
|
|
422
|
+
const idMap = packageIdMap(first.idMap);
|
|
423
|
+
for (const item of first.items){
|
|
424
|
+
if (item.verdict === 'skip' || item.targetId === undefined) continue;
|
|
425
|
+
const entry = resources.get(item.kind);
|
|
426
|
+
const hook = entry.hooks.problems;
|
|
427
|
+
if (!hook) continue;
|
|
428
|
+
const write = writeFor(entry, item, manifest.items[item.row], pkg.items[item.key], idMap);
|
|
429
|
+
const found = await hook(contextOf(input.orgId, item.kind, input.actorUid, job.id), write);
|
|
430
|
+
if (found.length) problems[item.key] = [
|
|
431
|
+
...found
|
|
432
|
+
];
|
|
433
|
+
}
|
|
434
|
+
mintedAt = 0;
|
|
435
|
+
const plan = planTransferPackage(_extends({}, base, {
|
|
436
|
+
problems,
|
|
437
|
+
newId
|
|
438
|
+
}));
|
|
439
|
+
const jobRef = transferJobsCollection(deps.firestore, input.orgId).doc(job.id);
|
|
440
|
+
const kept = {
|
|
441
|
+
items: plan.items,
|
|
442
|
+
references: plan.references,
|
|
443
|
+
order: plan.order,
|
|
444
|
+
idMap: plan.idMap
|
|
445
|
+
};
|
|
446
|
+
await writeJsonDoc(jobRef.collection('chunks').doc('0'), {
|
|
447
|
+
jobId: job.id,
|
|
448
|
+
index: 0,
|
|
449
|
+
package: true
|
|
450
|
+
}, kept);
|
|
451
|
+
const state = _extends({}, manifest.source ? {
|
|
452
|
+
source: manifest.source
|
|
453
|
+
} : {}, {
|
|
454
|
+
resources: [
|
|
455
|
+
...new Set(plan.items.filter((item)=>owned.has(item.kind)).map((item)=>item.kind))
|
|
456
|
+
],
|
|
457
|
+
summary: plan.summary,
|
|
458
|
+
blocking: plan.blocking,
|
|
459
|
+
acknowledgementsRequired: plan.acknowledgementsRequired,
|
|
460
|
+
unknownKinds: plan.unknownKinds,
|
|
461
|
+
decisions: _extends({}, base.decisions),
|
|
462
|
+
dependencyChoices: _extends({}, base.dependencyChoices)
|
|
463
|
+
});
|
|
464
|
+
job = job.status === 'failed' ? moveJob(job, 'planned', clock(deps), {
|
|
465
|
+
package: state,
|
|
466
|
+
plannedAt: clock(deps)
|
|
467
|
+
}) : _extends({}, job, {
|
|
468
|
+
status: 'planned',
|
|
469
|
+
package: state,
|
|
470
|
+
plannedAt: clock(deps),
|
|
471
|
+
updatedAt: clock(deps)
|
|
472
|
+
});
|
|
473
|
+
await saveJob(deps, job);
|
|
474
|
+
return {
|
|
475
|
+
job,
|
|
476
|
+
plan,
|
|
477
|
+
created,
|
|
478
|
+
resources: Object.fromEntries([
|
|
479
|
+
...resources
|
|
480
|
+
].map(([key, entry])=>[
|
|
481
|
+
key,
|
|
482
|
+
{
|
|
483
|
+
label: entry.resource.label,
|
|
484
|
+
rules: rulesOf(entry)
|
|
485
|
+
}
|
|
486
|
+
]))
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
/** One planned item as its plugin writes it: references rewritten, a kept-both copy's new name and slug. */ function writeFor(entry, item, manifestItem, content, idMap) {
|
|
490
|
+
return _extends({
|
|
491
|
+
row: item.row,
|
|
492
|
+
item: manifestItem,
|
|
493
|
+
decision: item.decision,
|
|
494
|
+
targetId: item.targetId
|
|
495
|
+
}, item.rename ? {
|
|
496
|
+
rename: item.rename
|
|
497
|
+
} : {}, {
|
|
498
|
+
content: entry.hooks.remapIds(content, idMap)
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
const WRITE_VERDICTS = new Set([
|
|
502
|
+
'create',
|
|
503
|
+
'replace',
|
|
504
|
+
'keepBoth'
|
|
505
|
+
]);
|
|
506
|
+
/** What an item the plan does not write comes to. */ function unwrittenResult(item) {
|
|
507
|
+
if (item.verdict === 'fail') {
|
|
508
|
+
return _extends({
|
|
509
|
+
row: item.row,
|
|
510
|
+
outcome: 'failed',
|
|
511
|
+
reason: 'problems'
|
|
512
|
+
}, item.problems[0] ? {
|
|
513
|
+
message: item.problems.join(' ')
|
|
514
|
+
} : {});
|
|
515
|
+
}
|
|
516
|
+
return _extends({
|
|
517
|
+
row: item.row,
|
|
518
|
+
outcome: 'skipped'
|
|
519
|
+
}, item.reason ? {
|
|
520
|
+
reason: item.reason
|
|
521
|
+
} : {});
|
|
522
|
+
}
|
|
523
|
+
/**
|
|
524
|
+
* Writes the planned items (see the block header): the undo snapshot
|
|
525
|
+
* first, then each owning plugin's items in dependency order through the
|
|
526
|
+
* ledger, then the written hashes and the results. Called again until
|
|
527
|
+
* `done`; a plugin that throws fails the job, and calling again resumes it.
|
|
528
|
+
*/ export async function applyTransferPackage(deps, input) {
|
|
529
|
+
var _ref, _input_acknowledged;
|
|
530
|
+
let job = await readJob(deps, input.orgId, input.jobId);
|
|
531
|
+
if (job.kind !== 'package' || !job.package) throw new TransferEngineError('invalid', 400, 'This import is not a package.');
|
|
532
|
+
if (job.status === 'applied') return {
|
|
533
|
+
job,
|
|
534
|
+
done: true,
|
|
535
|
+
started: false,
|
|
536
|
+
resumed: false,
|
|
537
|
+
results: []
|
|
538
|
+
};
|
|
539
|
+
if (job.status !== 'planned' && job.status !== 'applying' && job.status !== 'failed') {
|
|
540
|
+
throw new TransferEngineError('state', 409, `This import is ${job.status}; plan it before applying.`);
|
|
541
|
+
}
|
|
542
|
+
const acknowledged = [
|
|
543
|
+
...(_ref = (_input_acknowledged = input.acknowledged) != null ? _input_acknowledged : job.package.acknowledged) != null ? _ref : []
|
|
544
|
+
];
|
|
545
|
+
if (job.applyStartedAt === undefined) {
|
|
546
|
+
if (job.package.blocking.length) {
|
|
547
|
+
throw new TransferEngineError('choicesNeeded', 422, 'Answer every choice the dry run asks before importing.', {
|
|
548
|
+
blocking: job.package.blocking
|
|
549
|
+
});
|
|
550
|
+
}
|
|
551
|
+
if (!canApplyTransferPackage(job.package, acknowledged)) {
|
|
552
|
+
const missing = missingPackageAcknowledgements(job.package, acknowledged);
|
|
553
|
+
if (!missing.length) throw new TransferEngineError('state', 409, 'The dry run writes nothing.');
|
|
554
|
+
throw new TransferEngineError('acknowledgementsMissing', 422, 'Acknowledge every warning before importing.', {
|
|
555
|
+
missing
|
|
556
|
+
});
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
let started = false;
|
|
560
|
+
let resumed = false;
|
|
561
|
+
job = await takeLease(deps, job, input.driver, input.deadlineMs + TRANSFER_LEASE_GRACE_MS, (current, now)=>{
|
|
562
|
+
var _current_cursor, _current_applyStartedAt, _current_results;
|
|
563
|
+
if (current.status === 'applying') return current;
|
|
564
|
+
started = true;
|
|
565
|
+
resumed = current.status === 'failed';
|
|
566
|
+
return moveJob(current, 'applying', now, _extends({}, current.applyStartedAt === undefined && current.package ? {
|
|
567
|
+
package: _extends({}, current.package, {
|
|
568
|
+
acknowledged: acknowledged
|
|
569
|
+
})
|
|
570
|
+
} : {}, {
|
|
571
|
+
cursor: (_current_cursor = current.cursor) != null ? _current_cursor : {
|
|
572
|
+
chunk: 0,
|
|
573
|
+
rowsDone: 0
|
|
574
|
+
},
|
|
575
|
+
applyStartedAt: (_current_applyStartedAt = current.applyStartedAt) != null ? _current_applyStartedAt : now,
|
|
576
|
+
results: (_current_results = current.results) != null ? _current_results : emptyResults()
|
|
577
|
+
}));
|
|
578
|
+
});
|
|
579
|
+
const jobRef = transferJobsCollection(deps.firestore, job.orgId).doc(job.id);
|
|
580
|
+
const timeLeft = ()=>input.deadlineMs - clock(deps);
|
|
581
|
+
try {
|
|
582
|
+
var _ref1;
|
|
583
|
+
var _job_package;
|
|
584
|
+
const plan = await readStoredPlan(deps, job);
|
|
585
|
+
const pkg = await loadPackage(deps, job);
|
|
586
|
+
const resources = await packageResources(deps, (_ref1 = (_job_package = job.package) == null ? void 0 : _job_package.resources) != null ? _ref1 : []);
|
|
587
|
+
const byKey = new Map(plan.items.map((item)=>[
|
|
588
|
+
item.key,
|
|
589
|
+
item
|
|
590
|
+
]));
|
|
591
|
+
const writes = plan.order.map((key)=>byKey.get(key)).filter((item)=>Boolean(item && WRITE_VERDICTS.has(item.verdict)));
|
|
592
|
+
for (const item of writes){
|
|
593
|
+
if (!resources.has(item.kind)) throw new Error(`nothing here writes ${item.kind} any more`);
|
|
594
|
+
}
|
|
595
|
+
// The snapshot first: nothing is written that undo could not put back.
|
|
596
|
+
const undoRef = jobRef.collection('undo').doc('0');
|
|
597
|
+
let entries = await readJsonDoc(undoRef);
|
|
598
|
+
if (!entries) {
|
|
599
|
+
entries = [];
|
|
600
|
+
const replaced = new Map();
|
|
601
|
+
for (const item of writes){
|
|
602
|
+
var _replaced_get;
|
|
603
|
+
if (item.verdict === 'replace') replaced.set(item.kind, [
|
|
604
|
+
...(_replaced_get = replaced.get(item.kind)) != null ? _replaced_get : [],
|
|
605
|
+
item.targetId
|
|
606
|
+
]);
|
|
607
|
+
}
|
|
608
|
+
const previous = new Map();
|
|
609
|
+
for (const [kind, ids] of replaced){
|
|
610
|
+
const entry = resources.get(kind);
|
|
611
|
+
for (const one of (await entry.hooks.readItems(contextOf(job.orgId, kind, input.actorUid, job.id), ids))){
|
|
612
|
+
previous.set(packageItemKey({
|
|
613
|
+
kind,
|
|
614
|
+
id: one.id
|
|
615
|
+
}), one.content);
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
for (const item of writes){
|
|
619
|
+
var _ref2, _ref3, _ref4, _ref5;
|
|
620
|
+
var _item_rename, _item_existing, _item_rename1, _item_existing1;
|
|
621
|
+
const targetKey = packageItemKey({
|
|
622
|
+
kind: item.kind,
|
|
623
|
+
id: item.targetId
|
|
624
|
+
});
|
|
625
|
+
const before = previous.get(targetKey);
|
|
626
|
+
entries.push(_extends({
|
|
627
|
+
row: item.row,
|
|
628
|
+
key: item.key,
|
|
629
|
+
resource: item.kind,
|
|
630
|
+
id: item.targetId
|
|
631
|
+
}, ((_ref2 = (_ref3 = (_item_rename = item.rename) == null ? void 0 : _item_rename.name) != null ? _ref3 : (_item_existing = item.existing) == null ? void 0 : _item_existing.name) != null ? _ref2 : item.name) ? {
|
|
632
|
+
name: (_ref4 = (_ref5 = (_item_rename1 = item.rename) == null ? void 0 : _item_rename1.name) != null ? _ref5 : (_item_existing1 = item.existing) == null ? void 0 : _item_existing1.name) != null ? _ref4 : item.name
|
|
633
|
+
} : {}, {
|
|
634
|
+
action: item.verdict === 'replace' ? 'updated' : 'created'
|
|
635
|
+
}, before !== undefined ? {
|
|
636
|
+
previous: before,
|
|
637
|
+
previousHash: await contentHash(before)
|
|
638
|
+
} : {}));
|
|
639
|
+
}
|
|
640
|
+
await writeJsonDoc(undoRef, {
|
|
641
|
+
jobId: job.id,
|
|
642
|
+
chunk: 0,
|
|
643
|
+
package: true
|
|
644
|
+
}, entries);
|
|
645
|
+
}
|
|
646
|
+
const ledgerCollection = jobRef.collection('ledger');
|
|
647
|
+
const ledger = new Map();
|
|
648
|
+
for (const doc of (await ledgerCollection.where('chunk', '==', 0).get()).docs){
|
|
649
|
+
const data = doc.data();
|
|
650
|
+
ledger.set(data.row, data.result);
|
|
651
|
+
}
|
|
652
|
+
const writer = {
|
|
653
|
+
async alreadyApplied (row) {
|
|
654
|
+
var _ledger_get;
|
|
655
|
+
return (_ledger_get = ledger.get(row)) != null ? _ledger_get : null;
|
|
656
|
+
},
|
|
657
|
+
async markApplied (result) {
|
|
658
|
+
if (ledger.has(result.row)) return;
|
|
659
|
+
try {
|
|
660
|
+
await ledgerCollection.doc(transferLedgerKey(job.id, result.row)).create(stored({
|
|
661
|
+
row: result.row,
|
|
662
|
+
chunk: 0,
|
|
663
|
+
result,
|
|
664
|
+
undoJson: null,
|
|
665
|
+
at: clock(deps)
|
|
666
|
+
}));
|
|
667
|
+
} catch (error) {
|
|
668
|
+
if (!isAlreadyExists(error)) throw error;
|
|
669
|
+
}
|
|
670
|
+
ledger.set(result.row, result);
|
|
671
|
+
},
|
|
672
|
+
timeLeftMs: timeLeft
|
|
673
|
+
};
|
|
674
|
+
const idMap = packageIdMap(plan.idMap);
|
|
675
|
+
// Each owner's run of items in dependency order, handed over together.
|
|
676
|
+
const runs = [];
|
|
677
|
+
for (const item of writes){
|
|
678
|
+
var _last_;
|
|
679
|
+
const last = runs[runs.length - 1];
|
|
680
|
+
if (last && ((_last_ = last[0]) == null ? void 0 : _last_.kind) === item.kind) last.push(item);
|
|
681
|
+
else runs.push([
|
|
682
|
+
item
|
|
683
|
+
]);
|
|
684
|
+
}
|
|
685
|
+
for (const run of runs){
|
|
686
|
+
var _run_;
|
|
687
|
+
const pending = run.filter((item)=>!ledger.has(item.row));
|
|
688
|
+
if (!pending.length) continue;
|
|
689
|
+
if (timeLeft() <= TRANSFER_MIN_CHUNK_BUDGET_MS) break;
|
|
690
|
+
const kind = (_run_ = run[0]) == null ? void 0 : _run_.kind;
|
|
691
|
+
const entry = resources.get(kind);
|
|
692
|
+
const items = pending.map((item)=>writeFor(entry, item, pkg.manifest.items[item.row], pkg.items[item.key], idMap));
|
|
693
|
+
const applied = await entry.hooks.writeItems(contextOf(job.orgId, kind, input.actorUid, job.id), items, writer);
|
|
694
|
+
const planned = new Set(pending.map((item)=>item.row));
|
|
695
|
+
for (const result of applied.results){
|
|
696
|
+
if (planned.has(result.row) && !ledger.has(result.row)) await writer.markApplied(result);
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
if (writes.some((item)=>!ledger.has(item.row))) {
|
|
700
|
+
job = _extends({}, job, {
|
|
701
|
+
lease: null,
|
|
702
|
+
updatedAt: clock(deps)
|
|
703
|
+
});
|
|
704
|
+
await saveJob(deps, job);
|
|
705
|
+
return {
|
|
706
|
+
job,
|
|
707
|
+
done: false,
|
|
708
|
+
started,
|
|
709
|
+
resumed,
|
|
710
|
+
results: []
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
// Every write landed: what each item holds now is what undo compares a later edit against.
|
|
714
|
+
const hashes = new Map();
|
|
715
|
+
for (const kind of new Set(writes.map((item)=>item.kind))){
|
|
716
|
+
const entry = resources.get(kind);
|
|
717
|
+
for (const one of (await entry.hooks.items(contextOf(job.orgId, kind, input.actorUid, job.id)))){
|
|
718
|
+
hashes.set(packageItemKey({
|
|
719
|
+
kind,
|
|
720
|
+
id: one.id
|
|
721
|
+
}), one.contentHash);
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
const settled = entries.map((entry)=>{
|
|
725
|
+
const written = ledger.get(entry.row);
|
|
726
|
+
const hash = hashes.get(packageItemKey({
|
|
727
|
+
kind: entry.resource,
|
|
728
|
+
id: entry.id
|
|
729
|
+
}));
|
|
730
|
+
return written && written.outcome !== 'failed' && hash ? _extends({}, entry, {
|
|
731
|
+
writtenHash: hash
|
|
732
|
+
}) : entry;
|
|
733
|
+
}).filter((entry)=>{
|
|
734
|
+
var _ledger_get;
|
|
735
|
+
return ((_ledger_get = ledger.get(entry.row)) == null ? void 0 : _ledger_get.outcome) !== 'failed';
|
|
736
|
+
});
|
|
737
|
+
await writeJsonDoc(undoRef, {
|
|
738
|
+
jobId: job.id,
|
|
739
|
+
chunk: 0,
|
|
740
|
+
package: true
|
|
741
|
+
}, settled);
|
|
742
|
+
const results = plan.items.map((item)=>WRITE_VERDICTS.has(item.verdict) ? ledger.get(item.row) : unwrittenResult(item));
|
|
743
|
+
const now = clock(deps);
|
|
744
|
+
// The results before the job: a crash between the two resumes into the same results from the ledger.
|
|
745
|
+
await jobRef.collection('results').doc('0').set(stored({
|
|
746
|
+
jobId: job.id,
|
|
747
|
+
index: 0,
|
|
748
|
+
results,
|
|
749
|
+
completedAt: now
|
|
750
|
+
}));
|
|
751
|
+
job = moveJob(job, 'applied', now, {
|
|
752
|
+
lease: null,
|
|
753
|
+
cursor: {
|
|
754
|
+
chunk: 1,
|
|
755
|
+
rowsDone: plan.items.length
|
|
756
|
+
},
|
|
757
|
+
results: summarizeTransferResults(results)
|
|
758
|
+
});
|
|
759
|
+
await saveJob(deps, job);
|
|
760
|
+
const ledgerDocs = (await ledgerCollection.where('chunk', '==', 0).get()).docs;
|
|
761
|
+
await eachLimited(ledgerDocs, TRANSFER_WRITE_CONCURRENCY, async (doc)=>{
|
|
762
|
+
await doc.ref.delete();
|
|
763
|
+
});
|
|
764
|
+
return {
|
|
765
|
+
job,
|
|
766
|
+
done: true,
|
|
767
|
+
started,
|
|
768
|
+
resumed,
|
|
769
|
+
results
|
|
770
|
+
};
|
|
771
|
+
} catch (error) {
|
|
772
|
+
if (error instanceof TransferEngineError) throw error;
|
|
773
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
774
|
+
job = moveJob(job, 'failed', clock(deps), {
|
|
775
|
+
error: {
|
|
776
|
+
code: 'applyFailed',
|
|
777
|
+
message: `The package could not be written: ${message}`,
|
|
778
|
+
chunk: 0
|
|
779
|
+
},
|
|
780
|
+
lease: null
|
|
781
|
+
});
|
|
782
|
+
await saveJob(deps, job);
|
|
783
|
+
return {
|
|
784
|
+
job,
|
|
785
|
+
done: false,
|
|
786
|
+
started,
|
|
787
|
+
resumed,
|
|
788
|
+
results: []
|
|
789
|
+
};
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
/*==========================================
|
|
793
|
+
* RESULTS
|
|
794
|
+
*=========================================*/ /** The result file of a package import: each item, what was decided, and what happened. */ export async function transferPackageResultFile(deps, job) {
|
|
795
|
+
var _job_fileName;
|
|
796
|
+
if (job.trimmedAt !== undefined) {
|
|
797
|
+
throw new TransferEngineError('state', 409, 'This import was cleared once its undo window closed; its results are still counted.');
|
|
798
|
+
}
|
|
799
|
+
const plan = await readStoredPlan(deps, job);
|
|
800
|
+
const jobRef = transferJobsCollection(deps.firestore, job.orgId).doc(job.id);
|
|
801
|
+
const results = new Map();
|
|
802
|
+
for (const doc of (await jobRef.collection('results').get()).docs){
|
|
803
|
+
var _doc_data_results;
|
|
804
|
+
for (const result of (_doc_data_results = doc.data().results) != null ? _doc_data_results : [])results.set(result.row, result);
|
|
805
|
+
}
|
|
806
|
+
const cell = (value)=>escapeCsvCell(value);
|
|
807
|
+
const lines = [
|
|
808
|
+
[
|
|
809
|
+
'Kind',
|
|
810
|
+
'Name',
|
|
811
|
+
'ID',
|
|
812
|
+
'Decision',
|
|
813
|
+
...TRANSFER_RESULT_COLUMNS
|
|
814
|
+
].map(cell).join(',')
|
|
815
|
+
];
|
|
816
|
+
for (const item of plan.items){
|
|
817
|
+
var _item_name, _ref, _result_message, _ref1;
|
|
818
|
+
const result = results.get(item.row);
|
|
819
|
+
lines.push([
|
|
820
|
+
item.kind,
|
|
821
|
+
(_item_name = item.name) != null ? _item_name : '',
|
|
822
|
+
item.id,
|
|
823
|
+
item.verdict,
|
|
824
|
+
(_ref = result == null ? void 0 : result.outcome) != null ? _ref : 'pending',
|
|
825
|
+
result ? (_result_message = result.message) != null ? _result_message : result.reason ? String(result.reason) : '' : '',
|
|
826
|
+
(_ref1 = result == null ? void 0 : result.recordId) != null ? _ref1 : ''
|
|
827
|
+
].map(cell).join(','));
|
|
828
|
+
}
|
|
829
|
+
const base = ((_job_fileName = job.fileName) != null ? _job_fileName : 'package').replace(/\.[a-z0-9]+$/i, '');
|
|
830
|
+
return {
|
|
831
|
+
csv: `${lines.join('\r\n')}\r\n`,
|
|
832
|
+
rows: plan.items.length,
|
|
833
|
+
fileName: `${base}-results.csv`
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
/**
|
|
837
|
+
* What undo does to one item, against what it holds now: an item the
|
|
838
|
+
* import created and nobody touched is deleted; one it replaced and nobody
|
|
839
|
+
* touched is restored; one edited since is a conflict; one already back
|
|
840
|
+
* where it was, or gone, needs nothing.
|
|
841
|
+
*/ function packageUndoStep(entry, currentHash) {
|
|
842
|
+
if (currentHash === undefined) return {
|
|
843
|
+
action: 'nothing',
|
|
844
|
+
entry
|
|
845
|
+
};
|
|
846
|
+
if (entry.action === 'updated' && entry.previousHash && currentHash === entry.previousHash) return {
|
|
847
|
+
action: 'nothing',
|
|
848
|
+
entry
|
|
849
|
+
};
|
|
850
|
+
if (entry.writtenHash && currentHash === entry.writtenHash) {
|
|
851
|
+
return {
|
|
852
|
+
action: entry.action === 'created' ? 'delete' : 'restore',
|
|
853
|
+
entry
|
|
854
|
+
};
|
|
855
|
+
}
|
|
856
|
+
return {
|
|
857
|
+
action: 'conflict',
|
|
858
|
+
entry
|
|
859
|
+
};
|
|
860
|
+
}
|
|
861
|
+
async function undoSteps(deps, job, actorUid) {
|
|
862
|
+
var _ref;
|
|
863
|
+
const entries = (_ref = await readJsonDoc(transferJobsCollection(deps.firestore, job.orgId).doc(job.id).collection('undo').doc('0'))) != null ? _ref : [];
|
|
864
|
+
const resources = await packageResources(deps, [
|
|
865
|
+
...new Set(entries.map((entry)=>entry.resource))
|
|
866
|
+
]);
|
|
867
|
+
const hashes = new Map();
|
|
868
|
+
for (const [kind, entry] of resources){
|
|
869
|
+
for (const one of (await entry.hooks.items(contextOf(job.orgId, kind, actorUid, job.id)))){
|
|
870
|
+
hashes.set(packageItemKey({
|
|
871
|
+
kind,
|
|
872
|
+
id: one.id
|
|
873
|
+
}), one.contentHash);
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
const steps = entries.map((entry)=>packageUndoStep(entry, hashes.get(packageItemKey({
|
|
877
|
+
kind: entry.resource,
|
|
878
|
+
id: entry.id
|
|
879
|
+
}))));
|
|
880
|
+
return {
|
|
881
|
+
steps,
|
|
882
|
+
resources
|
|
883
|
+
};
|
|
884
|
+
}
|
|
885
|
+
function assertPackageUndoOpen(job, now) {
|
|
886
|
+
var _job_undo;
|
|
887
|
+
if (job.kind !== 'package') throw new TransferEngineError('invalid', 400, 'This import is not a package.');
|
|
888
|
+
if (job.status !== 'applied') throw new TransferEngineError('state', 409, `This import is ${job.status}; only an applied import can be undone.`);
|
|
889
|
+
if (((_job_undo = job.undo) == null ? void 0 : _job_undo.status) === 'done') throw new TransferEngineError('state', 409, 'This import was already undone.');
|
|
890
|
+
if (!job.undo && !transferUndoAvailable(job, now)) {
|
|
891
|
+
throw new TransferEngineError('undoExpired', 410, 'An import can be undone for seven days; this one is older.');
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
/** What undo would do, writing nothing: counts, and every item edited since the import. */ export async function planTransferPackageUndo(deps, input) {
|
|
895
|
+
const job = await readJob(deps, input.orgId, input.jobId);
|
|
896
|
+
assertPackageUndoOpen(job, clock(deps));
|
|
897
|
+
const { steps } = await undoSteps(deps, job, input.actorUid);
|
|
898
|
+
const counts = emptyUndoCounts();
|
|
899
|
+
const conflicts = [];
|
|
900
|
+
for (const step of steps){
|
|
901
|
+
counts[step.action] += 1;
|
|
902
|
+
if (step.action === 'conflict') {
|
|
903
|
+
const { entry } = step;
|
|
904
|
+
conflicts.push(_extends({
|
|
905
|
+
row: entry.row,
|
|
906
|
+
key: entry.key,
|
|
907
|
+
resource: entry.resource,
|
|
908
|
+
id: entry.id
|
|
909
|
+
}, entry.name ? {
|
|
910
|
+
name: entry.name
|
|
911
|
+
} : {}, {
|
|
912
|
+
action: entry.action
|
|
913
|
+
}));
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
return {
|
|
917
|
+
job,
|
|
918
|
+
counts,
|
|
919
|
+
conflicts,
|
|
920
|
+
expiresAt: transferUndoExpiresAt(job)
|
|
921
|
+
};
|
|
922
|
+
}
|
|
923
|
+
/**
|
|
924
|
+
* Carries undo out through each plugin's `revertItems`: every untouched
|
|
925
|
+
* item, and every edited one the person (or `otherwise`) said to revert.
|
|
926
|
+
* A step a plugin's own rules refuse is counted as a conflict and left.
|
|
927
|
+
*/ export async function applyTransferPackageUndo(deps, input) {
|
|
928
|
+
var _input_decisions, _undo_decisions, _input_decisions1;
|
|
929
|
+
if (input.otherwise !== 'keep' && input.otherwise !== 'revert') {
|
|
930
|
+
throw new TransferEngineError('invalid', 400, 'Say what happens to an item edited since the import.');
|
|
931
|
+
}
|
|
932
|
+
let job = await readJob(deps, input.orgId, input.jobId);
|
|
933
|
+
assertPackageUndoOpen(job, clock(deps));
|
|
934
|
+
const decisions = _extends({}, (_input_decisions = input.decisions) != null ? _input_decisions : {});
|
|
935
|
+
let started = false;
|
|
936
|
+
job = await takeLease(deps, job, input.driver, input.deadlineMs + TRANSFER_LEASE_GRACE_MS, (current, now)=>{
|
|
937
|
+
if (current.undo) {
|
|
938
|
+
return _extends({}, current, {
|
|
939
|
+
undo: _extends({}, current.undo, {
|
|
940
|
+
otherwise: input.otherwise,
|
|
941
|
+
decisions: _extends({}, current.undo.decisions, decisions)
|
|
942
|
+
}),
|
|
943
|
+
updatedAt: now
|
|
944
|
+
});
|
|
945
|
+
}
|
|
946
|
+
started = true;
|
|
947
|
+
return _extends({}, current, {
|
|
948
|
+
updatedAt: now,
|
|
949
|
+
undo: {
|
|
950
|
+
status: 'running',
|
|
951
|
+
chunk: 0,
|
|
952
|
+
startedAt: now,
|
|
953
|
+
startedBy: input.actorUid,
|
|
954
|
+
otherwise: input.otherwise,
|
|
955
|
+
decisions,
|
|
956
|
+
counts: emptyUndoCounts()
|
|
957
|
+
}
|
|
958
|
+
});
|
|
959
|
+
});
|
|
960
|
+
let undo = job.undo;
|
|
961
|
+
Object.assign(decisions, (_undo_decisions = undo.decisions) != null ? _undo_decisions : {}, (_input_decisions1 = input.decisions) != null ? _input_decisions1 : {});
|
|
962
|
+
try {
|
|
963
|
+
const { steps, resources } = await undoSteps(deps, job, input.actorUid);
|
|
964
|
+
const counts = emptyUndoCounts();
|
|
965
|
+
const byResource = new Map();
|
|
966
|
+
for (const step of steps){
|
|
967
|
+
var _decisions_entry_id, _byResource_get;
|
|
968
|
+
const { entry } = step;
|
|
969
|
+
const decided = (_decisions_entry_id = decisions[entry.id]) != null ? _decisions_entry_id : input.otherwise;
|
|
970
|
+
if (step.action === 'nothing') {
|
|
971
|
+
counts.nothing += 1;
|
|
972
|
+
continue;
|
|
973
|
+
}
|
|
974
|
+
if (step.action === 'conflict' && decided !== 'revert') {
|
|
975
|
+
counts.conflict += 1;
|
|
976
|
+
continue;
|
|
977
|
+
}
|
|
978
|
+
const revert = entry.action === 'created' ? {
|
|
979
|
+
action: 'delete',
|
|
980
|
+
id: entry.id
|
|
981
|
+
} : {
|
|
982
|
+
action: 'restore',
|
|
983
|
+
id: entry.id,
|
|
984
|
+
content: entry.previous
|
|
985
|
+
};
|
|
986
|
+
byResource.set(entry.resource, [
|
|
987
|
+
...(_byResource_get = byResource.get(entry.resource)) != null ? _byResource_get : [],
|
|
988
|
+
revert
|
|
989
|
+
]);
|
|
990
|
+
}
|
|
991
|
+
for (const [kind, list] of byResource){
|
|
992
|
+
const entry = resources.get(kind);
|
|
993
|
+
if (!entry) {
|
|
994
|
+
counts.conflict += list.length;
|
|
995
|
+
continue;
|
|
996
|
+
}
|
|
997
|
+
const reverted = await entry.hooks.revertItems(contextOf(job.orgId, kind, input.actorUid, job.id), list);
|
|
998
|
+
for (const step of list){
|
|
999
|
+
if (reverted.done.includes(step.id)) counts[step.action] += 1;
|
|
1000
|
+
}
|
|
1001
|
+
counts.conflict += reverted.refused.length;
|
|
1002
|
+
}
|
|
1003
|
+
undo = _extends({}, undo, {
|
|
1004
|
+
chunk: 1,
|
|
1005
|
+
counts,
|
|
1006
|
+
status: 'done'
|
|
1007
|
+
});
|
|
1008
|
+
job = moveJob(_extends({}, job, {
|
|
1009
|
+
undo
|
|
1010
|
+
}), 'undone', clock(deps), {
|
|
1011
|
+
lease: null
|
|
1012
|
+
});
|
|
1013
|
+
await saveJob(deps, job);
|
|
1014
|
+
return {
|
|
1015
|
+
job,
|
|
1016
|
+
undo,
|
|
1017
|
+
done: true,
|
|
1018
|
+
started
|
|
1019
|
+
};
|
|
1020
|
+
} catch (error) {
|
|
1021
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1022
|
+
job = _extends({}, job, {
|
|
1023
|
+
lease: null,
|
|
1024
|
+
updatedAt: clock(deps),
|
|
1025
|
+
error: {
|
|
1026
|
+
code: 'undoFailed',
|
|
1027
|
+
message,
|
|
1028
|
+
chunk: 0
|
|
1029
|
+
}
|
|
1030
|
+
});
|
|
1031
|
+
await saveJob(deps, job);
|
|
1032
|
+
throw new TransferEngineError('failed', 500, `Undo stopped: ${message}. Try again to continue.`);
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
//# sourceMappingURL=transfer-packages.js.map
|