domma-cms 1.38.0 → 1.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/CLAUDE.md +17 -0
- package/admin/js/api.js +1 -1
- package/admin/js/app.js +1 -1
- package/admin/js/lib/plugins-arrange.js +2 -2
- package/admin/js/lib/wordpress-import-arrange.js +1 -1
- package/admin/js/templates/docs/usage-tools.html +18 -3
- package/admin/js/templates/plugins.html +2 -0
- package/admin/js/templates/wordpress-import.html +23 -5
- package/admin/js/views/plugins.js +24 -12
- package/admin/js/views/wordpress-import.js +10 -10
- package/bin/update.js +2 -0
- package/docs/DommaFeatures.md +7 -0
- package/docs/configuration.md +1 -0
- package/docs/plugin-development.md +11 -0
- package/package.json +1 -1
- package/server/routes/api/plugin-marketplace.js +249 -19
- package/server/routes/api/wordpress-import.js +62 -3
- package/server/server.js +1 -1
- package/server/services/siteGitignore.js +2 -0
- package/server/services/storeClient.js +333 -0
- package/server/services/tools.js +1 -1
- package/server/services/wpImport/crawl.js +619 -0
- package/server/services/wpImport/index.js +53 -8
- package/server/services/wpImport/plan.js +5 -3
- package/server/services/wpImport/platforms.js +186 -0
- package/server/services/wpImport/wxr.js +6 -2
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Plugin Marketplace API
|
|
2
|
+
* Plugin Marketplace API.
|
|
3
3
|
*
|
|
4
|
-
* GET /api/plugins/marketplace
|
|
5
|
-
* POST /api/plugins/marketplace/install
|
|
6
|
-
*
|
|
4
|
+
* GET /api/plugins/marketplace - the catalogue as this site sees it
|
|
5
|
+
* POST /api/plugins/marketplace/install - fetch a bundle and install it
|
|
6
|
+
* POST /api/plugins/marketplace/buy - self-hosted: start buying (or renewing) a Tool
|
|
7
|
+
* POST /api/plugins/marketplace/claim - self-hosted: has it been paid? (installs it when it has)
|
|
8
|
+
* DELETE /api/plugins/marketplace/purchases/:id - self-hosted: stop waiting for a purchase
|
|
9
|
+
* DELETE /api/plugins/marketplace/:slug - uninstall (a paid plugin also tells the manager, for billing)
|
|
7
10
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
11
|
+
* Two delivery paths behind the same screen:
|
|
12
|
+
* managed a site the manager forked buys from the fleet's own catalogue
|
|
13
|
+
* over loopback (managerClient.js) - rented, billed monthly.
|
|
14
|
+
* self-hosted any other site buys from the public Domma store (storeClient.js)
|
|
15
|
+
* - bought outright, with a term of updates. The claim secret and
|
|
16
|
+
* the licence keys stay on this server; the browser only ever sees
|
|
17
|
+
* the payment page's address.
|
|
18
|
+
* An owner instance (the manager itself) has neither, and the screen says so.
|
|
19
|
+
* Install from file (`POST /plugins/install-upload`) works everywhere.
|
|
15
20
|
*
|
|
16
21
|
* All routes require authentication + admin role.
|
|
17
22
|
*/
|
|
@@ -19,13 +24,74 @@ import {authenticate, requireAdmin} from '../../middleware/auth.js';
|
|
|
19
24
|
import {fetchCatalogue, fetchPluginBundle, isManaged, releasePlugin, siteSlug} from '../../services/managerClient.js';
|
|
20
25
|
import {cmsVersionShortfall, compareVersions, installedCmsVersion, installPlugin, uninstallPlugin} from '../../services/pluginInstaller.js';
|
|
21
26
|
import {discoverPlugins} from '../../services/plugins.js';
|
|
27
|
+
import {readEnvelope, readManifestFromTarball} from '../../services/pluginEnvelope.js';
|
|
28
|
+
import {originOf, trustedOrigin} from '../../services/passwordReset.js';
|
|
29
|
+
import {getUserById} from '../../services/users.js';
|
|
30
|
+
import * as storeClient from '../../services/storeClient.js';
|
|
31
|
+
import {classifyEntitlement} from '../../services/pluginEntitlement.js';
|
|
32
|
+
import fs from 'node:fs/promises';
|
|
33
|
+
import path from 'node:path';
|
|
34
|
+
|
|
35
|
+
const SLUG = /^[a-z0-9][a-z0-9-_]{0,63}$/;
|
|
36
|
+
/** Entitlement states a plugin runs under (pluginEntitlement.mayLoad). */
|
|
37
|
+
const LOADS = new Set(['valid', 'grace', 'support-ended']);
|
|
38
|
+
|
|
39
|
+
/** Messages the installer raises deliberately - things an admin can act on. Anything else stays vague (it can carry a path). */
|
|
40
|
+
function installError(err) {
|
|
41
|
+
const safe = err.expose === true || [
|
|
42
|
+
'Plugin already installed', 'Plugin signature invalid', 'Unknown public key',
|
|
43
|
+
'Invalid plugin slug', 'Refusing to downgrade', 'Refusing to upgrade'
|
|
44
|
+
].some(m => err.message.startsWith(m));
|
|
45
|
+
return safe ? err.message : 'Installation failed.';
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Download a plugin from the store and install it. The file is the store's
|
|
50
|
+
* signed bundle; an unsigned one is refused here whatever it says, since nothing
|
|
51
|
+
* the store serves is unsigned.
|
|
52
|
+
*/
|
|
53
|
+
async function installFromStore(slug, {version = null, update = false} = {}) {
|
|
54
|
+
const parts = readEnvelope(await storeClient.downloadPlugin(slug));
|
|
55
|
+
if (!parts.signature) throw Object.assign(new Error('The store sent an unsigned file - nothing was installed.'), {expose: true});
|
|
56
|
+
const manifest = readManifestFromTarball(parts.tarball);
|
|
57
|
+
if (manifest?.name !== slug) throw Object.assign(new Error('The store sent a different plugin - nothing was installed.'), {expose: true});
|
|
58
|
+
if (version && manifest.version !== version) {
|
|
59
|
+
throw Object.assign(new Error(`The store now has ${slug} ${manifest.version} - reload the Marketplace and try again.`), {expose: true});
|
|
60
|
+
}
|
|
61
|
+
const result = await installPlugin(slug, {
|
|
62
|
+
tarball: parts.tarball, signature: parts.signature, publicKeyId: 'marketplace-1',
|
|
63
|
+
entitlement: parts.entitlement, allowUpgrade: update === true
|
|
64
|
+
});
|
|
65
|
+
return {action: result?.action ?? 'install', from: result?.from ?? null, version: result?.version ?? manifest.version};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* After a renewal: the store's fresh entitlement for the version installed, written beside it - only when it is
|
|
70
|
+
* that same version and the entitlement verifies. True when written.
|
|
71
|
+
*/
|
|
72
|
+
async function refreshEntitlement(slug, installedVersion) {
|
|
73
|
+
const parts = readEnvelope(await storeClient.downloadPlugin(slug));
|
|
74
|
+
const manifest = readManifestFromTarball(parts.tarball);
|
|
75
|
+
if (!parts.entitlement || manifest?.name !== slug || manifest.version !== installedVersion) return false;
|
|
76
|
+
if (classifyEntitlement(slug, parts.entitlement).state !== 'valid') return false;
|
|
77
|
+
await fs.writeFile(path.resolve('plugins', slug, '.entitlement.json'), JSON.stringify(parts.entitlement, null, 2) + '\n', 'utf8');
|
|
78
|
+
return true;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** This site's public address, as the store will know it. */
|
|
82
|
+
async function siteAddress(request) {
|
|
83
|
+
return (await trustedOrigin(originOf(request)).catch(() => '')) || originOf(request);
|
|
84
|
+
}
|
|
22
85
|
|
|
23
86
|
export async function pluginMarketplaceRoutes(fastify) {
|
|
24
87
|
const adminOnly = { preHandler: [authenticate, requireAdmin] };
|
|
25
88
|
|
|
26
89
|
// GET /plugins/marketplace
|
|
27
|
-
fastify.get('/plugins/marketplace', adminOnly, async (
|
|
28
|
-
if (!isManaged())
|
|
90
|
+
fastify.get('/plugins/marketplace', adminOnly, async (request, reply) => {
|
|
91
|
+
if (!isManaged()) {
|
|
92
|
+
if (!storeClient.storeOpen()) return reply.send({ available: false, mode: storeClient.isOwnerInstance() ? 'owner' : 'none', catalogue: [] });
|
|
93
|
+
return reply.send(await storeCatalogue(request));
|
|
94
|
+
}
|
|
29
95
|
|
|
30
96
|
const catalogue = await fetchCatalogue();
|
|
31
97
|
|
|
@@ -66,7 +132,14 @@ export async function pluginMarketplaceRoutes(fastify) {
|
|
|
66
132
|
return reply.code(400).send({ error: 'Invalid version format. Expected semver (e.g. 1.0.0).' });
|
|
67
133
|
}
|
|
68
134
|
if (!isManaged()) {
|
|
69
|
-
return reply.code(503).send({ error: 'This site
|
|
135
|
+
if (!storeClient.storeOpen()) return reply.code(503).send({ error: 'This site has no store - install from a file instead.' });
|
|
136
|
+
try {
|
|
137
|
+
const r = await installFromStore(slug, { version, update: update === true });
|
|
138
|
+
return reply.send({ success: true, name: slug, ...r, restartRequired: true });
|
|
139
|
+
} catch (err) {
|
|
140
|
+
fastify.log.warn({ err }, '[marketplace] store install failed');
|
|
141
|
+
return reply.code(err instanceof storeClient.StoreError ? err.status : 400).send({ error: installError(err), code: err.code || null });
|
|
142
|
+
}
|
|
70
143
|
}
|
|
71
144
|
|
|
72
145
|
let bundle;
|
|
@@ -110,14 +183,171 @@ export async function pluginMarketplaceRoutes(fastify) {
|
|
|
110
183
|
// `expose` marks a refusal written for the admin (a plugin that needs
|
|
111
184
|
// a newer CMS, naming both versions) - it used to arrive here as a
|
|
112
185
|
// bare "Installation failed." while Install from file showed it.
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
186
|
+
return reply.code(400).send({ error: installError(err) });
|
|
187
|
+
}
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
// ── The self-hosted store ─────────────────────────────────────────────
|
|
191
|
+
|
|
192
|
+
/** The store's catalogue in the shape the screen already reads, plus where purchases stand. */
|
|
193
|
+
async function storeCatalogue(request) {
|
|
194
|
+
const plugins = await discoverPlugins();
|
|
195
|
+
const installed = new Map(plugins.map(m => [m.name, m.version]));
|
|
196
|
+
const cmsVersion = installedCmsVersion();
|
|
197
|
+
const site = await siteAddress(request);
|
|
198
|
+
// Connected to Domma: pick up what was bought for this site since last time (a failure just waits for next time).
|
|
199
|
+
await storeClient.syncConnection().catch(() => {});
|
|
200
|
+
let answer;
|
|
201
|
+
try {
|
|
202
|
+
answer = await storeClient.fetchStoreCatalogue(site, storeClient.heldKeys([...installed.keys()]));
|
|
203
|
+
} catch (err) {
|
|
204
|
+
return { available: true, mode: 'store', failed: err.message, site: null, catalogue: [], pending: [], connection: storeClient.connectionView() };
|
|
205
|
+
}
|
|
206
|
+
const pending = storeClient.pendingPurchases().map(p => ({ id: p.id, slug: p.slug, kind: p.kind, startedAt: p.startedAt }));
|
|
207
|
+
const catalogue = (Array.isArray(answer.plugins) ? answer.plugins : []).filter(e => SLUG.test(e?.slug || '')).map(e => {
|
|
208
|
+
const have = installed.get(e.slug);
|
|
209
|
+
const owned = e.tier !== 'paid' || e.owned === true;
|
|
210
|
+
return {
|
|
211
|
+
...e,
|
|
212
|
+
store: true,
|
|
213
|
+
entitled: owned && e.updates !== false,
|
|
214
|
+
// The words the screen already has: licensed, support ended, or not bought.
|
|
215
|
+
licenceStatus: e.tier !== 'paid' ? null : e.owned ? (e.updates ? 'active' : 'support-ended') : 'none',
|
|
216
|
+
installed: have !== undefined,
|
|
217
|
+
installedVersion: have ?? null,
|
|
218
|
+
// Already licensed here by other means (an owner licence, a file bought by email): nothing to buy.
|
|
219
|
+
licensedHere: have !== undefined && e.tier === 'paid' && LOADS.has(classifyEntitlement(e.slug).state),
|
|
220
|
+
updatable: have !== undefined && compareVersions(e.version, have) > 0,
|
|
221
|
+
needsCms: cmsVersionShortfall({ name: e.slug, displayName: e.displayName, minCmsVersion: e.minCmsVersion }, cmsVersion),
|
|
222
|
+
pending: pending.find(p => p.slug === e.slug)?.id ?? null
|
|
223
|
+
};
|
|
224
|
+
});
|
|
225
|
+
return { available: true, mode: 'store', site: answer.site || null, siteError: answer.siteError || null, cmsVersion, catalogue, pending, connection: storeClient.connectionView() };
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// POST /plugins/marketplace/buy {slug} -> {id, checkoutUrl}
|
|
229
|
+
fastify.post('/plugins/marketplace/buy', adminOnly, async (request, reply) => {
|
|
230
|
+
const slug = request.body?.slug;
|
|
231
|
+
if (!SLUG.test(String(slug || ''))) return reply.code(400).send({ error: 'Invalid plugin slug.' });
|
|
232
|
+
if (isManaged() || !storeClient.storeOpen()) return reply.code(409).send({ error: 'This site does not buy from the Domma store.' });
|
|
233
|
+
const user = await getUserById(request.user?.id);
|
|
234
|
+
if (!user?.email) return reply.code(400).send({ error: 'Your account has no email address for the receipt and licence key - add one under My profile.' });
|
|
235
|
+
try {
|
|
236
|
+
const out = await storeClient.startPurchase({ site: await siteAddress(request), slug, email: user.email, name: user.name || user.displayName || '' });
|
|
237
|
+
return reply.send(out);
|
|
238
|
+
} catch (err) {
|
|
239
|
+
return reply.code(err instanceof storeClient.StoreError ? err.status : 500).send({ error: err instanceof storeClient.StoreError ? err.message : 'The purchase could not be started.' });
|
|
118
240
|
}
|
|
119
241
|
});
|
|
120
242
|
|
|
243
|
+
// POST /plugins/marketplace/claim {id} -> {status, slug, installed?}
|
|
244
|
+
// Paid for a first purchase: installed here and then (the admin restarts to switch it on).
|
|
245
|
+
fastify.post('/plugins/marketplace/claim', adminOnly, async (request, reply) => {
|
|
246
|
+
const id = String(request.body?.id || '');
|
|
247
|
+
if (!/^[0-9a-f-]{36}$/i.test(id)) return reply.code(400).send({ error: 'Which purchase?' });
|
|
248
|
+
// One claim at a time: two tabs asking at once must not both install it.
|
|
249
|
+
const run = claimChain.then(() => claimAndInstall(id), () => claimAndInstall(id));
|
|
250
|
+
claimChain = run.catch(() => {});
|
|
251
|
+
const r = await run;
|
|
252
|
+
return reply.code(r.code || 200).send(r.body);
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
let claimChain = Promise.resolve();
|
|
256
|
+
async function claimAndInstall(id) {
|
|
257
|
+
let out;
|
|
258
|
+
try {
|
|
259
|
+
out = await storeClient.claimPurchase(id);
|
|
260
|
+
} catch (err) {
|
|
261
|
+
return { code: err instanceof storeClient.StoreError ? err.status : 500, body: { error: err instanceof storeClient.StoreError ? err.message : 'The purchase could not be checked.' } };
|
|
262
|
+
}
|
|
263
|
+
if (out.status !== 'paid') return { body: out };
|
|
264
|
+
const main = await installClaimed(out);
|
|
265
|
+
// What came free with it (the MongoDB Adapter with Email): installed beside it, the same way.
|
|
266
|
+
const included = [];
|
|
267
|
+
for (const slug of out.included || []) {
|
|
268
|
+
const r = await installClaimed({ ...out, slug, kind: out.kind, included: [] });
|
|
269
|
+
included.push({ slug, installed: r.body.installed === true, installError: r.body.installError || null });
|
|
270
|
+
}
|
|
271
|
+
return { code: main.code, body: { ...main.body, included } };
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/** One claimed Tool onto this site: installed, or its licence written beside a copy already here. */
|
|
275
|
+
async function installClaimed(out) {
|
|
276
|
+
const here = (await discoverPlugins()).find(m => m.name === out.slug);
|
|
277
|
+
if (here) {
|
|
278
|
+
// Already on disk (a renewal, or a copy that was here before it was bought): the licence file the purchase
|
|
279
|
+
// brings is written beside it - at once when the store has that same version; a newer one waits for Update.
|
|
280
|
+
const refreshed = await refreshEntitlement(out.slug, here.version).catch(() => false);
|
|
281
|
+
if (refreshed || out.kind === 'renew') return { body: { ...out, installed: true, refreshed } };
|
|
282
|
+
// A first purchase over an older copy: bring it up to the store's version, which carries the licence.
|
|
283
|
+
try {
|
|
284
|
+
const r = await installFromStore(out.slug, { update: true });
|
|
285
|
+
return { body: { ...out, installed: true, refreshed: true, ...r, restartRequired: true } };
|
|
286
|
+
} catch (err) {
|
|
287
|
+
fastify.log.warn({ err }, '[marketplace] update after purchase failed');
|
|
288
|
+
return { body: { ...out, installed: true, refreshed: false, installError: installError(err) } };
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
if (out.kind === 'renew') return { body: { ...out, installed: false } };
|
|
292
|
+
try {
|
|
293
|
+
const r = await installFromStore(out.slug);
|
|
294
|
+
return { body: { ...out, installed: true, ...r, restartRequired: true } };
|
|
295
|
+
} catch (err) {
|
|
296
|
+
fastify.log.warn({ err }, '[marketplace] install after purchase failed');
|
|
297
|
+
// Paid and the key kept: Install on the Browse tab tries again, and the email has the file.
|
|
298
|
+
return { body: { ...out, installed: false, installError: installError(err) } };
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// POST /plugins/marketplace/key {slug, key} -> {owned}
|
|
303
|
+
// A key bought on the public /marketplace page (or found in the email): kept, then checked with the store.
|
|
304
|
+
fastify.post('/plugins/marketplace/key', adminOnly, async (request, reply) => {
|
|
305
|
+
if (isManaged() || !storeClient.storeOpen()) return reply.code(409).send({ error: 'This site does not buy from the Domma store.' });
|
|
306
|
+
const slug = String(request.body?.slug || '');
|
|
307
|
+
try {
|
|
308
|
+
const before = storeClient.readState().keys[slug];
|
|
309
|
+
await storeClient.saveKey(slug, request.body?.key);
|
|
310
|
+
const answer = await storeClient.fetchStoreCatalogue(await siteAddress(request), storeClient.heldKeys([slug]));
|
|
311
|
+
const row = (answer.plugins || []).find(e => e.slug === slug);
|
|
312
|
+
if (!row?.owned) {
|
|
313
|
+
// Not this site's key for this Tool: put back what was there.
|
|
314
|
+
await storeClient.changeState(s => { if (before) s.keys[slug] = before; else delete s.keys[slug]; });
|
|
315
|
+
return reply.code(400).send({ error: answer.siteError || `That key is not a licence for this Tool on ${answer.site || 'this site'}. A licence is for the address it was bought for.` });
|
|
316
|
+
}
|
|
317
|
+
return reply.send({ owned: true, updates: row.updates === true });
|
|
318
|
+
} catch (err) {
|
|
319
|
+
return reply.code(err instanceof storeClient.StoreError ? err.status : 500).send({ error: err instanceof storeClient.StoreError ? err.message : 'The key could not be checked.' });
|
|
320
|
+
}
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
// Connect to Domma: POST {email?} starts it (a link goes to that email), POST /check asks, DELETE disconnects.
|
|
324
|
+
fastify.post('/plugins/marketplace/connect', adminOnly, async (request, reply) => {
|
|
325
|
+
if (isManaged() || !storeClient.storeOpen()) return reply.code(409).send({ error: 'This site does not buy from the Domma store.' });
|
|
326
|
+
const email = String(request.body?.email || '').trim() || (await getUserById(request.user?.id))?.email || '';
|
|
327
|
+
try {
|
|
328
|
+
return reply.send({ connection: await storeClient.startConnection(await siteAddress(request), email) });
|
|
329
|
+
} catch (err) {
|
|
330
|
+
return reply.code(err instanceof storeClient.StoreError ? err.status : 500).send({ error: err instanceof storeClient.StoreError ? err.message : 'Could not connect.' });
|
|
331
|
+
}
|
|
332
|
+
});
|
|
333
|
+
fastify.post('/plugins/marketplace/connect/check', adminOnly, async (_request, reply) => {
|
|
334
|
+
try {
|
|
335
|
+
return reply.send({ connection: await storeClient.syncConnection() });
|
|
336
|
+
} catch (err) {
|
|
337
|
+
return reply.code(err instanceof storeClient.StoreError ? err.status : 500).send({ error: err instanceof storeClient.StoreError ? err.message : 'Could not check.' });
|
|
338
|
+
}
|
|
339
|
+
});
|
|
340
|
+
fastify.delete('/plugins/marketplace/connect', adminOnly, async (_request, reply) => {
|
|
341
|
+
await storeClient.forgetConnection();
|
|
342
|
+
return reply.send({ success: true });
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
// DELETE /plugins/marketplace/purchases/:id - stop waiting (nothing is cancelled at the store).
|
|
346
|
+
fastify.delete('/plugins/marketplace/purchases/:id', adminOnly, async (request, reply) => {
|
|
347
|
+
await storeClient.dropPurchase(String(request.params.id || ''));
|
|
348
|
+
return reply.send({ success: true });
|
|
349
|
+
});
|
|
350
|
+
|
|
121
351
|
// DELETE /plugins/marketplace/:slug
|
|
122
352
|
fastify.delete('/plugins/marketplace/:slug', adminOnly, async (request, reply) => {
|
|
123
353
|
const { slug } = request.params;
|
|
@@ -5,7 +5,11 @@
|
|
|
5
5
|
*
|
|
6
6
|
* GET /status staged uploads, the running job, recent runs, what posts and products can become
|
|
7
7
|
* GET /count the sidebar badge
|
|
8
|
-
* POST /upload a WXR export (.xml, or a .zip holding one) - multipart, field "file"
|
|
8
|
+
* POST /upload a WXR export (.xml, or a .zip holding one) - multipart, field "file";
|
|
9
|
+
* Squarespace's export is one too (services/wpImport/platforms.js)
|
|
10
|
+
* POST /crawl {url} -> read a live site (Wix, or any) into an export, in the background
|
|
11
|
+
* GET /crawl that reading's progress; when done, `stagedId` is the export waiting to run
|
|
12
|
+
* DELETE /crawl stop it
|
|
9
13
|
* POST /staged/:id/uploads the wp-content/uploads folder as a .zip - multipart, field "file"
|
|
10
14
|
* DELETE /staged/:id/uploads drop that zip
|
|
11
15
|
* POST /staged/:id/widgets the widgets, as a .wie file (the "Widget Importer & Exporter" plugin's) - multipart, field "file"
|
|
@@ -38,6 +42,7 @@ import {
|
|
|
38
42
|
stagingPath, startRun, updateStagingMeta, UPLOADS_ZIP_MAX_BYTES, WIE_MAX_BYTES, widgetsMeta, WXR_MAX_BYTES, zipRel
|
|
39
43
|
} from '../../services/wpImport/index.js';
|
|
40
44
|
import {readWie} from '../../services/wpImport/widgets.js';
|
|
45
|
+
import {crawlSite, CrawlError, startUrl} from '../../services/wpImport/crawl.js';
|
|
41
46
|
import {getReport, isId, listHistory, listStaging, removeHistory} from '../../services/wpImport/store.js';
|
|
42
47
|
import {openZip} from '../../services/wpImport/zip.js';
|
|
43
48
|
import {seoEngine} from '../../services/seo/index.js';
|
|
@@ -64,6 +69,43 @@ async function saveUpload(file, dest, max) {
|
|
|
64
69
|
|
|
65
70
|
const mb = (n) => `${Math.round(n / 1048576)} MB`;
|
|
66
71
|
|
|
72
|
+
/**
|
|
73
|
+
* The one site being read at a time (a crawl is minutes of outbound requests).
|
|
74
|
+
* Kept in memory: a restart drops it, and the person starts it again.
|
|
75
|
+
*/
|
|
76
|
+
let crawl = null;
|
|
77
|
+
let crawlAbort = null;
|
|
78
|
+
export const crawlState = () => (crawl ? {...crawl} : null);
|
|
79
|
+
|
|
80
|
+
/** Read a site in the background; when it is read, its export is staged like an upload. */
|
|
81
|
+
export function beginCrawl(url, by, opts = {}) {
|
|
82
|
+
const home = startUrl(url);
|
|
83
|
+
crawl = {url: home, status: 'running', phase: 'Starting', found: 0, read: 0, failed: 0, at: new Date().toISOString(), by};
|
|
84
|
+
crawlAbort = new AbortController();
|
|
85
|
+
const mine = crawl;
|
|
86
|
+
const done = (async () => {
|
|
87
|
+
try {
|
|
88
|
+
const {xml, summary} = await crawlSite(home, {
|
|
89
|
+
signal: crawlAbort.signal, fetchOpts: opts.fetchOpts, maxPages: opts.maxPages,
|
|
90
|
+
onProgress: (p) => Object.assign(mine, p)
|
|
91
|
+
});
|
|
92
|
+
if (crawl !== mine) return;
|
|
93
|
+
if (Buffer.byteLength(xml) > WXR_MAX_BYTES) throw new CrawlError(`The site came to more than ${mb(WXR_MAX_BYTES)} - too big to import in one go.`);
|
|
94
|
+
try { allowInGitignore(process.cwd(), [IMPORT_RULE]); } catch { /* not a managed site repo */ }
|
|
95
|
+
const host = new URL(home).hostname.replace(/[^\w.-]/g, '_');
|
|
96
|
+
const id = await createStaging({fileName: `${host} (read ${new Date().toISOString().slice(0, 10)}).xml`, kind: 'xml', by, source: 'crawl', crawl: summary});
|
|
97
|
+
await fs.promises.writeFile(stagingPath(id, 'export.xml'), xml, {mode: 0o600});
|
|
98
|
+
await updateStagingMeta(id, {size: Buffer.byteLength(xml)});
|
|
99
|
+
const meta = await inspectStaged(id);
|
|
100
|
+
Object.assign(mine, {status: 'done', phase: 'Read', stagedId: id, summary, meta});
|
|
101
|
+
} catch (err) {
|
|
102
|
+
if (crawl !== mine) return;
|
|
103
|
+
Object.assign(mine, {status: crawlAbort?.signal.aborted ? 'stopped' : 'failed', error: err.message || String(err)});
|
|
104
|
+
}
|
|
105
|
+
})();
|
|
106
|
+
return {state: crawlState(), done};
|
|
107
|
+
}
|
|
108
|
+
|
|
67
109
|
/**
|
|
68
110
|
* What the person running the import may do with users: whether they may add
|
|
69
111
|
* accounts at all (users.create), and the roles they may give - every role on
|
|
@@ -162,6 +204,7 @@ export async function wordpressImportRoutes(fastify, opts = {}) {
|
|
|
162
204
|
shop: shopForScreen(shop),
|
|
163
205
|
staged: await listStaging(),
|
|
164
206
|
running: job ? {...job} : null,
|
|
207
|
+
crawl: crawlState(),
|
|
165
208
|
history: (await listHistory()).slice(0, 50),
|
|
166
209
|
blog: {available: blog.available, basePath: blog.basePath, comments: Boolean(blog.comments), reason: blog.reason || ''},
|
|
167
210
|
redirects: Boolean(seoEngine()),
|
|
@@ -176,7 +219,7 @@ export async function wordpressImportRoutes(fastify, opts = {}) {
|
|
|
176
219
|
const staged = await listStaging();
|
|
177
220
|
if (job) {
|
|
178
221
|
return {count: 1, label: `Importing - ${job.phase}`, tone: 'live',
|
|
179
|
-
details: {title: 'Import
|
|
222
|
+
details: {title: 'Import a site', rows: [{label: 'Progress', value: `${job.done} of ${job.total}`}], items: [], empty: ''}};
|
|
180
223
|
}
|
|
181
224
|
const last = runs[0];
|
|
182
225
|
return {
|
|
@@ -184,7 +227,7 @@ export async function wordpressImportRoutes(fastify, opts = {}) {
|
|
|
184
227
|
label: runs.length ? `${plural(runs.length, 'import')} so far` : 'Nothing imported yet',
|
|
185
228
|
tone: 'info',
|
|
186
229
|
details: {
|
|
187
|
-
title: 'Import
|
|
230
|
+
title: 'Import a site',
|
|
188
231
|
rows: [
|
|
189
232
|
{label: 'Imports', value: String(runs.length)},
|
|
190
233
|
{label: 'Waiting to run', value: String(staged.length)},
|
|
@@ -218,6 +261,22 @@ export async function wordpressImportRoutes(fastify, opts = {}) {
|
|
|
218
261
|
}
|
|
219
262
|
});
|
|
220
263
|
|
|
264
|
+
fastify.post('/crawl', may, async (request, reply) => {
|
|
265
|
+
if (crawl?.status === 'running') return reply.code(409).send({error: `Already reading ${crawl.url} - wait for it, or stop it.`});
|
|
266
|
+
try {
|
|
267
|
+
return beginCrawl(request.body?.url, request.user?.email || '', opts.crawl || {}).state;
|
|
268
|
+
} catch (err) {
|
|
269
|
+
if (err instanceof CrawlError) return reply.code(400).send({error: err.message});
|
|
270
|
+
return fail(reply, err);
|
|
271
|
+
}
|
|
272
|
+
});
|
|
273
|
+
fastify.get('/crawl', may, async () => ({crawl: crawlState()}));
|
|
274
|
+
fastify.delete('/crawl', may, async () => {
|
|
275
|
+
if (crawl?.status === 'running') crawlAbort?.abort();
|
|
276
|
+
else crawl = null;
|
|
277
|
+
return {crawl: crawlState()};
|
|
278
|
+
});
|
|
279
|
+
|
|
221
280
|
fastify.post('/staged/:id/uploads', may, async (request, reply) => {
|
|
222
281
|
const {id} = request.params;
|
|
223
282
|
const meta = isId(id) ? await stagingMeta(id) : null;
|
package/server/server.js
CHANGED
|
@@ -679,7 +679,7 @@ for (const item of coreToolItems) toolEntry(item.permission, item);
|
|
|
679
679
|
// Launchers sit in Content beside Menus and Context Menus; switched off, no entry (toolEntry).
|
|
680
680
|
if (isToolEnabled('launchers')) registerSidebarItem({folder: 'content', item: {source: 'Built in', text: 'Launchers', url: '#/launchers', icon: 'rocket', permission: 'settings', countUrl: '/api/launchers/_count', inventory: true}});
|
|
681
681
|
else if (isToolStaffOnly('launchers')) registerSidebarItem({folder: 'content', item: {source: 'Built in', text: 'Launchers', url: '#/launchers', icon: 'rocket', permission: 'settings', countUrl: '/api/launchers/_count', inventory: true, staffOnly: true}});
|
|
682
|
-
toolEntry('wordpress-import', {text: 'Import
|
|
682
|
+
toolEntry('wordpress-import', {text: 'Import a site', url: '#/wordpress-import', icon: 'download', permission: 'import', countUrl: '/api/import/wordpress/count', inventory: true});
|
|
683
683
|
// Todo's to-dos on the calendar (registered when routes/api/todo.js loads).
|
|
684
684
|
if (!isToolEnabled('todo')) {
|
|
685
685
|
const {unregisterCalendarSource} = await import('./services/hooks.js');
|
|
@@ -332,6 +332,8 @@ export function buildGitignore({sitePlugins = [], pluginContent = {}, sitePublic
|
|
|
332
332
|
// Each admin's own sidebar Frequent list (keyed by user id).
|
|
333
333
|
'/content/sidebar/frequent/',
|
|
334
334
|
'/config/connections.json*',
|
|
335
|
+
// The Domma store's licence keys and payment claims (services/storeClient.js).
|
|
336
|
+
'/config/store.json*',
|
|
335
337
|
'/config/keys/',
|
|
336
338
|
'/plugins/*/data/*.jsonl',
|
|
337
339
|
'/plugins/*/backups/',
|