domma-cms 1.38.0 → 1.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/CLAUDE.md +17 -0
- package/admin/js/api.js +1 -1
- package/admin/js/app.js +1 -1
- package/admin/js/lib/plugins-arrange.js +2 -2
- package/admin/js/lib/wordpress-import-arrange.js +1 -1
- package/admin/js/templates/docs/usage-tools.html +18 -3
- package/admin/js/templates/plugins.html +2 -0
- package/admin/js/templates/wordpress-import.html +23 -5
- package/admin/js/views/plugins.js +24 -12
- package/admin/js/views/wordpress-import.js +10 -10
- package/bin/update.js +2 -0
- package/docs/DommaFeatures.md +7 -0
- package/docs/configuration.md +1 -0
- package/docs/plugin-development.md +11 -0
- package/package.json +1 -1
- package/server/routes/api/plugin-marketplace.js +249 -19
- package/server/routes/api/wordpress-import.js +62 -3
- package/server/server.js +1 -1
- package/server/services/siteGitignore.js +2 -0
- package/server/services/storeClient.js +333 -0
- package/server/services/tools.js +1 -1
- package/server/services/wpImport/crawl.js +619 -0
- package/server/services/wpImport/index.js +53 -8
- package/server/services/wpImport/plan.js +5 -3
- package/server/services/wpImport/platforms.js +186 -0
- package/server/services/wpImport/wxr.js +6 -2
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Domma store - how a SELF-HOSTED site buys and installs Tools.
|
|
3
|
+
*
|
|
4
|
+
* A site a manager runs (managerClient.js) gets its catalogue over the fleet
|
|
5
|
+
* token. A self-hosted one has no token and no manager; its Marketplace asks the
|
|
6
|
+
* public store instead (Domma CMS Manager's site-manager `store/*` routes):
|
|
7
|
+
*
|
|
8
|
+
* catalogue what is on sale, priced for self-hosting, and - for the licence
|
|
9
|
+
* keys this site holds - whether it owns each and has updates
|
|
10
|
+
* checkout a Tool for this site's address -> a payment page and a CLAIM
|
|
11
|
+
* secret, which is kept here and never shown to the browser
|
|
12
|
+
* claim once paid, the claim returns the licence key
|
|
13
|
+
* download the signed .dcmsplugin for a key (free plugins need none), which
|
|
14
|
+
* the ordinary installer then checks and installs
|
|
15
|
+
*
|
|
16
|
+
* Bought outright: the plugin keeps running offline for good; support (newer
|
|
17
|
+
* versions) lasts a term and is renewed with a one-off payment. One licence per
|
|
18
|
+
* site address.
|
|
19
|
+
*
|
|
20
|
+
* Where: `DOMMA_STORE_URL` in the environment, else server.json `storeUrl`, else
|
|
21
|
+
* Domma's own. `storeUrl: false` switches the store off. It is not offered on a
|
|
22
|
+
* managed site (it buys through its manager), nor on an owner instance (a
|
|
23
|
+
* wildcard entitlement - the manager itself, where every plugin is written)
|
|
24
|
+
* unless a store is named there on purpose.
|
|
25
|
+
*
|
|
26
|
+
* Kept in `config/store.json`: the keys this site was given and the purchases
|
|
27
|
+
* still waiting for payment. A key also lives in each installed plugin's
|
|
28
|
+
* `.entitlement.json` (`licenceKey`), which is read too.
|
|
29
|
+
*
|
|
30
|
+
* @module server/services/storeClient
|
|
31
|
+
*/
|
|
32
|
+
import fs from 'node:fs';
|
|
33
|
+
import path from 'node:path';
|
|
34
|
+
import {getConfig} from '../config.js';
|
|
35
|
+
import {isManaged} from './managerClient.js';
|
|
36
|
+
import {classifyEntitlement, readEntitlement} from './pluginEntitlement.js';
|
|
37
|
+
|
|
38
|
+
export const DEFAULT_STORE_URL = 'https://dcm.dommajs.org/api/plugins/site-manager/store';
|
|
39
|
+
/** A purchase not paid for after this long is let go (the store's own claim lasts 14 days). */
|
|
40
|
+
export const PENDING_DAYS = 14;
|
|
41
|
+
const SLUG = /^[a-z0-9][a-z0-9-_]{0,63}$/;
|
|
42
|
+
const TIMEOUT = 15_000;
|
|
43
|
+
|
|
44
|
+
/** The store's address, or '' when it is switched off. */
|
|
45
|
+
export function storeUrl() {
|
|
46
|
+
const env = process.env.DOMMA_STORE_URL;
|
|
47
|
+
if (env !== undefined) return env === '' || env === 'false' ? '' : (safeStoreUrl(env) || '');
|
|
48
|
+
let configured;
|
|
49
|
+
try { configured = getConfig('server')?.storeUrl; } catch { configured = undefined; }
|
|
50
|
+
if (configured === false) return '';
|
|
51
|
+
if (typeof configured === 'string') return safeStoreUrl(configured) || '';
|
|
52
|
+
return DEFAULT_STORE_URL;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* HTTPS only - the claim secret and licence keys travel in these requests. Plain HTTP is allowed to this machine
|
|
57
|
+
* alone (a store on loopback, as the tests run one). Anything else switches the store off rather than leak.
|
|
58
|
+
*/
|
|
59
|
+
export function safeStoreUrl(raw) {
|
|
60
|
+
let u;
|
|
61
|
+
try { u = new URL(String(raw).trim()); } catch { return null; }
|
|
62
|
+
const loopback = ['127.0.0.1', 'localhost', '[::1]'].includes(u.hostname);
|
|
63
|
+
if (u.protocol !== 'https:' && !(u.protocol === 'http:' && loopback)) return null;
|
|
64
|
+
return `${u.origin}${u.pathname.replace(/\/+$/, '')}`;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** An owner instance: a verified wildcard entitlement. Every plugin is already its own. */
|
|
68
|
+
export function isOwnerInstance() {
|
|
69
|
+
try {
|
|
70
|
+
const e = readEntitlement('__store-owner-probe__');
|
|
71
|
+
return e?.slug === '*' && ['valid', 'grace', 'support-ended'].includes(classifyEntitlement('__store-owner-probe__', e).state);
|
|
72
|
+
} catch {
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Was a store named on purpose (server.json `storeUrl` or `DOMMA_STORE_URL`), rather than the default? */
|
|
78
|
+
export function storeNamed() {
|
|
79
|
+
if (process.env.DOMMA_STORE_URL) return true;
|
|
80
|
+
try { return typeof getConfig('server')?.storeUrl === 'string'; } catch { return false; }
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Does this site buy from the store? Never a managed site. An owner instance only when a store is named on
|
|
85
|
+
* purpose - that is how the manager's own admin shows the store screen for a look at it.
|
|
86
|
+
*/
|
|
87
|
+
export function storeOpen() {
|
|
88
|
+
return !isManaged() && Boolean(storeUrl()) && (!isOwnerInstance() || storeNamed());
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// ---------------------------------------------------------------------------
|
|
92
|
+
// What this site holds - config/store.json
|
|
93
|
+
// ---------------------------------------------------------------------------
|
|
94
|
+
|
|
95
|
+
const STATE_FILE = () => path.resolve('config', 'store.json');
|
|
96
|
+
|
|
97
|
+
export function readState() {
|
|
98
|
+
try {
|
|
99
|
+
const raw = JSON.parse(fs.readFileSync(STATE_FILE(), 'utf8'));
|
|
100
|
+
return {
|
|
101
|
+
keys: raw?.keys && typeof raw.keys === 'object' ? raw.keys : {},
|
|
102
|
+
pending: Array.isArray(raw?.pending) ? raw.pending : [],
|
|
103
|
+
connection: raw?.connection && typeof raw.connection === 'object' ? raw.connection : null
|
|
104
|
+
};
|
|
105
|
+
} catch {
|
|
106
|
+
return {keys: {}, pending: [], connection: null};
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function writeState(state) {
|
|
111
|
+
const file = STATE_FILE();
|
|
112
|
+
const tmp = `${file}.tmp-${process.pid}`;
|
|
113
|
+
fs.writeFileSync(tmp, JSON.stringify(state, null, 2) + '\n', {mode: 0o600});
|
|
114
|
+
fs.renameSync(tmp, file);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
let chain = Promise.resolve();
|
|
118
|
+
/** Every change to config/store.json, one at a time. */
|
|
119
|
+
export function changeState(fn) {
|
|
120
|
+
const run = chain.then(() => {
|
|
121
|
+
const state = readState();
|
|
122
|
+
const out = fn(state);
|
|
123
|
+
writeState(state);
|
|
124
|
+
return out;
|
|
125
|
+
});
|
|
126
|
+
chain = run.catch(() => {});
|
|
127
|
+
return run;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The licence keys this site holds, by plugin: config/store.json first, then
|
|
132
|
+
* what each installed plugin's entitlement carries (a plugin installed from a
|
|
133
|
+
* file bought by email).
|
|
134
|
+
*/
|
|
135
|
+
export function heldKeys(installedSlugs = []) {
|
|
136
|
+
const keys = {...readState().keys};
|
|
137
|
+
for (const slug of installedSlugs) {
|
|
138
|
+
if (keys[slug] || !SLUG.test(slug)) continue;
|
|
139
|
+
try {
|
|
140
|
+
const own = JSON.parse(fs.readFileSync(path.resolve('plugins', slug, '.entitlement.json'), 'utf8'));
|
|
141
|
+
if (own?.slug === slug && typeof own.licenceKey === 'string') keys[slug] = own.licenceKey;
|
|
142
|
+
} catch { /* none */ }
|
|
143
|
+
}
|
|
144
|
+
return keys;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Purchases still waiting, minus any past PENDING_DAYS. */
|
|
148
|
+
export function pendingPurchases(now = Date.now()) {
|
|
149
|
+
return readState().pending.filter((p) => now - Date.parse(p.startedAt) < PENDING_DAYS * 86_400_000);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// ---------------------------------------------------------------------------
|
|
153
|
+
// Talking to the store
|
|
154
|
+
// ---------------------------------------------------------------------------
|
|
155
|
+
|
|
156
|
+
/** A refusal from the store, carried to the admin as its own words. */
|
|
157
|
+
export class StoreError extends Error {
|
|
158
|
+
constructor(message, status = 502, code = null) {
|
|
159
|
+
super(message);
|
|
160
|
+
this.status = status;
|
|
161
|
+
this.code = code;
|
|
162
|
+
this.expose = true;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
async function call(pathname, body) {
|
|
167
|
+
const base = storeUrl();
|
|
168
|
+
if (!base) throw new StoreError('The store is switched off on this site.', 503);
|
|
169
|
+
let res;
|
|
170
|
+
try {
|
|
171
|
+
res = await fetch(`${base}${pathname}`, {
|
|
172
|
+
method: 'POST',
|
|
173
|
+
headers: {'Content-Type': 'application/json', Accept: 'application/json'},
|
|
174
|
+
body: JSON.stringify(body),
|
|
175
|
+
signal: AbortSignal.timeout(TIMEOUT)
|
|
176
|
+
});
|
|
177
|
+
} catch (err) {
|
|
178
|
+
console.warn(`[storeClient] ${pathname}: ${err.message}`);
|
|
179
|
+
throw new StoreError('The Domma store could not be reached. Check this server can make outgoing HTTPS requests, then try again.', 503);
|
|
180
|
+
}
|
|
181
|
+
let data = null;
|
|
182
|
+
try { data = await res.json(); } catch { /* not JSON */ }
|
|
183
|
+
if (!res.ok) {
|
|
184
|
+
throw new StoreError(typeof data?.error === 'string' ? data.error.slice(0, 400) : `The store answered ${res.status}.`, res.status >= 500 ? 502 : res.status, data?.code || null);
|
|
185
|
+
}
|
|
186
|
+
return data || {};
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** The store's catalogue for this site (`site` = its public origin). */
|
|
190
|
+
export function fetchStoreCatalogue(site, keys) {
|
|
191
|
+
return call('/catalogue', {site, keys});
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Start buying (or renewing) a Tool. The claim secret is kept here; the browser
|
|
196
|
+
* is given only the payment page.
|
|
197
|
+
*
|
|
198
|
+
* @returns {Promise<{id: string, checkoutUrl: string, kind: string, price: number, currency: string}>}
|
|
199
|
+
*/
|
|
200
|
+
export async function startPurchase({site, slug, email, name}) {
|
|
201
|
+
const key = readState().keys[slug] || heldKeys([slug])[slug] || '';
|
|
202
|
+
const out = await call('/checkout', {site, slug, email, name, ...(key && {key})});
|
|
203
|
+
if (typeof out.id !== 'string' || typeof out.claim !== 'string' || typeof out.checkoutUrl !== 'string' || !/^https:\/\//i.test(out.checkoutUrl)) {
|
|
204
|
+
throw new StoreError('The store gave an answer this site does not understand.', 502);
|
|
205
|
+
}
|
|
206
|
+
await changeState((s) => {
|
|
207
|
+
// An earlier checkout for the same Tool stays: if its tab is paid after all, it can still be claimed
|
|
208
|
+
// (the store refunds a second payment for a site that already has it, and hands either claim the key).
|
|
209
|
+
s.pending.push({id: out.id, claim: out.claim, slug, kind: out.kind, startedAt: new Date().toISOString()});
|
|
210
|
+
});
|
|
211
|
+
return {id: out.id, checkoutUrl: out.checkoutUrl, kind: out.kind, price: out.price, currency: out.currency};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Has a purchase been paid? Paid: the key is kept and the purchase leaves the
|
|
216
|
+
* waiting list. `{status, slug, kind}`; status is the store's (pending, paid,
|
|
217
|
+
* failed, refunded, revoked, expired) or 'unknown' for a purchase this site
|
|
218
|
+
* does not hold.
|
|
219
|
+
*/
|
|
220
|
+
export async function claimPurchase(id) {
|
|
221
|
+
const p = readState().pending.find((x) => x.id === id);
|
|
222
|
+
if (!p) return {status: 'unknown'};
|
|
223
|
+
const out = await call('/claim', {id: p.id, claim: p.claim});
|
|
224
|
+
if (out.status === 'paid' && typeof out.licenceKey === 'string') {
|
|
225
|
+
await changeState((s) => {
|
|
226
|
+
s.keys[p.slug] = out.licenceKey;
|
|
227
|
+
// Given free with it (the MongoDB Adapter with Email): kept the same way.
|
|
228
|
+
for (const inc of Array.isArray(out.included) ? out.included : []) {
|
|
229
|
+
if (SLUG.test(String(inc?.slug || '')) && KEY_FORMAT.test(String(inc?.licenceKey || ''))) s.keys[inc.slug] = inc.licenceKey;
|
|
230
|
+
}
|
|
231
|
+
s.pending = s.pending.filter((x) => x.id !== id);
|
|
232
|
+
});
|
|
233
|
+
} else if (['failed', 'refunded', 'revoked', 'expired'].includes(out.status)) {
|
|
234
|
+
await changeState((s) => { s.pending = s.pending.filter((x) => x.id !== id); });
|
|
235
|
+
}
|
|
236
|
+
return {status: out.status, slug: p.slug, kind: p.kind,
|
|
237
|
+
included: (Array.isArray(out.included) ? out.included : []).map((i) => i?.slug).filter((x) => SLUG.test(String(x || '')))};
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** A licence key typed in (bought on the public /marketplace page, or from the email): kept for that plugin. */
|
|
241
|
+
export const KEY_FORMAT = /^DOMMA-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}$/;
|
|
242
|
+
export function saveKey(slug, key) {
|
|
243
|
+
const k = String(key || '').trim().toUpperCase();
|
|
244
|
+
if (!SLUG.test(String(slug || '')) || !KEY_FORMAT.test(k)) throw new StoreError('That does not look like a licence key - it is like DOMMA-1A2B-3C4D-5E6F.', 400);
|
|
245
|
+
return changeState((s) => { s.keys[slug] = k; });
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// ---------------------------------------------------------------------------
|
|
249
|
+
// Connect to Domma - the licences bought for this site's address with one email, picked up by themselves
|
|
250
|
+
// ---------------------------------------------------------------------------
|
|
251
|
+
|
|
252
|
+
/** What the screen may know about the connection: never its secret. */
|
|
253
|
+
export function connectionView(state = readState()) {
|
|
254
|
+
const c = state.connection;
|
|
255
|
+
return c ? {status: c.status, email: c.email, site: c.site || null, startedAt: c.startedAt || null} : null;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Ask to connect: the store emails `email` a link to say yes. The secret stays here. */
|
|
259
|
+
export async function startConnection(site, email) {
|
|
260
|
+
const out = await call('/connect/start', {site, email});
|
|
261
|
+
if (typeof out.id !== 'string' || typeof out.secret !== 'string') throw new StoreError('The store gave an answer this site does not understand.', 502);
|
|
262
|
+
await changeState((s) => { s.connection = {id: out.id, secret: out.secret, email: out.email || email, site: out.site || null, status: 'pending', startedAt: new Date().toISOString()}; });
|
|
263
|
+
return connectionView();
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Bring the connection up to date: once approved, every key the store hands over is kept (a key typed in for
|
|
268
|
+
* another Tool is left alone). A connection the store no longer knows, or one never approved in time, is dropped.
|
|
269
|
+
*/
|
|
270
|
+
export async function syncConnection() {
|
|
271
|
+
const c = readState().connection;
|
|
272
|
+
if (!c) return null;
|
|
273
|
+
let out;
|
|
274
|
+
try {
|
|
275
|
+
out = await call('/connect/keys', {id: c.id, secret: c.secret});
|
|
276
|
+
} catch (err) {
|
|
277
|
+
if (err.status === 404) { await changeState((s) => { s.connection = null; }); return null; }
|
|
278
|
+
throw err;
|
|
279
|
+
}
|
|
280
|
+
await changeState((s) => {
|
|
281
|
+
if (!s.connection || s.connection.id !== c.id) return;
|
|
282
|
+
if (out.status === 'connected') {
|
|
283
|
+
s.connection.status = 'connected';
|
|
284
|
+
for (const [slug, key] of Object.entries(out.keys || {})) {
|
|
285
|
+
if (SLUG.test(slug) && KEY_FORMAT.test(String(key))) s.keys[slug] = String(key);
|
|
286
|
+
}
|
|
287
|
+
} else if (out.status === 'expired') {
|
|
288
|
+
s.connection = null;
|
|
289
|
+
}
|
|
290
|
+
});
|
|
291
|
+
return connectionView();
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** Disconnect: the store forgets it too (the keys already here stay - they are this site's). */
|
|
295
|
+
export async function forgetConnection() {
|
|
296
|
+
const c = readState().connection;
|
|
297
|
+
if (!c) return;
|
|
298
|
+
try { await call('/connect/forget', {id: c.id, secret: c.secret}); } catch { /* gone there already */ }
|
|
299
|
+
await changeState((s) => { s.connection = null; });
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** Forget a purchase this site is waiting for (the person gave up on it). */
|
|
303
|
+
export function dropPurchase(id) {
|
|
304
|
+
return changeState((s) => { s.pending = s.pending.filter((x) => x.id !== id); });
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* The signed .dcmsplugin for a plugin - free, or for the key this site holds.
|
|
309
|
+
*
|
|
310
|
+
* @returns {Promise<Buffer>}
|
|
311
|
+
*/
|
|
312
|
+
export async function downloadPlugin(slug) {
|
|
313
|
+
if (!SLUG.test(slug)) throw new StoreError('Invalid plugin slug.', 400);
|
|
314
|
+
const base = storeUrl();
|
|
315
|
+
if (!base) throw new StoreError('The store is switched off on this site.', 503);
|
|
316
|
+
const key = heldKeys([slug])[slug] || '';
|
|
317
|
+
const url = `${base.replace(/\/store$/, '')}/plugins/${encodeURIComponent(slug)}/download${key ? `?licence=${encodeURIComponent(key)}` : ''}`;
|
|
318
|
+
let res;
|
|
319
|
+
try {
|
|
320
|
+
res = await fetch(url, {signal: AbortSignal.timeout(60_000)});
|
|
321
|
+
} catch (err) {
|
|
322
|
+
console.warn(`[storeClient] download ${slug}: ${err.message}`);
|
|
323
|
+
throw new StoreError('The Domma store could not be reached. Check this server can make outgoing HTTPS requests, then try again.', 503);
|
|
324
|
+
}
|
|
325
|
+
if (!res.ok) {
|
|
326
|
+
let data = null;
|
|
327
|
+
try { data = await res.json(); } catch { /* not JSON */ }
|
|
328
|
+
throw new StoreError(typeof data?.error === 'string' ? data.error.slice(0, 400) : `The store answered ${res.status}.`, res.status >= 500 ? 502 : 403, data?.code || null);
|
|
329
|
+
}
|
|
330
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
331
|
+
if (buf.length > 64 * 1024 * 1024) throw new StoreError('That download is larger than any plugin should be.', 502);
|
|
332
|
+
return buf;
|
|
333
|
+
}
|
package/server/services/tools.js
CHANGED
|
@@ -65,7 +65,7 @@ export const CORE_TOOLS = Object.freeze([
|
|
|
65
65
|
{name: 'analytics', displayName: 'Analytics', icon: 'chart-bar', description: 'Page views and visitors, counted on this server.'},
|
|
66
66
|
{name: 'launchers', displayName: 'Launchers', icon: 'message-circle', description: 'Discs pinned to the page that open a page, a form or a chat - in a modal or a slideover.'},
|
|
67
67
|
{name: 'seo', displayName: 'SEO', icon: 'search', description: 'How the site looks to search engines: an audit of every page, redirects, the sitemap and robots.txt.'},
|
|
68
|
-
{name: 'wordpress-import', displayName: 'Import
|
|
68
|
+
{name: 'wordpress-import', displayName: 'Import a site', icon: 'download', description: 'Bring a site across from WordPress, Squarespace or Wix: pages, posts, media, menus, comments, WooCommerce products and redirects from the old addresses. Wix and other sites without an export are read from the live site.'}
|
|
69
69
|
]);
|
|
70
70
|
|
|
71
71
|
const CORE_NAMES = new Set(CORE_TOOLS.map(t => t.name));
|