@pterodoc/wordpress 0.2.0

package/lib/index.js

@@ -0,0 +1,655 @@
+import { renderNavigationStub } from '@pterodoc/core/render';
+export { renderNavigationStub } from '@pterodoc/core/render';
+import { USER_AGENT, TargetError, isBlockedByDefault, joinPath, segments, slugify, titleCase } from '@pterodoc/core/util';
+
+/**
+ * A small WordPress REST client, scoped to what publishing needs.
+ *
+ * Authentication is an Application Password over HTTP Basic. `fetch` and the
+ * clock are injectable so the whole sync can be tested without a network.
+ */
+/** Fields needed to compare a remote page with a rendered one. */
+const PAGE_FIELDS = 'id,parent,slug,status,link,title,menu_order,template';
+/** Those, plus the content only fetched when a page is about to be compared. */
+const FULL_PAGE_FIELDS = `${PAGE_FIELDS},content,excerpt,meta`;
+/** The retry policy used when a site configures none. */
+const DEFAULT_RETRY = { attempts: 4, baseDelayMs: 1000, maxDelayMs: 30_000 };
+class WpClient {
+    baseUrl;
+    user;
+    appPassword;
+    lang;
+    methodOverride;
+    retry;
+    doFetch;
+    sleep;
+    log;
+    /** How many requests have been made, for the run summary. */
+    requestCount = 0;
+    constructor(options) {
+        this.baseUrl = options.baseUrl.replace(/\/+$/, '');
+        this.user = options.user ?? '';
+        this.appPassword = (options.appPassword ?? '').replace(/\s+/g, '');
+        this.lang = options.lang ?? '';
+        this.methodOverride = options.methodOverride === true;
+        this.retry = options.retry ?? DEFAULT_RETRY;
+        this.doFetch = options.fetch ?? globalThis.fetch;
+        this.sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+        this.log = options.log ?? (() => { });
+    }
+    /** Headers every request carries. */
+    headers() {
+        const token = Buffer.from(`${this.user}:${this.appPassword}`, 'utf8').toString('base64');
+        return {
+            Authorization: `Basic ${token}`,
+            Accept: 'application/json',
+            'User-Agent': USER_AGENT,
+        };
+    }
+    /**
+     * Perform one REST call, retrying only what is worth retrying.
+     */
+    async request(method, path, options = {}) {
+        const url = new URL(`${this.baseUrl}/wp-json/wp/v2${path}`);
+        for (const [key, value] of Object.entries(options.query ?? {})) {
+            if (value !== undefined && value !== '')
+                url.searchParams.set(key, String(value));
+        }
+        if (this.lang)
+            url.searchParams.set('lang', this.lang);
+        const headers = this.headers();
+        const init = { method, headers, redirect: 'error' };
+        if (options.form) {
+            init.body = options.form;
+        }
+        else if (options.body !== undefined) {
+            headers['Content-Type'] = 'application/json';
+            init.body = JSON.stringify(options.body);
+        }
+        if (this.methodOverride && method === 'DELETE') {
+            init.method = 'POST';
+            headers['X-HTTP-Method-Override'] = 'DELETE';
+        }
+        let lastError;
+        let retryAfterMs;
+        for (let attempt = 0; attempt <= this.retry.attempts; attempt += 1) {
+            if (attempt > 0) {
+                // Retry-After applies to the response that carried it, not to every
+                // later attempt, so it is consumed rather than remembered.
+                const backoff = Math.min(this.retry.baseDelayMs * 2 ** (attempt - 1), this.retry.maxDelayMs);
+                const wait = retryAfterMs ?? backoff;
+                retryAfterMs = undefined;
+                this.log(`retry ${attempt}/${this.retry.attempts} in ${wait}ms: ${method} ${url.pathname}`);
+                await this.sleep(wait);
+            }
+            this.requestCount += 1;
+            let response;
+            try {
+                response = await this.doFetch(url.href, init);
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                lastError = new TargetError(/redirect/i.test(message)
+                    ? `${method} ${url.pathname} was redirected. Check that WP_URL matches the site exactly, including https and www.`
+                    : `${method} ${url.pathname} failed: ${message}`, { status: 0, method, url: url.href });
+                if (/redirect/i.test(message))
+                    throw lastError;
+                continue;
+            }
+            const text = await response.text();
+            if (response.ok) {
+                try {
+                    return { data: (text ? JSON.parse(text) : null), headers: response.headers };
+                }
+                catch {
+                    throw new TargetError(`${method} ${url.pathname} returned a success body that is not JSON.`, {
+                        status: response.status,
+                        method,
+                        url: url.href,
+                        bodySnippet: text.slice(0, 200),
+                    });
+                }
+            }
+            let code;
+            let message = `${response.status} ${response.statusText}`;
+            const contentType = response.headers.get('content-type') ?? '';
+            if (contentType.includes('json')) {
+                try {
+                    const parsed = JSON.parse(text);
+                    code = parsed.code;
+                    if (parsed.message)
+                        message = `${message}: ${parsed.message}`;
+                }
+                catch {
+                    /* fall through to the snippet */
+                }
+            }
+            else if (text) {
+                message = `${message} (the response was not JSON; a firewall or security plugin may be blocking the REST API)`;
+            }
+            lastError = new TargetError(`${method} ${url.pathname} — ${message}`, {
+                status: response.status,
+                code,
+                method,
+                url: url.href,
+                bodySnippet: text.slice(0, 200),
+            });
+            const retryAfter = Number(response.headers.get('retry-after'));
+            if (Number.isFinite(retryAfter) && retryAfter > 0)
+                retryAfterMs = retryAfter * 1000;
+            if (response.status !== 429 && response.status < 500)
+                throw lastError;
+        }
+        throw lastError ?? new TargetError(`${method} ${url.pathname} failed.`, { status: 0, method, url: url.href });
+    }
+    /** Follow `X-WP-TotalPages` to the end of a collection. */
+    async listAll(path, query) {
+        const all = [];
+        let page = 1;
+        let totalPages = 1;
+        do {
+            const { data, headers } = await this.request('GET', path, {
+                query: { ...query, per_page: 100, page },
+            });
+            if (Array.isArray(data))
+                all.push(...data);
+            const header = Number(headers.get('x-wp-totalpages'));
+            totalPages = Number.isFinite(header) && header > 0 ? header : 1;
+            page += 1;
+        } while (page <= totalPages);
+        return all;
+    }
+}
+
+/**
+ * The WordPress media library.
+ *
+ * A file's identity is the hash of its contents, carried in the media slug.
+ * Keeping the identity on the server rather than in a local cache is what lets
+ * a fresh checkout, or a CI runner that has never seen the site, avoid
+ * uploading everything again.
+ */
+/** The slug that identifies a file uploaded by pterodoc. */
+function mediaSlug(prefix, hash) {
+    return `${prefix}-${hash}`;
+}
+/** Read the hash back out of a slug, when the slug is one of ours. */
+function hashFromSlug(prefix, slug) {
+    const marker = `${prefix}-`;
+    if (!slug.startsWith(marker))
+        return undefined;
+    const hash = slug.slice(marker.length);
+    return /^[0-9a-f]{16}$/.test(hash) ? hash : undefined;
+}
+/**
+ * Everything pterodoc has already uploaded to this site, by content hash.
+ */
+async function loadMediaIndex(client, prefix) {
+    const found = await client.listAll('/media', {
+        search: `${prefix}-`,
+        _fields: 'id,slug,source_url,mime_type',
+    });
+    const byHash = new Map();
+    for (const item of found) {
+        const hash = hashFromSlug(prefix, item.slug);
+        if (!hash || byHash.has(hash))
+            continue;
+        byHash.set(hash, {
+            id: item.id,
+            hash,
+            url: item.source_url,
+            filename: item.slug,
+            mime: item.mime_type,
+        });
+    }
+    return byHash;
+}
+/**
+ * Upload one file.
+ *
+ * WordPress derives a slug from the filename on create, so the identifying
+ * slug has to be set in a second request.
+ */
+async function uploadMedia(client, upload, prefix) {
+    const slug = mediaSlug(prefix, upload.hash);
+    const form = new FormData();
+    const bytes = upload.bytes;
+    const view = new Uint8Array(bytes.byteLength);
+    view.set(bytes);
+    form.append('file', new File([view], upload.filename, { type: upload.mime }));
+    if (upload.title)
+        form.append('title', upload.title);
+    if (upload.alt)
+        form.append('alt_text', upload.alt);
+    let created;
+    try {
+        ({ data: created } = await client.request('POST', '/media', { form }));
+    }
+    catch (error) {
+        if (error instanceof TargetError && error.status === 400 && isBlockedByDefault(upload.mime)) {
+            throw new TargetError(`${upload.filename} was refused. WordPress blocks ${upload.mime} uploads unless a plugin allows them, because such a file can carry script.`, { status: error.status, code: error.code, method: error.method, url: error.url });
+        }
+        throw error;
+    }
+    const { data: updated } = await client.request('POST', `/media/${created.id}`, {
+        body: { slug, ...(upload.title ? { title: upload.title } : {}), ...(upload.alt ? { alt_text: upload.alt } : {}) },
+    });
+    return {
+        id: updated.id,
+        hash: upload.hash,
+        url: updated.source_url || created.source_url,
+        filename: upload.filename,
+        mime: updated.mime_type || upload.mime,
+    };
+}
+
+/**
+ * WordPress pages: finding them, comparing them, writing them, removing them.
+ *
+ * A page's identity is its parent and its slug, which is what makes a re-run
+ * rewrite only what actually differs.
+ */
+/** Convert a WordPress page into the shape the reconciler compares. */
+function toRemotePage(page) {
+    return {
+        id: page.id,
+        parent: page.parent,
+        slug: page.slug,
+        status: page.status,
+        link: page.link,
+        title: page.title?.raw ?? page.title?.rendered ?? '',
+        ...(page.content?.raw !== undefined ? { content: page.content.raw } : {}),
+        ...(page.excerpt?.raw !== undefined ? { excerpt: page.excerpt.raw } : {}),
+        menuOrder: page.menu_order,
+        template: page.template,
+        meta: page.meta,
+    };
+}
+/** Fetch every page on the site. */
+async function fetchPageIndex(client) {
+    const pages = await client.listAll('/pages', {
+        status: 'any',
+        context: 'edit',
+        _fields: PAGE_FIELDS,
+    });
+    return pages.map(toRemotePage);
+}
+/** Fetch one page with the fields needed to compare it. */
+async function fetchPage(client, id) {
+    const { data } = await client.request('GET', `/pages/${id}`, {
+        query: { context: 'edit', _fields: FULL_PAGE_FIELDS },
+    });
+    return toRemotePage(data);
+}
+/**
+ * Find a page by its position in the tree.
+ *
+ * The index is searched when one was supplied, because a whole-site index is
+ * one request where per-page lookups are hundreds.
+ */
+async function findPage(client, parent, slug, index, log = () => { }) {
+    let candidates;
+    if (index) {
+        candidates = index.filter((page) => page.parent === parent && page.slug === slug);
+    }
+    else {
+        const { data } = await client.request('GET', '/pages', {
+            query: { parent, slug, status: 'any', context: 'edit', per_page: 100, _fields: PAGE_FIELDS },
+        });
+        candidates = (Array.isArray(data) ? data : []).map(toRemotePage);
+    }
+    if (candidates.length > 1) {
+        log(`${candidates.length} pages share parent ${parent} and slug "${slug}"; using id ${candidates[0].id}.`);
+    }
+    return candidates[0];
+}
+/** Create a page, checking that WordPress honoured the slug we asked for. */
+async function createPage(client, input) {
+    const { data } = await client.request('POST', '/pages', { body: input });
+    if (input.slug && data.slug !== input.slug) {
+        throw new TargetError(`WordPress stored the new page as "${data.slug}" rather than "${input.slug}". Another page, possibly one in the trash, already holds that slug. The page it created is id ${data.id}.`, { status: 200, method: 'POST', url: '/pages' });
+    }
+    return toRemotePage(data);
+}
+/** Update a page. */
+async function updatePage(client, id, input) {
+    const { data } = await client.request('POST', `/pages/${id}`, { body: input });
+    return toRemotePage(data);
+}
+/**
+ * Move a page to the trash.
+ *
+ * Never a permanent delete: recovering from a mistaken prune should not
+ * require a database backup.
+ */
+async function trashPage(client, id) {
+    await client.request('DELETE', `/pages/${id}`);
+}
+/** Normalise a value for comparison, so whitespace alone is not a difference. */
+const normalise = (value) => String(value ?? '').replace(/\r\n/g, '\n').trim();
+/**
+ * Which fields of an existing page differ from the rendered one.
+ *
+ * @param remote The page as WordPress holds it.
+ * @param rendered The page as pterodoc would publish it.
+ * @param context The expected parent, slug, status and template.
+ */
+function diffPage(remote, rendered, context) {
+    const changed = [];
+    if (normalise(remote.title) !== normalise(rendered.title))
+        changed.push('title');
+    if (normalise(remote.content) !== normalise(rendered.content))
+        changed.push('content');
+    if (normalise(remote.excerpt) !== normalise(rendered.excerpt))
+        changed.push('excerpt');
+    if (remote.status !== context.status)
+        changed.push('status');
+    if (!context.isRoot && remote.menuOrder !== rendered.menuOrder)
+        changed.push('menu_order');
+    if ((remote.template ?? '') !== context.template)
+        changed.push('template');
+    if (remote.parent !== context.parentId)
+        changed.push('parent');
+    if (remote.slug !== context.slug)
+        changed.push('slug');
+    // Metadata is only compared where the site actually exposes the field, so a
+    // site without the SEO plugin does not report a difference on every run.
+    for (const [key, value] of Object.entries(rendered.meta)) {
+        if (remote.meta && key in remote.meta && normalise(remote.meta[key]) !== normalise(value)) {
+            changed.push(`meta.${key}`);
+        }
+    }
+    return changed;
+}
+/**
+ * Pages below a root that no rendered page accounts for.
+ *
+ * Deepest first, so a parent is never trashed before its children.
+ */
+function computePrune(index, rootId, keepIds) {
+    const childrenOf = new Map();
+    for (const page of index) {
+        const siblings = childrenOf.get(page.parent);
+        if (siblings)
+            siblings.push(page);
+        else
+            childrenOf.set(page.parent, [page]);
+    }
+    const owned = [];
+    const walk = (parentId, depth) => {
+        for (const page of childrenOf.get(parentId) ?? []) {
+            owned.push({ page, depth });
+            walk(page.id, depth + 1);
+        }
+    };
+    walk(rootId, 0);
+    return owned
+        .filter(({ page }) => !keepIds.has(page.id))
+        .sort((a, b) => b.depth - a.depth)
+        .map(({ page }) => page);
+}
+
+/**
+ * Where pages live on WordPress.
+ *
+ * URL policy belongs to the target: the renderer asks for a path and gets one
+ * back, without knowing whether the site nests pages, uses a subdirectory
+ * install, or publishes versions under their own segment.
+ */
+/**
+ * Segments that come before a page's own path.
+ *
+ * A version or a locale only earns a segment when it is not the primary one,
+ * so a single-version, single-locale site publishes exactly where it did
+ * before any of this existed.
+ */
+function prefixSegments(policy, context) {
+    const parts = [...policy.rootSegments, ...policy.baseSegments];
+    if (policy.primaryLocale !== undefined && context.locale !== policy.primaryLocale) {
+        parts.push(slugify(context.locale));
+    }
+    if (policy.primaryVersion !== undefined && context.versionName !== policy.primaryVersion) {
+        parts.push(slugify(context.versionName));
+    }
+    return parts;
+}
+/** The absolute site path of a page. */
+function hrefFor(policy, treePath, context) {
+    return joinPath([...prefixSegments(policy, context), ...segments(treePath)]);
+}
+/**
+ * The path pages hang from, and the slug of the page that owns the tree.
+ *
+ * Everything above the owned page is created once if missing and never edited;
+ * the owned page is the documentation root itself.
+ */
+function splitOwnership(policy) {
+    const all = [...policy.rootSegments, ...policy.baseSegments];
+    const rootSlug = all[all.length - 1];
+    if (rootSlug === undefined) {
+        throw new Error('There is nowhere to publish: the root path and the base are both empty.');
+    }
+    return { stubSegments: all.slice(0, -1), rootSlug };
+}
+
+/**
+ * Finding out whether the pterodoc WordPress plugin is installed.
+ *
+ * The plugin registers one option and exposes it over REST, so asking for the
+ * site's settings answers both questions at once: whether it is there, and what
+ * it is configured with. That second half matters, because the plugin styles a
+ * site by class prefix and a prefix that disagrees with the one pterodoc writes
+ * is a setup that looks broken for no visible reason.
+ */
+/**
+ * Ask a site whether the plugin is installed.
+ *
+ * Never throws: an unreachable or unauthorised site is a thing to report in a
+ * diagnostic, not a reason to fail one.
+ *
+ * @param client A client for the site.
+ */
+async function detectPlugin(client) {
+    try {
+        const { data } = await client.request('GET', '/settings');
+        const settings = data.pterodoc_settings;
+        if (!settings)
+            return { installed: false };
+        return {
+            installed: true,
+            ...(settings.classPrefix ? { classPrefix: settings.classPrefix } : {}),
+        };
+    }
+    catch (error) {
+        const status = error instanceof TargetError ? error.status : undefined;
+        if (status === 401 || status === 403) {
+            return {
+                installed: false,
+                unknown: 'these credentials may not read the site settings, so the plugin could not be checked',
+            };
+        }
+        return {
+            installed: false,
+            unknown: error instanceof Error ? error.message : 'the site could not be asked',
+        };
+    }
+}
+
+/** The WordPress target. */
+/** What WordPress can do. */
+const WORDPRESS_CAPABILITIES = {
+    // The navigation block lists children of a page id, so ids must exist first.
+    needsIdsBeforeRender: true,
+    supportsMedia: true,
+    supportsPrune: true,
+    supportsHierarchy: true,
+    supportsExcerpt: true,
+    supportsMeta: true,
+    supportsTemplates: true,
+    supportsDrafts: true,
+};
+/** Content a page holds between being created and being rendered. */
+const PLACEHOLDER = '<!-- wp:paragraph -->\n<p>Publishing…</p>\n<!-- /wp:paragraph -->';
+/** Create the WordPress target. */
+function createWordpressTarget(options, deps = {}) {
+    const log = deps.log ?? (() => { });
+    const { stubSegments, rootSlug } = splitOwnership(options.policy);
+    // The documentation root has no path of its own in the tree, so its slug is
+    // the last segment of the configured path rather than anything the model
+    // supplied.
+    const slugFor = (page) => page.path === '' ? rootSlug : page.slug;
+    const makeClient = (locale) => new WpClient({
+        baseUrl: options.url,
+        user: options.user,
+        appPassword: options.appPassword,
+        // A site with one language per subtree wants each subtree tagged.
+        lang: options.lang || (options.policy.primaryLocale && locale !== options.policy.primaryLocale ? locale : ''),
+        methodOverride: options.methodOverride,
+        retry: options.retry ?? DEFAULT_RETRY,
+        ...(deps.fetch ? { fetch: deps.fetch } : {}),
+        ...(deps.sleep ? { sleep: deps.sleep } : {}),
+        log,
+    });
+    return {
+        name: 'wordpress',
+        capabilities: WORDPRESS_CAPABILITIES,
+        rootPath: hrefFor(options.policy, '', { versionName: '', locale: options.policy.primaryLocale ?? '' }),
+        hrefFor(treePath, context) {
+            return hrefFor(options.policy, treePath, context);
+        },
+        async open(context) {
+            const client = makeClient(context.locale);
+            const dryRun = context.dryRun;
+            let index;
+            return {
+                async loadIndex() {
+                    index ??= await fetchPageIndex(client);
+                    return index;
+                },
+                async ensureRootParent() {
+                    const created = [];
+                    let parentId = 0;
+                    for (const slug of stubSegments) {
+                        if (parentId === null) {
+                            created.push({ path: `/${slug}/`, id: null });
+                            continue;
+                        }
+                        const existing = await findPage(client, parentId, slug, index, log);
+                        if (existing) {
+                            if (existing.status === 'trash') {
+                                throw new TargetError(`The page "/${slug}/" is in the trash. Restore it, or delete it permanently, and run again.`, { status: 409, method: 'GET', url: `/pages?slug=${slug}` });
+                            }
+                            parentId = existing.id;
+                            continue;
+                        }
+                        if (dryRun) {
+                            created.push({ path: `/${slug}/`, id: null });
+                            parentId = null;
+                            continue;
+                        }
+                        // A page created only so the documentation has a parent: it lists
+                        // what is below it and claims nothing else.
+                        const page = await createPage(client, {
+                            title: titleCase(slug),
+                            slug,
+                            parent: parentId,
+                            status: 'publish',
+                            content: PLACEHOLDER,
+                        });
+                        await updatePage(client, page.id, { content: renderNavigationStub(page.id) });
+                        created.push({ path: `/${slug}/`, id: page.id });
+                        parentId = page.id;
+                    }
+                    return { id: parentId, created };
+                },
+                async ensurePage(request) {
+                    const warnings = [];
+                    if (request.parentId === null)
+                        return { id: null, created: true, warnings };
+                    const slug = request.isRoot ? rootSlug : request.slug;
+                    const existing = await findPage(client, request.parentId, slug, index, log);
+                    if (existing) {
+                        if (existing.status === 'trash') {
+                            warnings.push(`${request.path || '(root)'} matches a page in the trash. It will be republished; restore or delete it permanently if that is not what you want.`);
+                        }
+                        return { id: existing.id, created: false, warnings };
+                    }
+                    if (dryRun)
+                        return { id: null, created: true, warnings };
+                    // Created as a draft: a placeholder must never appear in navigation.
+                    const created = await createPage(client, {
+                        title: request.title,
+                        slug,
+                        parent: request.parentId,
+                        status: 'draft',
+                        menu_order: request.menuOrder,
+                        content: PLACEHOLDER,
+                    });
+                    return { id: created.id, created: true, warnings };
+                },
+                async fetchPage(id) {
+                    return fetchPage(client, id);
+                },
+                diffPage(remote, rendered, parentId) {
+                    return diffPage(remote, rendered, {
+                        parentId,
+                        status: options.status,
+                        template: options.template,
+                        isRoot: rendered.path === '',
+                        slug: slugFor(rendered),
+                    });
+                },
+                async writePage(id, page, parentId) {
+                    const warnings = [];
+                    const body = {
+                        title: page.title,
+                        content: page.content,
+                        excerpt: page.excerpt,
+                        parent: parentId,
+                        slug: slugFor(page),
+                        status: options.status,
+                        menu_order: page.menuOrder,
+                        template: options.template,
+                    };
+                    if (Object.keys(page.meta).length > 0)
+                        body.meta = page.meta;
+                    try {
+                        await updatePage(client, id, body);
+                    }
+                    catch (error) {
+                        // A locked-down site may reject the metadata or the template. The
+                        // page itself matters more than either, so try again without them.
+                        const status = error.status;
+                        if (status === 400 && (body.meta || body.template)) {
+                            delete body.meta;
+                            delete body.template;
+                            warnings.push(`${page.path || '(root)'}: WordPress refused the template or the metadata, so the page was published without them.`);
+                            await updatePage(client, id, body);
+                        }
+                        else
+                            throw error;
+                    }
+                    return { warnings };
+                },
+                computePrune(pages, rootId, keepIds) {
+                    return computePrune(pages, rootId, keepIds);
+                },
+                async removePage(page) {
+                    await trashPage(client, page.id);
+                },
+                async loadMediaIndex() {
+                    return loadMediaIndex(client, options.mediaSlugPrefix);
+                },
+                async uploadMedia(upload) {
+                    return uploadMedia(client, upload, options.mediaSlugPrefix);
+                },
+                requestCount() {
+                    return client.requestCount;
+                },
+            };
+        },
+    };
+}
+
+export { DEFAULT_RETRY, PLACEHOLDER, WORDPRESS_CAPABILITIES, WpClient, createWordpressTarget, detectPlugin, hrefFor, prefixSegments, splitOwnership };
+//# sourceMappingURL=index.js.map