@morphemecms/cms 0.0.0-stage → 0.1.0

package/scripts/built-site.mjs

@@ -0,0 +1,99 @@
+// The built site served on this computer, with what the editor asks of GitHub (through the site's Worker) answered from
+// the files in this repository, for the scripts that open the editor in a browser (editor-screens.mjs,
+// app-screenshots.mjs): no sign-in, and nothing changes.
+import { createServer } from 'node:http';
+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { extname, join, posix } from 'node:path';
+
+const ROOT = `${process.cwd()}/`;
+
+// The built site, as the host serves it: a folder is its index.html.
+const types = {
+  '.html': 'text/html',
+  '.js': 'text/javascript',
+  '.css': 'text/css',
+  '.json': 'application/json',
+  '.svg': 'image/svg+xml',
+  '.jpg': 'image/jpeg',
+  '.png': 'image/png',
+  '.webp': 'image/webp',
+  '.webmanifest': 'application/manifest+json',
+};
+
+/** Serves the built site in `dist`; gives the server and its address. */
+export async function serveBuilt(dist) {
+  const server = createServer((req, res) => {
+    let path = join(dist, decodeURIComponent(new URL(req.url, 'http://x').pathname));
+    if (existsSync(path) && statSync(path).isDirectory()) path = join(path, 'index.html');
+    if (!existsSync(path)) return res.writeHead(404).end();
+    res.writeHead(200, { 'content-type': types[extname(path)] ?? 'application/octet-stream' }).end(readFileSync(path));
+  });
+  await new Promise((done) => server.listen(0, '127.0.0.1', done));
+  return { server, site: `http://127.0.0.1:${server.address().port}` };
+}
+
+// GitHub, as the site's Worker answers the editor (packages/editor/worker/editor-github.js), answered from the files on disk: who is
+// signed in (with write access), the commit the branch is at, the files in the folders and at the paths asked for, and
+// their texts (a file's blob id is "blob:<path>"). A question about anything else (the versions of a file, what was
+// deleted) gets no data, as for a file without history. `texts`: files with other texts than on disk, by their paths.
+const TEXT = /\.(md|mdx|jsonc?|ya?ml|txt|ts|js|mjs|css|astro|svg)$/;
+const listed = (path, texts) => ({ path, oid: `blob:${path}`, binary: !(path in texts) && !TEXT.test(path) });
+function filesIn(dir, depth, texts) {
+  if (!existsSync(join(ROOT, dir)) || !statSync(join(ROOT, dir)).isDirectory()) return [];
+  const files = readdirSync(join(ROOT, dir), { withFileTypes: true }).flatMap((d) => {
+    const path = posix.join(dir, d.name);
+    if (d.isDirectory()) return depth > 1 ? filesIn(path, depth - 1, texts) : [];
+    return [listed(path, texts)];
+  });
+  // The files given that are not on disk, in their folder.
+  const more = Object.keys(texts)
+    .filter((path) => posix.dirname(path) === dir && !files.some((f) => f.path === path))
+    .map((path) => listed(path, texts));
+  return [...files, ...more];
+}
+function answer(query, texts) {
+  const repository = { ref: { target: { oid: 'head' } }, viewerPermission: 'ADMIN' };
+  for (const [, key, path] of query.matchAll(/(f\d+): object\(expression: "[^:"]+:([^"]+)"\)/g)) {
+    const full = join(ROOT, path);
+    if (path in texts) repository[key] = { oid: `blob:${path}`, text: texts[path] };
+    else if (!existsSync(full) || statSync(full).isDirectory()) repository[key] = null;
+    else repository[key] = { oid: `blob:${path}`, text: TEXT.test(path) ? readFileSync(full, 'utf8') : null };
+  }
+  return { data: { viewer: { login: 'editor-screens', name: 'Anna Jensen' }, repository } };
+}
+function workerAnswer(route, body, texts) {
+  if (route === 'graphql') return answer(body.query ?? '', texts);
+  if (route === 'head') return { commit: 'head', tree: 'root' };
+  if (route === 'files')
+    return {
+      commit: 'head',
+      tree: 'root',
+      folders: (body.folders ?? []).map((dir) => filesIn(dir, body.depth ?? 3, texts)),
+      paths: (body.paths ?? []).map((path) =>
+        path in texts || (existsSync(join(ROOT, path)) && statSync(join(ROOT, path)).isFile()) ? listed(path, texts) : null,
+      ),
+    };
+  if (route === 'trees') return { trees: (body.at ?? []).map(() => null) };
+  if (route === 'texts') {
+    const paths = (body.oids ?? []).map((oid) => oid.replace(/^blob:/, ''));
+    return { texts: Object.fromEntries(paths.map((path) => [`blob:${path}`, texts[path] ?? readFileSync(join(ROOT, path), 'utf8')])) };
+  }
+  return null;
+}
+
+/** Opens the editor (/editor/) of the site at `site` in `page`, signed in, with GitHub answered from the files on disk
+ * (`texts`: some with other texts), and waits until it has loaded the site. */
+export async function openEditor(page, site, texts = {}) {
+  // R2 caching under "Avanceret": Cloudflare has the bucket (packages/editor/worker/r2-status.js).
+  await page.route(`${site}/api/r2?**`, (r) =>
+    r.fulfill({ contentType: 'application/json', body: JSON.stringify({ bound: false, bucket: 'ready' }) }),
+  );
+  await page.route(`${site}/api/github/**`, (r) => {
+    const route = new URL(r.request().url()).pathname.slice('/api/github/'.length);
+    const found = workerAnswer(route, JSON.parse(r.request().postData() || '{}'), texts);
+    return found ? r.fulfill({ contentType: 'application/json', body: JSON.stringify(found) }) : r.fulfill({ status: 404 });
+  });
+  await page.goto(`${site}/editor/`);
+  // The editor has loaded the site when the start screen has its links.
+  await page.waitForFunction(() => document.querySelectorAll('a[href^="#"]').length > 3, null, { timeout: 60_000 });
+}

package/scripts/check-consent.mjs

@@ -0,0 +1,113 @@
+// Checks the consent log (packages/editor/worker/consent.js): the Worker's own code with D1 played by Node's SQLite (node:sqlite) and
+// GitHub's answer about a sign-in played here. An answer from the site's own page is kept with the build's commit;
+// one from another site, with a wrong consent id or with a category the banner does not have is not; only someone
+// who can save the site reads, saves as CSV or deletes them; the hourly clean-up deletes those older than two years,
+// and without the binding nothing is kept. Nothing goes to GitHub or Cloudflare. Run: npm run check:worker
+import { DatabaseSync } from 'node:sqlite';
+import { workerModules } from './worker-modules.mjs';
+
+const [{ forgetOldConsents, handleConsent, KEEP_DAYS }] = await workerModules('consent.js');
+
+const problems = [];
+const expect = (ok, what) => ok || problems.push(what);
+
+/** D1, as much of it as the Worker uses, on an SQLite database in memory. */
+function d1() {
+  const db = new DatabaseSync(':memory:');
+  const statement = (sql, args = []) => ({
+    bind: (...values) => statement(sql, values),
+    run: async () => ({ meta: { changes: Number(db.prepare(sql).run(...args).changes) } }),
+    all: async () => ({ results: db.prepare(sql).all(...args) }),
+    first: async (column) => db.prepare(sql).get(...args)?.[column] ?? null,
+  });
+  return { db, prepare: (sql) => statement(sql), batch: async (list) => Promise.all(list.map((s) => s.run())) };
+}
+
+// GitHub: the token "editor" may save the site, "guest" may only read it.
+globalThis.fetch = async (url, init) => {
+  const token = init?.headers?.authorization?.replace('Bearer ', '');
+  if (!String(url).startsWith('https://api.github.com/')) return new Response(null, { status: 404 });
+  if (token === 'editor') return Response.json({ permissions: { pull: true, push: true } });
+  if (token === 'guest') return Response.json({ permissions: { pull: true, push: false } });
+  return new Response(null, { status: 401 });
+};
+
+const SITE = 'https://site.example';
+const COMMIT = 'abc123';
+const ASSETS = { fetch: async () => Response.json({ commit: COMMIT }) };
+const ID = '0f8fad5b-d9cb-469f-a165-70867728950e';
+const OTHER = '7c9e6679-7425-40de-944b-e07fc1f90ae7';
+
+const visitor = (body, origin = SITE) =>
+  new Request(`${SITE}/api/consent`, { method: 'POST', headers: { origin, 'content-type': 'text/plain' }, body: JSON.stringify(body) });
+const editor = (token, path = '', init = {}) =>
+  new Request(`${SITE}/api/consent${path}`, {
+    ...init,
+    headers: { cookie: `editor_session=${token}`, 'sec-fetch-site': 'same-origin', ...init.headers },
+  });
+
+const store = d1();
+const env = { CONSENT: store, ASSETS };
+
+// What a visitor sends.
+const sent = async (body, origin) => (await handleConsent(visitor(body, origin), env)).status;
+expect((await sent({ id: ID, choices: { statistics: true, marketing: false } })) === 204, 'et svar fra sitet blev ikke gemt');
+expect((await sent({ id: OTHER, choices: { statistics: false } })) === 204, 'et andet svar fra sitet blev ikke gemt');
+expect(
+  (await sent({ id: ID, choices: { statistics: true } }, 'https://other.example')) === 403,
+  'et svar fra en anden side blev taget imod',
+);
+expect((await sent({ id: 'not-an-id', choices: { statistics: true } })) === 400, 'et svar med et forkert id blev taget imod');
+expect((await sent({ id: ID, choices: { necessary: true } })) === 400, 'et svar om nødvendige cookies blev taget imod');
+expect((await sent({ id: ID, choices: { statistics: 'yes' } })) === 400, 'et svar, der ikke er ja eller nej, blev taget imod');
+expect((await sent({ id: ID, choices: {} })) === 400, 'et svar uden kategorier blev taget imod');
+const kept = store.db.prepare('SELECT * FROM consents ORDER BY row').all();
+expect(kept.length === 2, `der blev gemt ${kept.length} svar, ikke 2`);
+expect(kept[0]?.version === COMMIT, `svaret blev gemt med versionen ${kept[0]?.version}, ikke ${COMMIT}`);
+expect(
+  !JSON.stringify(kept).includes('origin') && Object.keys(kept[0] ?? {}).join() === 'row,id,at,choices,version',
+  'der blev gemt mere end svaret',
+);
+
+// Who reads them.
+expect((await handleConsent(editor('guest'), env)).status === 403, 'en, der ikke kan gemme sitet, kunne læse samtykkerne');
+expect((await handleConsent(editor('nobody'), env)).status === 403, 'en uden login kunne læse samtykkerne');
+const listed = await (await handleConsent(editor('editor'), env)).json();
+expect(listed.total === 2 && listed.entries.length === 2, `listen havde ${listed.entries?.length} af ${listed.total} svar, ikke 2`);
+expect(listed.entries.find((e) => e.id === ID)?.choices.statistics === true, 'listen gav ikke svaret, som det blev givet');
+const found = await (await handleConsent(editor('editor', `?q=${ID.slice(0, 8)}`), env)).json();
+expect(found.entries.length === 1 && found.entries[0].id === ID, 'søgningen på et id fandt ikke kun det id');
+
+// As a file.
+const file = await handleConsent(editor('editor', '?format=csv'), env);
+const lines = (await file.text()).trim().split('\r\n');
+expect(file.headers.get('content-disposition')?.startsWith('attachment'), 'CSV-filen hentes ikke som en fil');
+expect(lines[0] === '"id","at","statistics","marketing","version"', `CSV-filens første linje er ${lines[0]}`);
+expect(lines.length === 3, `CSV-filen har ${lines.length - 1} svar, ikke 2`);
+
+// Deleted, when a visitor asks.
+const removed = await handleConsent(editor('editor', '', { method: 'DELETE', body: JSON.stringify({ id: OTHER }) }), env);
+expect((await removed.json()).deleted === 1, 'svaret blev ikke slettet');
+expect(
+  (await handleConsent(editor('guest', '', { method: 'DELETE', body: JSON.stringify({ id: ID }) }), env)).status === 403,
+  'en, der ikke kan gemme sitet, kunne slette et svar',
+);
+
+// Those older than two years are deleted each hour; the others stay.
+const old = new Date(Date.now() - (KEEP_DAYS + 1) * 24 * 60 * 60 * 1000).toISOString();
+store.db.prepare('INSERT INTO consents (id, at, choices, version) VALUES (?, ?, ?, ?)').run(OTHER, old, '{"statistics":true}', COMMIT);
+await forgetOldConsents(env);
+const left = store.db.prepare('SELECT id FROM consents').all();
+expect(left.length === 1 && left[0].id === ID, `efter oprydningen er der ${left.length} svar, ikke kun det nye`);
+
+// Without the binding (a preview of a branch) nothing is kept, and the editor is told so.
+const none = { ASSETS };
+expect((await handleConsent(visitor({ id: ID, choices: { statistics: true } }), none)).status === 204, 'et svar uden D1 gav en fejl');
+expect((await handleConsent(editor('editor'), none)).status === 503, 'editoren fik ikke at vide, at der ikke gemmes samtykker');
+await forgetOldConsents(none);
+
+if (problems.length) {
+  console.error(`${problems.length} fejl i samtykkeloggen:\n${problems.join('\n')}`);
+  process.exit(1);
+}
+console.log('Samtykkeloggen virker: svarene gemmes med sitets version, kun editoren kan læse, hente og slette dem, og gamle slettes.');

package/scripts/check-content.mjs

@@ -0,0 +1,149 @@
+// Checks the site's content by the editor's own rules before it is built (prebuild, and
+// `npm run check:content`): every block in the text of the pages and the data files the editor writes
+// (checkSite.ts in the editor, packages/editor/). Stops with what is wrong, so a broken page never goes
+// out; the site that is out stays as it is; among it every theme (themeProblems, with the icon packs it brings) and
+// the system's icon packs and the choice of one (iconPackProblems), and so an icon the site or the editor draws that a
+// pack has none for, and the choice of motion scheme (motionProblems). Says too which of the site's texts the
+// editor has no translation of (@site/texts, a block's texts.ts); that does not stop the build. The editor's
+// code is TypeScript and TSX, so it is loaded with Vite, as the site's own build does.
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { join, relative } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { config } from '../site/siteConfig.mjs';
+import { createServer } from 'vite';
+import {
+  aliases,
+  blockProblems,
+  iconPackProblems,
+  motionProblems,
+  installedThemes,
+  themeFolder,
+  themePlugin,
+  themeProblems,
+} from '@morphemecms/cms/site/theme.mjs';
+import { appFoldersPlugin, appIds, appWorkersProblems } from '@morphemecms/cms/lib/apps.mjs';
+import { codePackageProblems, declarationProblems, ownerId } from '@morphemecms/cms/lib/licenses.mjs';
+import { notesOf, releaseProblems } from './releases.mjs';
+
+const root = `${process.cwd()}/`;
+// This package's own code, from the site's root (packages/editor/ or node_modules/@morphemecms/cms/).
+const own = relative(root, fileURLToPath(new URL('..', import.meta.url)));
+const { allowedCspProblems, appCspProblems } = await import('../site/siteCsp.mjs');
+const { appCookieProblems } = await import('../site/appCookies.mjs');
+
+// With the redirects' other two files (siteRedirects in src/site/config.ts): none may send on through another.
+const files = ['src/content', 'src/data']
+  .filter((dir) => existsSync(dir))
+  .flatMap((dir) =>
+    readdirSync(dir, { recursive: true, withFileTypes: true })
+      .filter((e) => e.isFile() && /\.(md|json)$/.test(e.name))
+      .map((e) => join(e.parentPath, e.name)),
+  )
+  .concat([config.siteRedirects.rules, config.siteRedirects.map])
+  .map((path) => ({ path, text: readFileSync(path, 'utf8') }));
+
+const server = await createServer({
+  configFile: false,
+  logLevel: 'error',
+  appType: 'custom',
+  // As in packages/editor/astro.mjs: the editor takes the site from src/site/ as @site, and the theme in use as @theme.
+  plugins: [themePlugin(), appFoldersPlugin(process.cwd())],
+  resolve: { alias: aliases() },
+  server: { middlewareMode: true, hmr: false, ws: false },
+});
+let result, missing;
+// The licences ("Licenser" under "Indstillinger", @morphemecms/cms/lib/licenses.mjs): each app's own and those of the
+// third-party work it brings (its defineApp), installed or not, as each theme's (themeProblems), with the packages they
+// declare, which they give what a package lacks for.
+const licenses = [];
+const declared = new Set();
+// The apps and themes of the app store, against what the catalog has released of them (releases.mjs).
+let catalog = {};
+const releases = [];
+try {
+  const { problemsOf, untranslated } = await server.ssrLoadModule('@morphemecms/cms/src/checkSite.ts');
+  result = problemsOf(files);
+  missing = untranslated();
+  const { config } = await server.ssrLoadModule('/src/site/config.ts');
+  if (config.store && existsSync(config.store.catalog)) catalog = JSON.parse(readFileSync(config.store.catalog, 'utf8'));
+  for (const id of appIds(root)) {
+    const app = (await server.ssrLoadModule(`/src/apps/${id}/index.ts`)).default;
+    for (const x of app.thirdParty ?? []) if (x?.package) declared.add(x.package);
+    if (app.version)
+      releases.push(
+        ...releaseProblems(catalog, {
+          kind: 'apps',
+          id,
+          folder: `src/apps/${id}`,
+          version: app.version,
+          notes: notesOf(app.releaseNotes),
+          where: `src/apps/${id}/index.ts`,
+        }),
+      );
+    licenses.push(
+      ...declarationProblems({
+        license: app.license,
+        thirdParty: app.thirdParty,
+        folder: join(root, 'src/apps', id),
+        where: `src/apps/${id}/index.ts`,
+        what: `appen ${app.label}`,
+        owner: ownerId('app', id),
+        root,
+      }),
+    );
+  }
+} finally {
+  await server.close();
+}
+
+for (const [locale, files] of Object.entries(missing)) {
+  for (const [file, texts] of Object.entries(files)) {
+    console.warn(`\nEditoren viser ${texts.length} af sitets tekster uoversat på ${locale}. Skriv dem i ${file}:\n`);
+    for (const text of texts) console.warn(`  ${JSON.stringify(text)}`);
+  }
+}
+
+// Every theme installed has what a theme must have (the standard theme all of it), so the owner can swap to any of them
+// on "Udseende".
+result.problems.push(...installedThemes().flatMap(themeProblems), ...iconPackProblems(), ...motionProblems());
+// Every block is drawn from its view, on the site and in the editor alike.
+result.problems.push(...blockProblems());
+// What each app loads from elsewhere is in its csp.json, so the pages' Content-Security-Policy lets it through.
+result.problems.push(...appCspProblems());
+// The addresses allowed in the editor after the policy stopped them are written as an app's csp.json.
+result.problems.push(...allowedCspProblems());
+// The cookies each app sets are in its cookies.json, for the cookie list in the cookie policy.
+result.problems.push(...appCookieProblems());
+// The Worker has the part of each app that has one (worker/apps.js).
+result.problems.push(...appWorkersProblems(root));
+// Every app and theme says what its licence is and what it brings (the icon packs say theirs, themeProblems), and
+// every package the site's and the editor's code imports, and those they depend on, has a licence the editor can show
+// under "Licenser" (more than the build ships, which stops on its own when something it ships has none).
+for (const theme of installedThemes())
+  for (const x of JSON.parse(readFileSync(`${themeFolder(theme)}/theme.json`, 'utf8')).thirdParty ?? [])
+    if (x?.package) declared.add(x.package);
+result.problems.push(...licenses, ...codePackageProblems(root, [join(own, 'src'), join(own, 'lib'), 'src'], declared));
+// What is changed in an app or a theme of the store since it was released has its next version, once, with its note.
+for (const theme of installedThemes()) {
+  const manifest = JSON.parse(readFileSync(`${themeFolder(theme)}/theme.json`, 'utf8'));
+  if (manifest.store === true)
+    releases.push(
+      ...releaseProblems(catalog, {
+        kind: 'themes',
+        id: theme,
+        folder: themeFolder(theme),
+        version: manifest.version,
+        notes: notesOf(manifest.releaseNotes, true),
+        where: `${themeFolder(theme)}/theme.json`,
+      }),
+    );
+}
+result.problems.push(...releases);
+
+if (result.problems.length) {
+  console.error(`\nIndholdet følger ikke editorens regler (${result.problems.length}):\n`);
+  for (const line of result.problems) console.error(`  ${line}`);
+  console.error('\nRet filerne (eller blokkens felter i src/apps/<app>/blocks/), så bygges sitet igen.\n');
+  process.exit(1);
+}
+console.log(`${result.files} filer og ${result.blocks} blokke følger editorens regler.`);

package/scripts/check-demo.mjs

@@ -0,0 +1,24 @@
+// The CMS's demo site (demo/, demo-site.mjs) still works with the CMS: its content checked by the editor's rules
+// (check-content.mjs), each file the same after a save in the editor (editor-roundtrip.mjs) and the site built, with the
+// standard theme, in a folder of its own. Run it when the CMS, the
+// standard theme or the demo has changed: npx morpheme check:demo.
+import { execFileSync } from 'node:child_process';
+import { rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { binOf } from './bin.mjs';
+import { demoSite } from './demo-site.mjs';
+
+const dir = demoSite();
+try {
+  const run = (args) => execFileSync(process.execPath, args, { cwd: dir, stdio: 'inherit' });
+  run([fileURLToPath(new URL('check-content.mjs', import.meta.url))]);
+  run([fileURLToPath(new URL('editor-roundtrip.mjs', import.meta.url))]);
+  run([binOf('astro'), 'build', '--outDir', join(dir, 'dist')]);
+  console.log('\nDemoen virker med CMS’et.');
+} catch {
+  console.error('\nDemoen virker ikke med CMS’et (se ovenfor).');
+  process.exitCode = 1;
+} finally {
+  rmSync(dir, { recursive: true, force: true });
+}

package/scripts/check-themes.mjs

@@ -0,0 +1,45 @@
+// Every theme installed (src/themes/) works as the site's theme, so the owner can swap to any of them: with each of the
+// others than the one in use (src/data/theme.json), the editor's check of the content (check-content.mjs), the site
+// built (in a folder of its own, so dist/ stays the theme in use's) and every screen of the editor opened
+// (editor-screens.mjs). The theme in use is checked by the steps before (npm run build, npm run check:screens).
+// src/data/theme.json is the theme being checked while it runs, and as it was after.
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { activeTheme, installedThemes } from '@morphemecms/cms/site/theme.mjs';
+import { binOf } from './bin.mjs';
+
+const ROOT = `${process.cwd()}/`;
+const file = join(ROOT, 'src/data/theme.json');
+const before = readFileSync(file, 'utf8');
+const others = installedThemes().filter((t) => t !== activeTheme());
+const run = (cmd, args) => execFileSync(cmd, args, { cwd: ROOT, stdio: 'inherit' });
+
+const failed = [];
+try {
+  for (const theme of others) {
+    console.log(`\nTemaet ${theme}:`);
+    writeFileSync(file, `${JSON.stringify({ theme }, null, 2)}\n`);
+    const out = mkdtempSync(join(tmpdir(), `theme-${theme}-`));
+    try {
+      run(process.execPath, [fileURLToPath(new URL('check-content.mjs', import.meta.url))]);
+      run(process.execPath, [binOf('astro'), 'build', '--outDir', out]);
+      run(process.execPath, [fileURLToPath(new URL('editor-screens.mjs', import.meta.url)), out]);
+    } catch {
+      failed.push(theme);
+    } finally {
+      rmSync(out, { recursive: true, force: true });
+    }
+  }
+} finally {
+  writeFileSync(file, before);
+}
+
+if (failed.length) {
+  console.error(`\nDet virker ikke med ${failed.length === 1 ? 'temaet' : 'temaerne'} ${failed.join(', ')} (se ovenfor).`);
+  process.exit(1);
+}
+if (!others.length) console.log('Der er ingen andre temaer installeret i src/themes/.');
+else console.log(`\n${others.length === 1 ? 'Temaet' : 'Temaerne'} ${others.join(', ')} virker også som sitets tema.`);