@openbooknetwork/kitap 0.0.0-stage → 0.3.0

package/src/upload.mjs

@@ -0,0 +1,114 @@
+// kitap upload: puts the images and the engine's objects each story links to into the asset store, then the story's
+// hash tree, so kitap build --strict finds them with one request.
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { storyAssets } from './build.mjs';
+import { checkOriginal, contentType, downloadOriginal, sha256Of } from './image.mjs';
+import { typeOf } from './routes.mjs';
+import { storeReader, storeWriter, missingKeys, missingIn, pool, describeMissing } from './store.mjs';
+import { mb, plural, progressLine } from './util.mjs';
+
+const SIZES = { large: 1280, small: 320 };
+
+// The WebP sizes of an original, fitted inside 1280 and 320 px squares. A raster original is never enlarged, an SVG is
+// drawn at each size, and an animated GIF or WebP keeps its frames.
+export async function webpSizes(bytes, ext) {
+  const sharp = (await import('sharp')).default;
+  const svg = ext === '.svg', animated = ext === '.gif' || ext === '.webp';
+  // lines and lettering of maps and diagrams stay sharp at the higher quality
+  const quality = svg || ext === '.png' ? 90 : 80;
+  const meta = svg ? await sharp(bytes).metadata() : null;
+  const out = {};
+  for (const [name, n] of Object.entries(SIZES)) {
+    // an SVG is drawn at the density that makes its longer side n pixels, so the size is drawn, not scaled up
+    const density = svg ? Math.min(100000, Math.max(1, 72 * n / Math.max(meta.width, meta.height))) : undefined;
+    let img = sharp(bytes, { animated, ...(density ? { density } : {}) });
+    if (!animated) img = img.rotate(); // upright by the EXIF orientation, which the WebP does not carry
+    out[name] = await img.resize(n, n, { fit: 'inside', withoutEnlargement: !svg }).webp({ quality }).toBuffer();
+  }
+  return out;
+}
+
+// The original of an image: the file on this machine, else the copy in the store, else the download from its url.
+async function originalOf(im, reader, inStore) {
+  const ext = path.extname(im.keys.original);
+  let bytes = null, from;
+  if (im.file) { bytes = fs.readFileSync(im.file); from = path.join(im.dir, path.basename(im.file)); }
+  else if (inStore && (bytes = await reader.get(im.keys.original))) from = `the store's ${im.keys.original}`;
+  else if (im.url) { bytes = (await downloadOriginal(im.url)).bytes; from = im.url; }
+  else throw new Error(`${im.dir}: the original is not on this machine, in the store or at a url in image.yaml, add it to the folder`);
+  if (sha256Of(bytes) !== im.sha256) throw new Error(`${im.dir}: ${from} does not match the sha256 of image.yaml${im.file ? ', kitap rehash writes the new one' : ''}`);
+  await checkOriginal(bytes, ext, from);
+  return bytes;
+}
+
+// Puts the objects of story, else of every story, into the store of site.yaml's assets: the originals and their WebP
+// sizes, the engine's objects, and last each story's hash tree with latest.json naming it. dryRun lists what would go
+// and writes nothing. Returns the keys uploaded, or with dryRun the keys it would upload.
+export async function upload({ root = process.cwd(), story = null, dryRun = false, log = console.log, progress = progressLine, cache = path.join(root, '.cache', 'kitap'), engine = null, wait, ...options } = {}) {
+  const { assets, stories, engine: e } = storyAssets({ root, story, log, engine, cache });
+  const reader = storeReader(assets.url, { wait }), writer = dryRun ? null : storeWriter(assets, { wait });
+  // an upload lists the bucket once, a thousand keys a request, and a dry run asks through the public URL alone
+  const held = writer ? new Set((await writer.list()).map(o => o.key)) : null;
+  if (held) log(`the store holds ${held.size.toLocaleString('en')} objects`);
+  let cur = null;
+  const seen = new Set(), done = new Set(), planned = new Set(), present = new Set(), sum = { images: 0, imageBytes: 0, tiles: 0, tileBytes: 0 };
+  const put = async (key, body, type, cacheControl) => {
+    if (done.has(key)) return 0;
+    await writer.put(key, body, type, cacheControl);
+    done.add(key);
+    cur.n++; cur.bytes += body.length;
+    progress({ ...cur, done: false });
+    return body.length;
+  };
+  for (const s of stories) {
+    const { missing: list, complete } = held ? missingIn(held, s.tree) : await missingKeys(reader, s.tree, { headAll: true });
+    const missing = new Set(list.filter(k => !done.has(k)));
+    for (const k of Object.values(s.tree.dirs).flatMap(x => x.keys)) if (!missing.has(k) && !done.has(k)) present.add(k);
+    if (complete) { log(`${s.story}: the store holds every object`); continue; }
+    const images = s.images.filter(im => Object.values(im.keys).some(k => missing.has(k)) && !seen.has(im.keys.original));
+    for (const im of images) seen.add(im.keys.original);
+    const objects = Object.values(s.groups).flat().filter(k => missing.has(k));
+    if (dryRun) {
+      if (missing.size) log(`${s.story}: would upload ${describeMissing(s.tree, [...missing], e.describeObjects)}`);
+      for (const im of images) if (!im.file && !im.url && missing.has(im.keys.original)) throw new Error(`${im.dir}: the original is not on this machine, in the store or at a url in image.yaml, add it to the folder`);
+      sum.images += images.length; sum.tiles += objects.length;
+      for (const k of missing) planned.add(k);
+      continue;
+    }
+    cur = { what: s.story, n: 0, total: missing.size, bytes: 0 };
+    // each image fetches its original once, makes the sizes it lacks and puts its objects, eight images at a time
+    await pool(images, 8, async im => {
+      const original = await originalOf(im, reader, !missing.has(im.keys.original));
+      const ext = path.extname(im.keys.original);
+      let bytes = missing.has(im.keys.original) ? await put(im.keys.original, original, contentType(ext)) : 0;
+      if (missing.has(im.keys.large) || missing.has(im.keys.small)) {
+        const sizes = await webpSizes(original, ext);
+        for (const name of Object.keys(SIZES)) if (missing.has(im.keys[name])) bytes += await put(im.keys[name], sizes[name], 'image/webp');
+      }
+      sum.images++; sum.imageBytes += bytes;
+    });
+    // the engine makes its missing objects into a folder of their own, and each goes up under its key
+    if (objects.length) {
+      const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kitap-objects-'));
+      try {
+        await e.make({ stories: [{ ...s, groups: Object.fromEntries(Object.entries(s.groups).map(([g, keys]) => [g, keys.filter(k => missing.has(k))])) }], cache, out: dir, log, options });
+        await pool(objects, 8, async k => { sum.tileBytes += await put(k, fs.readFileSync(path.join(dir, k)), typeOf(k)); sum.tiles++; });
+      } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+    }
+    progress({ ...cur, done: true });
+    // the tree goes last, once every object it names is in the store, and latest.json names it
+    const tree = Buffer.from(JSON.stringify(s.tree));
+    await writer.put(`meta/${s.story}/${s.tree.root}.json`, tree, 'application/json');
+    await writer.put(`meta/${s.story}/latest.json`, Buffer.from(JSON.stringify({ format: s.tree.format, story: s.story, root: s.tree.root })), 'application/json', 'no-cache');
+    log(`${s.story}: uploaded, its tree is meta/${s.story}/${s.tree.root}.json`);
+  }
+  const inStore = [...present].filter(k => !done.has(k)).length;
+  // the engine's objects in its own words, such as "198 elevation tiles", else by count
+  const others = [...(dryRun ? planned : done)].filter(k => !k.startsWith('images/') && !k.startsWith('meta/'));
+  const described = e.describeObjects ? e.describeObjects(others) : plural(others.length, 'other object');
+  log(dryRun ? `would upload ${plural(sum.images, 'image')} and ${described}, ${inStore.toLocaleString('en')} objects are in the store`
+    : `uploaded ${plural(sum.images, 'image')} (${mb(sum.imageBytes)}) and ${described} (${mb(sum.tileBytes)}), ${inStore.toLocaleString('en')} were in the store`);
+  return [...(dryRun ? planned : done)];
+}

package/src/util.mjs

@@ -0,0 +1,19 @@
+// Small helpers the build, harita zones and harita image share.
+import fs from 'fs';
+import path from 'path';
+import crypto from 'crypto';
+
+export const hash = value => crypto.createHash('sha256').update(typeof value === 'string' ? value : JSON.stringify(value)).digest('hex');
+export const listDirs = p => fs.existsSync(p) ? fs.readdirSync(p).filter(d => fs.statSync(path.join(p, d)).isDirectory()).sort() : [];
+// a folder's id: its name without the number that orders it
+export const idOf = name => name.replace(/^\d+-/, '');
+export const boxesMeet = (a, b) => a[0] <= b[2] && b[0] <= a[2] && a[1] <= b[3] && b[1] <= a[3];
+export const mb = bytes => `${(bytes / 1e6).toFixed(1)} MB`;
+export const plural = (n, word) => `${n.toLocaleString('en')} ${word}${n === 1 ? '' : 's'}`;
+// Progress of an upload: on a terminal one line rewritten in place, elsewhere a line every thousand objects. what names
+// the story or the folder going up, and done clears the line once its summary is about to print.
+export const progressLine = ({ what, n, total, bytes, done }) => {
+  const line = `${what}: ${n.toLocaleString('en')} of ${total.toLocaleString('en')} objects, ${mb(bytes)}`;
+  if (process.stdout.isTTY) process.stdout.write(`\r\x1b[K${done ? '' : line}`);
+  else if (!done && n % 1000 === 0) console.log(line);
+};

package/src/vendor.mjs

@@ -0,0 +1,67 @@
+// The scripts and fonts the pages load, copied from the packages of kitap and of the engine into dist/vendor/, so a
+// page loads nothing from another host. A package's folder names its version, so a browser can keep its files for good.
+import fs from 'fs';
+import path from 'path';
+import { createRequire } from 'module';
+import { fileURLToPath } from 'url';
+
+const require = createRequire(import.meta.url);
+const PKG = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+// a package's folder and version, found up from its main file, since some packages export no package.json. resolve is
+// the require of the package that depends on it: kitap's for the fonts, the engine's for its libraries.
+function pkg(name, resolve = require.resolve) {
+  for (let dir = path.dirname(resolve(name)); dir !== path.dirname(dir); dir = path.dirname(dir)) {
+    const file = path.join(dir, 'package.json');
+    if (fs.existsSync(file)) { const meta = JSON.parse(fs.readFileSync(file, 'utf8')); if (meta.name === name && meta.version) return { dir, version: meta.version }; }
+  }
+  throw new Error(`the package ${name} is missing, reinstall the engine`);
+}
+const folder = (name, resolve) => `${name.split('/').pop()}@${pkg(name, resolve).version}`;
+
+// the subsets the pages need: Latin, and Latin Extended for Turkish and the other languages written in Latin letters
+const FONTS = [
+  { family: 'Fraunces', name: '@fontsource-variable/fraunces', css: 'opsz.css', files: /^fraunces-latin(-ext)?-opsz-normal$/ },
+  { family: 'Public Sans', name: '@fontsource-variable/public-sans', css: 'index.css', files: /^public-sans-latin(-ext)?-wght-normal$/ },
+];
+const licence = dir => ['LICENSE', 'LICENSE.txt', 'LICENSE.md'].map(f => path.join(dir, f)).find(f => fs.existsSync(f));
+
+// Writes dist/vendor/: the engine's libraries, each [package, file] of engine.client.libraries resolved from the
+// engine's own package, the fonts, and whatever the engine's vendor hook copies. Returns the libraries' paths under
+// dist/ in order, for the story page's <script> tags.
+export function writeVendor(dist, engine = null) {
+  const out = path.join(dist, 'vendor');
+  fs.rmSync(out, { recursive: true, force: true });
+  const copy = (from, to) => { fs.mkdirSync(path.dirname(to), { recursive: true }); fs.copyFileSync(from, to); };
+  const paths = [];
+  const resolve = engine?.dir ? createRequire(path.join(engine.dir, 'package.json')).resolve : require.resolve;
+  // a library is [package, file] for a script the page loads, or [package, folder] for a folder a module engine imports from
+  const imports = {};
+  const vendored = (name, file) => {
+    const { dir } = pkg(name, resolve), to = path.join(out, folder(name, resolve)), from = path.join(dir, file), base = path.basename(file.replace(/\/$/, ''));
+    const isDir = fs.statSync(from).isDirectory();
+    if (isDir) fs.cpSync(from, path.join(to, base), { recursive: true }); else copy(from, path.join(to, base));
+    const lic = licence(dir) ?? licence(path.join(dir, path.dirname(file)));
+    if (lic) copy(lic, path.join(to, path.basename(lic)));
+    return `vendor/${folder(name, resolve)}/${base}${isDir ? '/' : ''}`;
+  };
+  for (const [name, file] of engine?.client?.libraries ?? []) paths.push(vendored(name, file));
+  // an import is [package, file], or [package, folder, file] when the file imports its siblings
+  for (const [spec, [name, file, entry]] of Object.entries(engine?.client?.imports ?? {})) imports[spec] = entry ? `${vendored(name, file)}${entry}` : vendored(name, file);
+  // the packages' @font-face rules for the subsets above, under the family names the pages use
+  const rules = [];
+  for (const f of FONTS) {
+    const { dir } = pkg(f.name), to = folder(f.name);
+    for (const block of fs.readFileSync(path.join(dir, f.css), 'utf8').split('/* ').slice(1)) {
+      const id = block.slice(0, block.indexOf(' */'));
+      if (!f.files.test(id)) continue;
+      copy(path.join(dir, 'files', `${id}.woff2`), path.join(out, to, `${id}.woff2`));
+      rules.push(block.slice(block.indexOf('@font-face')).trim()
+        .replace(/font-family: '[^']*'/, `font-family: '${f.family}'`).replace(/url\(\.\/files\//, `url(${to}/`));
+    }
+    copy(licence(dir), path.join(out, to, 'LICENSE'));
+  }
+  fs.writeFileSync(path.join(out, 'fonts.css'), rules.join('\n') + '\n');
+  engine?.vendor?.(out);
+  return { paths: paths.filter(p => !p.endsWith('/')), imports };
+}

package/src/worker.mjs

@@ -0,0 +1,47 @@
+// The Worker that serves the site from the pages bucket. A content project deploys `export default site()` with the
+// bucket bound as SITE. It answers paths as harita dev does, see routes.mjs, and keeps each answer in the data
+// centre's cache for the ten minutes a reader's browser keeps it.
+import { resolve } from './routes.mjs';
+
+const CACHE = 'public, max-age=600';
+// a folder of dist/vendor/ names its package's version, so its files never change
+const cacheFor = key => /^vendor\/[^/]+@[^/]+\//.test(key) ? 'public, max-age=31536000, immutable' : CACHE;
+
+// The answer to a request's URL from the bucket, with the object's Content-Type and ETag.
+async function answer(bucket, url) {
+  let path;
+  try { path = decodeURIComponent(url.pathname); } catch { path = url.pathname; }
+  // the object a path names is read once: the check and the answer share it
+  const objects = new Map();
+  const exists = async key => { if (!objects.has(key)) objects.set(key, await bucket.get(key)); return Boolean(objects.get(key)); };
+  const redirect = async p => (await (await bucket.get('redirects.json'))?.json())?.[p] ?? null;
+  const r = await resolve(path, exists, { redirect, query: url.search });
+  if (r.location) return new Response(null, { status: 301, headers: { location: r.location, 'cache-control': CACHE } });
+  const obj = objects.get(r.key) ?? await bucket.get(r.key);
+  if (!obj) return new Response('not found', { status: 404, headers: { 'content-type': 'text/plain; charset=utf-8', 'cache-control': CACHE } });
+  return new Response(obj.body, { status: r.status, headers: { 'content-type': obj.httpMetadata?.contentType ?? 'application/octet-stream', etag: obj.httpEtag, 'cache-control': cacheFor(r.key) } });
+}
+
+// A 304 for a page the browser holds, else the answer, without its body for HEAD.
+function reply(request, res) {
+  const etag = res.headers.get('etag'), given = request.headers.get('if-none-match');
+  if (res.status === 200 && etag && given && given.split(',').map(s => s.trim()).some(t => t === etag || t === '*'))
+    return new Response(null, { status: 304, headers: { etag, 'cache-control': res.headers.get('cache-control') } });
+  return request.method === 'HEAD' ? new Response(null, res) : res;
+}
+
+// The Worker: fetch(request, env, ctx) with env.SITE the bucket. The data centre's cache holds each URL's answer.
+export function site() {
+  return {
+    async fetch(request, env, ctx) {
+      if (request.method !== 'GET' && request.method !== 'HEAD') return new Response('method not allowed', { status: 405, headers: { allow: 'GET, HEAD' } });
+      const url = new URL(request.url), cache = globalThis.caches?.default ?? null, cacheKey = new Request(url.href);
+      let res = cache ? await cache.match(cacheKey) : null;
+      if (!res) {
+        res = await answer(env.SITE, url);
+        if (cache) { const keep = cache.put(cacheKey, res.clone()); ctx?.waitUntil ? ctx.waitUntil(keep) : await keep; }
+      }
+      return reply(request, res);
+    },
+  };
+}