create-shapio 0.4.0 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-shapio",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Scaffold a new Shapio project (npx create-shapio my-cms) or a site starter (--site astro|next|sveltekit)",
   "license": "Apache-2.0",
   "homepage": "https://shapio.dev",

package/templates/astro/package.json

@@ -19,8 +19,8 @@
     "typecheck": "tsc -p tsconfig.json"
   },
   "dependencies": {
-    "@shapio/client": "^0.4.0",
-    "@shapio/visual": "^0.4.0",
+    "@shapio/client": "^0.5.0",
+    "@shapio/visual": "^0.5.0",
     "astro": "7.3.5"
   },
   "devDependencies": {

package/templates/astro/scripts/smokeHttp.ts

@@ -1,4 +1,6 @@
 import { createHmac } from 'node:crypto';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
 import { parseArgs } from 'node:util';
 
 /**
@@ -13,6 +15,10 @@ import { parseArgs } from 'node:util';
  * `--url` is the running site's origin (default: SMOKE_URL, then http://localhost:4321). `--revalidate` (the
  * Next.js starter) also checks its on-demand revalidation route: an unsigned POST to /api/revalidate is refused,
  * and an event signed with SHAPIO_WEBHOOK_SECRET is answered 200 with the paths it revalidated.
+ *
+ * `--save <dir>` also writes each checked page's rendered markup to `<dir>` (scripts and the framework's own
+ * asset links removed: they name build-specific chunks), so two builds of a site can be compared with
+ * `diff -r`, e.g. the Next.js starter reading over HTTP and in process.
  */
 const DRAFT_TITLE = 'Coming soon: our winter projects';
 const ARTICLE = 'modelling-without-a-deploy';
@@ -85,6 +91,18 @@ const fetchPage = async (origin: string, path: string) => {
   return { status: response.status, html: await response.text() };
 };
 
+/** The page as rendered: without scripts and framework asset links, which differ from build to build. */
+const renderedMarkup = (html: string) =>
+  html
+    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/g, '')
+    .replace(/<link\b[^>]*\/_next\/[^>]*>/g, '')
+    .replace(/<meta\b[^>]*name="next-size-adjust"[^>]*>/g, '');
+
+const savePage = async (directory: string, path: string, html: string) => {
+  const name = path.replace(/^\/|\/$/g, '').replace(/\//g, '__') || 'index';
+  await writeFile(join(directory, `${name}.html`), `${renderedMarkup(html)}\n`);
+};
+
 const REVALIDATE_PATH = '/api/revalidate/';
 
 const postEvent = (origin: string, body: string, headers: Record<string, string>) =>
@@ -137,12 +155,22 @@ const checkRevalidation = async (origin: string) => {
 
 const main = async () => {
   const { values } = parseArgs({
-    options: { url: { type: 'string' }, revalidate: { type: 'boolean', default: false } },
+    options: {
+      url: { type: 'string' },
+      revalidate: { type: 'boolean', default: false },
+      save: { type: 'string' },
+    },
   });
   const origin = values.url ?? process.env.SMOKE_URL ?? 'http://localhost:4321';
+  if (values.save) {
+    await mkdir(values.save, { recursive: true });
+  }
   for (const page of PAGES) {
     const { status, html } = await fetchPage(origin, page.path);
     check(status === 200, `${page.path}: HTTP ${status}`);
+    if (values.save) {
+      await savePage(values.save, page.path, html);
+    }
     for (const [needle, label] of page.expect) {
       check(html.includes(needle), `${page.path}: ${label}`);
     }

package/templates/next/.env.example

@@ -15,3 +15,7 @@ NEXT_TELEMETRY_DISABLED=1
 # NEXT_PUBLIC_SHAPIO_URL=https://cms.example.com
 # Optional: this site's public origin; pages then carry a canonical URL and og:url.
 # SITE_URL=https://www.example.com
+# Optional: read Shapio's delivery API in this server process instead of over HTTP (@shapio/local), from
+# Shapio's own database. A read-only database role is enough. Previews still use SHAPIO_URL.
+# SHAPIO_MODE=in-process
+# DATABASE_URL=postgres://shapio_site:password@localhost:5432/shapio

package/templates/next/README.md

@@ -22,16 +22,17 @@ npm test            # unit tests (path mapping, signature check)
 `next.config.ts` pins the latest one when the build starts); `SHAPIO_SITE` picks the site on a multi-site
 instance. Next.js telemetry is off: `NEXT_TELEMETRY_DISABLED=1` in the npm scripts and `.env.example`.
 
-| File                         | Role                                                                |
-| ---------------------------- | ------------------------------------------------------------------- |
-| `next.config.ts`             | pins the snapshot; `/` redirects to `/en/`; `frame-ancestors`       |
-| `src/lib/shapio.ts`          | reads the delivery API with `@shapio/client` at the pinned snapshot |
-| `src/app/[locale]/…`         | the layout (site settings), pages, articles and the colophon        |
-| `src/components/…`           | sections, articles and responsive images                            |
-| `src/app/preview/`           | draft preview with visual editing (`@shapio/visual`)                |
-| `src/app/api/revalidate/`    | the webhook target: on-demand revalidation of changed pages         |
-| `src/lib/revalidation*.ts`   | the snapshot diff and which pages a change touches                  |
-| `shapio/`, `scripts/seed.ts` | the models and the seed                                             |
+| File                         | Role                                                               |
+| ---------------------------- | ------------------------------------------------------------------ |
+| `next.config.ts`             | pins the snapshot; `/` redirects to `/en/`; `frame-ancestors`      |
+| `src/lib/shapio.ts`          | reads the delivery API at the pinned snapshot                      |
+| `src/lib/shapioClient.ts`    | over HTTP (`@shapio/client`) or in process (`@shapio/local`)       |
+| `src/app/[locale]/…`         | the layout (site settings), pages, articles and the colophon       |
+| `src/components/…`           | sections, articles and responsive images                           |
+| `src/app/preview/`           | draft preview with visual editing (`@shapio/visual`)               |
+| `src/app/api/revalidate/`    | the webhook target: on-demand revalidation of changed pages        |
+| `src/lib/revalidation*.ts`   | the snapshot diff, and which pages and cache tags a change touches |
+| `shapio/`, `scripts/seed.ts` | the models and the seed                                            |
 
 Preview: the seed creates a deployment connection named **Preview** that opens drafts at
 `http://localhost:3000/preview/` (set `SITE_URL` when seeding for another address). Add that origin to Shapio's
@@ -39,6 +40,18 @@ Preview: the seed creates a deployment connection named **Preview** that opens d
 re-render the draft ([visual editing](https://github.com/mybrokengnome/shapio/blob/main/documentation/visual-editing.md)). `NEXT_PUBLIC_SHAPIO_URL` sets the Shapio URL the browser
 calls, when it differs from `SHAPIO_URL`.
 
+## Reading in process
+
+By default the site reads Shapio's delivery API over HTTP. With `SHAPIO_MODE=in-process` and `DATABASE_URL` set
+to Shapio's own database (PostgreSQL or MySQL), it reads the same API as function calls in the Next.js server
+process (`@shapio/local`), with no HTTP hop: the same delivery token, the same answers, the same pages.
+`next.config.ts` lists `@shapio/local` in `serverExternalPackages`, so each server process opens one database
+pool. Previews and the browser still use `SHAPIO_URL`, and the Shapio server keeps running (it applies schema
+changes and sends the revalidation webhook). Install the same `@shapio/local` version as the server's release;
+the scaffolded `package.json` pins it. Reads in process do not go through `fetch`, so the cache tags below do
+not apply to them; `revalidatePath` still refreshes the pages. Guide:
+[in-process delivery](https://github.com/mybrokengnome/shapio/blob/main/documentation/in-process.md).
+
 ## Incremental rebuilds (on-demand revalidation)
 
 With `npm run start`, a publish refreshes only the pages it changed, without a rebuild:
@@ -55,7 +68,11 @@ With `npm run start`, a publish refreshes only the pages it changed, without a r
    changed between the snapshot the site shows and the current one (`/api/snapshots/changes`), and calls
    `revalidatePath` for those pages: an article's page and its locale's list, a page by its slug (`home` is
    `/en/`), the list and every article page for an author, every page for the `siteSettings` singleton or a
-   schema change. A changed slug refreshes the old and the new address.
+   schema change. A changed slug refreshes the old and the new address. Reads are tagged by `@shapio/client` with
+   the site, the model and the entry, so the route also calls `revalidateTag` for each changed model and entry;
+   `site.updated` and a schema change expire the site tag, which every read carries. Content reads are pinned to
+   a snapshot, so they sit in Next's data cache (`force-cache`) regardless; the client sets no cache mode, so
+   `site.get()` stays fresh on every build (the data cache outlives `next build`).
 3. The site moves to the new snapshot: revalidated pages re-render at it, and the snapshot is kept in
    `.next/shapio-revalidate.json` so a restart carries on from it. `next build` empties `.next/`, so a new build
    starts again from its own snapshot.

package/templates/next/next.config.ts

@@ -1,6 +1,6 @@
-import { createClient } from '@shapio/client';
 import type { NextConfig } from 'next';
 import { PHASE_PRODUCTION_BUILD } from 'next/constants.js';
+import { createShapioClient } from './src/lib/shapioClient';
 
 /**
  * Pins one publication snapshot for the whole `next build`: Next renders pages in several worker processes, so
@@ -13,17 +13,15 @@ const pinSnapshot = async (): Promise<string> => {
   if (process.env.SHAPIO_SNAPSHOT) {
     return process.env.SHAPIO_SNAPSHOT;
   }
-  const token = process.env.SHAPIO_DELIVERY_TOKEN;
-  if (!token) {
-    throw new Error('Set SHAPIO_DELIVERY_TOKEN to build the site; `npm run seed` writes one to .env');
+  // Over HTTP or in process, as the pages read (SHAPIO_MODE).
+  const client = createShapioClient();
+  try {
+    const { snapshot } = await client.snapshots.current();
+    return String(snapshot);
+  } finally {
+    // An in-process client holds database connections; this process has no further use for them.
+    await client.close?.();
   }
-  const client = createClient({
-    baseUrl: process.env.SHAPIO_URL || 'http://localhost:4300',
-    token,
-    site: process.env.SHAPIO_SITE || undefined,
-  });
-  const { snapshot } = await client.snapshots.current();
-  return String(snapshot);
 };
 
 const originOf = (url: string) => {
@@ -47,6 +45,9 @@ const frameAncestors = () => {
 
 const nextConfig = async (phase: string): Promise<NextConfig> => ({
   trailingSlash: true,
+  // SHAPIO_MODE=in-process: loaded from node_modules once per server process (one database pool), not
+  // bundled into every route.
+  serverExternalPackages: ['@shapio/local'],
   headers: async () => [
     { source: '/:path*', headers: [{ key: 'Content-Security-Policy', value: frameAncestors() }] },
   ],

package/templates/next/package.json

@@ -17,8 +17,9 @@
     "test": "vitest run"
   },
   "dependencies": {
-    "@shapio/client": "^0.4.0",
-    "@shapio/visual": "^0.4.0",
+    "@shapio/client": "^0.5.0",
+    "@shapio/local": "0.5.0",
+    "@shapio/visual": "^0.5.0",
     "next": "16.3.8",
     "react": "19.3.0",
     "react-dom": "19.3.0"

package/templates/next/scripts/smokeHttp.ts

@@ -1,4 +1,6 @@
 import { createHmac } from 'node:crypto';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
 import { parseArgs } from 'node:util';
 
 /**
@@ -13,6 +15,10 @@ import { parseArgs } from 'node:util';
  * `--url` is the running site's origin (default: SMOKE_URL, then http://localhost:4321). `--revalidate` (the
  * Next.js starter) also checks its on-demand revalidation route: an unsigned POST to /api/revalidate is refused,
  * and an event signed with SHAPIO_WEBHOOK_SECRET is answered 200 with the paths it revalidated.
+ *
+ * `--save <dir>` also writes each checked page's rendered markup to `<dir>` (scripts and the framework's own
+ * asset links removed: they name build-specific chunks), so two builds of a site can be compared with
+ * `diff -r`, e.g. the Next.js starter reading over HTTP and in process.
  */
 const DRAFT_TITLE = 'Coming soon: our winter projects';
 const ARTICLE = 'modelling-without-a-deploy';
@@ -85,6 +91,18 @@ const fetchPage = async (origin: string, path: string) => {
   return { status: response.status, html: await response.text() };
 };
 
+/** The page as rendered: without scripts and framework asset links, which differ from build to build. */
+const renderedMarkup = (html: string) =>
+  html
+    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/g, '')
+    .replace(/<link\b[^>]*\/_next\/[^>]*>/g, '')
+    .replace(/<meta\b[^>]*name="next-size-adjust"[^>]*>/g, '');
+
+const savePage = async (directory: string, path: string, html: string) => {
+  const name = path.replace(/^\/|\/$/g, '').replace(/\//g, '__') || 'index';
+  await writeFile(join(directory, `${name}.html`), `${renderedMarkup(html)}\n`);
+};
+
 const REVALIDATE_PATH = '/api/revalidate/';
 
 const postEvent = (origin: string, body: string, headers: Record<string, string>) =>
@@ -137,12 +155,22 @@ const checkRevalidation = async (origin: string) => {
 
 const main = async () => {
   const { values } = parseArgs({
-    options: { url: { type: 'string' }, revalidate: { type: 'boolean', default: false } },
+    options: {
+      url: { type: 'string' },
+      revalidate: { type: 'boolean', default: false },
+      save: { type: 'string' },
+    },
   });
   const origin = values.url ?? process.env.SMOKE_URL ?? 'http://localhost:4321';
+  if (values.save) {
+    await mkdir(values.save, { recursive: true });
+  }
   for (const page of PAGES) {
     const { status, html } = await fetchPage(origin, page.path);
     check(status === 200, `${page.path}: HTTP ${status}`);
+    if (values.save) {
+      await savePage(values.save, page.path, html);
+    }
     for (const [needle, label] of page.expect) {
       check(html.includes(needle), `${page.path}: ${label}`);
     }

package/templates/next/src/app/api/revalidate/route.ts

@@ -8,7 +8,9 @@ import { readWebhookEvent } from '../../../lib/webhookEvent';
  * POST /api/revalidate/ (with the slash: `trailingSlash` redirects the other form, and Shapio's webhooks do not
  * follow redirects): the target of the Shapio webhook the seed creates (publish, unpublish, delete,
  * change set shipped, schema change, site settings). A signed delivery refreshes the pages whose content changed since the
- * snapshot the site shows (src/lib/revalidation.ts). A failure answers 500, so Shapio retries the delivery.
+ * snapshot the site shows and expires the cached reads tagged with their models and entries; `site.updated` and
+ * schema changes expire the site tag, which every read carries (src/lib/revalidation.ts). A failure answers 500,
+ * so Shapio retries the delivery.
  */
 export const POST = async (request: Request) => {
   const read = await readWebhookEvent(request, webhookSecret());

package/templates/next/src/lib/config.ts

@@ -12,6 +12,10 @@
  *   URL (and `og:url`) unless the entry sets its own.
  * - NEXT_PUBLIC_SHAPIO_URL: the Shapio URL the browser calls for previews (defaults to SHAPIO_URL; inlined into
  *   the client bundle at build time).
+ * - SHAPIO_MODE: `http` (default) reads Shapio's delivery API over HTTP; `in-process` reads the same API as
+ *   function calls in this server process (@shapio/local), from Shapio's database at DATABASE_URL. Previews
+ *   and the browser still use SHAPIO_URL. See documentation/in-process.md.
+ * - DATABASE_URL: Shapio's own database (PostgreSQL or MySQL), for SHAPIO_MODE=in-process only.
  */
 const read = (name: string): string | undefined => {
   const value = process.env[name];
@@ -32,6 +36,24 @@ export const deliveryToken = () => {
 
 export const siteKey = () => read('SHAPIO_SITE');
 
+export type ShapioMode = 'http' | 'in-process';
+
+export const shapioMode = (): ShapioMode => {
+  const mode = read('SHAPIO_MODE') ?? 'http';
+  if (mode !== 'http' && mode !== 'in-process') {
+    throw new Error(`SHAPIO_MODE must be "http" or "in-process", got "${mode}"`);
+  }
+  return mode;
+};
+
+export const databaseUrl = () => {
+  const url = read('DATABASE_URL');
+  if (!url) {
+    throw new Error("SHAPIO_MODE=in-process reads Shapio's database: set DATABASE_URL to it");
+  }
+  return url;
+};
+
 export const siteUrl = () => read('SITE_URL');
 
 export const configuredSnapshot = (): number | undefined => {

package/templates/next/src/lib/revalidation.ts

@@ -1,8 +1,9 @@
 import { ShapioApiError, type SnapshotChange, type SnapshotChangeKind } from '@shapio/client';
-import { revalidatePath } from 'next/cache';
-import { configuredSnapshot, isSnapshotPinned } from './config';
+import { revalidatePath, revalidateTag } from 'next/cache';
+import { configuredSnapshot, isSnapshotPinned, siteKey } from './config';
 import { advanceTo, liveSnapshot } from './liveSnapshot';
 import { log } from './log';
+import { changeTags, siteTags } from './revalidationTags';
 import {
   EVERY_PAGE,
   revalidationTargets,
@@ -13,12 +14,21 @@ import { shapio } from './shapio';
 
 /**
  * On-demand revalidation (`next start`): asks Shapio what changed between the snapshot the site shows and
- * the current one (`/api/snapshots/changes`), revalidates the pages those changes touch and moves the site to
- * the new snapshot. The diff, not the event, is the source of truth: one call covers a change set's many
- * entries, retried or missed deliveries, and both locales of a publish.
+ * the current one (`/api/snapshots/changes`), revalidates the pages those changes touch and the cached reads
+ * tagged with their models and entries, and moves the site to the new snapshot. The diff, not the event, is the
+ * source of truth: one call covers a change set's many entries, retried or missed deliveries, and both locales
+ * of a publish.
  */
 export type RevalidationResult =
-  { from: number; to: number; changed: number; revalidated: RevalidationTarget[] } | { skipped: string };
+  | { from: number; to: number; changed: number; revalidated: RevalidationTarget[]; tags: string[] }
+  | { skipped: string };
+
+/** Expires the cached reads at once (not stale-while-revalidate): the next render reads the new content. */
+const expireTags = (tags: readonly string[]) => {
+  for (const tag of tags) {
+    revalidateTag(tag, { expire: 0 });
+  }
+};
 
 /** Models whose pages are addressed by slug. */
 const SLUGGED_MODELS: ReadonlySet<string> = new Set(['page', 'article']);
@@ -65,18 +75,20 @@ const revalidateNow = async (): Promise<RevalidationResult> => {
   const from = liveSnapshot();
   const { snapshot: to } = await shapio().snapshots.current();
   if (to <= from) {
-    return { from, to: from, changed: 0, revalidated: [] };
+    return { from, to: from, changed: 0, revalidated: [], tags: [] };
   }
   const diff = await shapio().snapshots.allChanges({ from, to });
   const { schemaVersions } = diff;
   const schemaChanged = schemaVersions.from === null || schemaVersions.from !== schemaVersions.to;
   const targets = revalidationTargets(await changedEntries(diff.items, from, to), schemaChanged);
+  const tags = changeTags(diff.items, schemaChanged, siteKey());
   await advanceTo(to, () => {
+    expireTags(tags);
     for (const target of targets) {
       revalidatePath(target.path, target.type);
     }
   });
-  return { from, to, changed: diff.items.length, revalidated: targets };
+  return { from, to, changed: diff.items.length, revalidated: targets, tags };
 };
 
 /** One run at a time: deliveries that arrive together diff from the snapshot the previous run reached. */
@@ -89,22 +101,25 @@ export const revalidateChanges = (): Promise<RevalidationResult> => {
     log(
       'skipped' in result
         ? `Revalidation skipped: ${result.skipped}`
-        : `Snapshots ${result.from} → ${result.to}: revalidated ${result.revalidated.length} path(s)`,
+        : `Snapshots ${result.from} → ${result.to}: revalidated ${result.revalidated.length} path(s), ${result.tags.length} tag(s)`,
     );
     return result;
   });
 };
 
 /**
- * Site-wide changes that publish no snapshot (the site's SEO defaults: `site.updated`): every page is
- * revalidated at the snapshot the site already shows.
+ * Site-wide changes that publish no snapshot (the site's SEO defaults: `site.updated`): the site tag expires
+ * every cached read (`site.get()` among them) and every page is revalidated at the snapshot the site already
+ * shows.
  */
 export const revalidateEveryPage = (): RevalidationResult => {
   if (isSnapshotPinned()) {
     return { skipped: `the build is pinned to snapshot ${configuredSnapshot()} by SHAPIO_SNAPSHOT` };
   }
+  const tags = siteTags(siteKey());
+  expireTags(tags);
   revalidatePath(EVERY_PAGE.path, EVERY_PAGE.type);
   const at = liveSnapshot();
   log(`Site settings changed: revalidated every page at snapshot ${at}`);
-  return { from: at, to: at, changed: 0, revalidated: [EVERY_PAGE] };
+  return { from: at, to: at, changed: 0, revalidated: [EVERY_PAGE], tags };
 };

package/templates/next/src/lib/revalidationTags.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, it } from 'vitest';
+import { changeTags, siteTags } from './revalidationTags';
+
+describe('changeTags', () => {
+  it("tags each changed entry's model and the entry, without duplicates", () => {
+    expect(
+      changeTags(
+        [
+          { routeKey: 'articles', id: 'a1' },
+          { routeKey: 'articles', id: 'a2' },
+          { routeKey: 'siteSettings', id: 's1' },
+          { routeKey: 'articles', id: 'a1' },
+        ],
+        false,
+        'marketing',
+      ),
+    ).toEqual([
+      'shapio:articles',
+      'shapio:articles:a1',
+      'shapio:articles:a2',
+      'shapio:siteSettings',
+      'shapio:siteSettings:s1',
+    ]);
+  });
+
+  it('revalidates the site tag (every read) after a schema change', () => {
+    expect(changeTags([{ routeKey: 'articles', id: 'a1' }], true, 'marketing')).toEqual([
+      'shapio:site:marketing',
+    ]);
+    expect(changeTags([], true, undefined)).toEqual(['shapio:site']);
+  });
+
+  it('has nothing to revalidate when nothing changed', () => {
+    expect(changeTags([], false, undefined)).toEqual([]);
+  });
+});
+
+describe('siteTags', () => {
+  it("is the client's site tag: keyed when the site is configured, bare otherwise", () => {
+    expect(siteTags('marketing')).toEqual(['shapio:site:marketing']);
+    expect(siteTags(undefined)).toEqual(['shapio:site']);
+  });
+});

package/templates/next/src/lib/revalidationTags.ts

@@ -0,0 +1,30 @@
+import { shapioTags, type SnapshotChange } from '@shapio/client';
+
+/**
+ * What a set of Shapio changes means for Next's data cache: the arguments for `revalidateTag`, matching the
+ * tags `@shapio/client` puts on every delivery read (`shapioTags`). Pure, so the mapping is unit-tested;
+ * src/lib/revalidation.ts feeds it from the snapshot diff, which names each changed entry by its route key.
+ */
+export type ChangedEntryRef = Pick<SnapshotChange, 'routeKey' | 'id'>;
+
+/** Every delivery read of the site carries this tag: the site's settings, and everything after a schema change. */
+export const siteTags = (siteKey: string | undefined): string[] => [shapioTags.site(siteKey)];
+
+/**
+ * The tags to revalidate for the entries that changed between two snapshots: each entry's model and the entry
+ * itself. A schema change (fields added, converted or removed) can change any read, so it revalidates the
+ * site tag, which every read carries.
+ */
+export const changeTags = (
+  changes: readonly ChangedEntryRef[],
+  schemaChanged: boolean,
+  siteKey: string | undefined,
+): string[] => {
+  if (schemaChanged) {
+    return siteTags(siteKey);
+  }
+  const tags = new Set(
+    changes.flatMap(({ routeKey, id }) => [shapioTags.model(routeKey), shapioTags.entry(routeKey, id)]),
+  );
+  return [...tags].sort();
+};

package/templates/next/src/lib/shapio.ts

@@ -1,14 +1,25 @@
-import { createClient, ShapioApiError, type DeliveryListQuery, type DeliverySite } from '@shapio/client';
-import { deliveryToken, shapioUrl, siteKey } from './config';
+import {
+  ShapioApiError,
+  type DeliveryListQuery,
+  type DeliverySite,
+  type ShapioDeliveryClient,
+} from '@shapio/client';
 import { liveSnapshot } from './liveSnapshot';
+import { createShapioClient } from './shapioClient';
 import type { Locale } from './site';
 import type { Article, Page, SiteSettings } from './types';
 
 /**
- * The site's read side: Shapio's delivery API through `@shapio/client`, at one publication snapshot. Next
+ * The site's read side: Shapio's delivery API (over HTTP, or in this process with SHAPIO_MODE=in-process:
+ * src/lib/shapioClient.ts), at one publication snapshot. Next
  * renders pages in several worker processes, so the snapshot is pinned once in next.config.ts (it sets
  * SHAPIO_SNAPSHOT for the whole build) and every request here sends it. Under `next start`, /api/revalidate
  * moves that snapshot forward (src/lib/liveSnapshot.ts).
+ *
+ * The client tags every read with the site, model and entry, and /api/revalidate expires the tags of what
+ * changed (src/lib/revalidationTags.ts) as well as the pages. No `next: { cache }` option: reads pinned to a
+ * snapshot are `force-cache` regardless, and `site.get()` (no snapshot) stays fresh on every build, since Next's
+ * data cache outlives `next build`.
  */
 const PAGE_SIZE = 100;
 
@@ -22,11 +33,8 @@ const PAGE_FIELDS = ['title', 'slug', 'description', 'sections', 'seo'];
 const ARTICLE_FIELDS = ['title', 'slug', 'excerpt', 'body', 'cover', 'author', 'publishedOn', 'seo'];
 const SITE_SETTINGS_FIELDS = ['siteName', 'tagline', 'footer', 'colophon'];
 
-export const createShapio = () =>
-  createClient({ baseUrl: shapioUrl(), token: deliveryToken(), site: siteKey() });
-
-let client: ReturnType<typeof createShapio> | undefined;
-export const shapio = () => (client ??= createShapio());
+let client: ShapioDeliveryClient | undefined;
+export const shapio = () => (client ??= createShapioClient());
 
 const snapshot = liveSnapshot;
 
@@ -80,8 +88,9 @@ export const getSiteSettings = async (locale: Locale): Promise<SiteSettings | nu
 };
 
 /**
- * The site and its SEO defaults (name and title template per locale, default image, Twitter handle). Read on
- * every render: a `site.updated` webhook revalidates every page, which then shows the new defaults. Null when
+ * The site and its SEO defaults (name and title template per locale, default image, Twitter handle). Cached
+ * under the site tag: a `site.updated` webhook expires it and revalidates every page, which then shows the new
+ * defaults. Null when
  * the Shapio predates them (the pages still render, with plain titles).
  */
 export const getSite = async (): Promise<DeliverySite | null> => {

package/templates/next/src/lib/shapioClient.ts

@@ -0,0 +1,17 @@
+import { createClient, type ShapioDeliveryClient } from '@shapio/client';
+import { createLocalClient } from '@shapio/local';
+import { databaseUrl, deliveryToken, shapioMode, shapioUrl, siteKey } from './config';
+
+/**
+ * The client the site reads with: Shapio's delivery API over HTTP (SHAPIO_MODE=http, the default), or the same
+ * API in this process (SHAPIO_MODE=in-process, `@shapio/local`), reading Shapio's database directly. Both
+ * expose the same `delivery`, `site` and `snapshots` calls and answer them identically, with the same delivery
+ * token deciding what the site may read.
+ */
+export const createShapioClient = (): ShapioDeliveryClient & { close?: () => Promise<void> } => {
+  const site = siteKey();
+  if (shapioMode() === 'in-process') {
+    return createLocalClient({ databaseUrl: databaseUrl(), token: deliveryToken(), site });
+  }
+  return createClient({ baseUrl: shapioUrl(), token: deliveryToken(), site });
+};

package/templates/sveltekit/package.json

@@ -21,8 +21,8 @@
     "typecheck": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --fail-on-warnings"
   },
   "dependencies": {
-    "@shapio/client": "^0.4.0",
-    "@shapio/visual": "^0.4.0"
+    "@shapio/client": "^0.5.0",
+    "@shapio/visual": "^0.5.0"
   },
   "devDependencies": {
     "@sveltejs/adapter-static": "4.0.0",

package/templates/sveltekit/scripts/smokeHttp.ts

@@ -1,4 +1,6 @@
 import { createHmac } from 'node:crypto';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
 import { parseArgs } from 'node:util';
 
 /**
@@ -13,6 +15,10 @@ import { parseArgs } from 'node:util';
  * `--url` is the running site's origin (default: SMOKE_URL, then http://localhost:4321). `--revalidate` (the
  * Next.js starter) also checks its on-demand revalidation route: an unsigned POST to /api/revalidate is refused,
  * and an event signed with SHAPIO_WEBHOOK_SECRET is answered 200 with the paths it revalidated.
+ *
+ * `--save <dir>` also writes each checked page's rendered markup to `<dir>` (scripts and the framework's own
+ * asset links removed: they name build-specific chunks), so two builds of a site can be compared with
+ * `diff -r`, e.g. the Next.js starter reading over HTTP and in process.
  */
 const DRAFT_TITLE = 'Coming soon: our winter projects';
 const ARTICLE = 'modelling-without-a-deploy';
@@ -85,6 +91,18 @@ const fetchPage = async (origin: string, path: string) => {
   return { status: response.status, html: await response.text() };
 };
 
+/** The page as rendered: without scripts and framework asset links, which differ from build to build. */
+const renderedMarkup = (html: string) =>
+  html
+    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/g, '')
+    .replace(/<link\b[^>]*\/_next\/[^>]*>/g, '')
+    .replace(/<meta\b[^>]*name="next-size-adjust"[^>]*>/g, '');
+
+const savePage = async (directory: string, path: string, html: string) => {
+  const name = path.replace(/^\/|\/$/g, '').replace(/\//g, '__') || 'index';
+  await writeFile(join(directory, `${name}.html`), `${renderedMarkup(html)}\n`);
+};
+
 const REVALIDATE_PATH = '/api/revalidate/';
 
 const postEvent = (origin: string, body: string, headers: Record<string, string>) =>
@@ -137,12 +155,22 @@ const checkRevalidation = async (origin: string) => {
 
 const main = async () => {
   const { values } = parseArgs({
-    options: { url: { type: 'string' }, revalidate: { type: 'boolean', default: false } },
+    options: {
+      url: { type: 'string' },
+      revalidate: { type: 'boolean', default: false },
+      save: { type: 'string' },
+    },
   });
   const origin = values.url ?? process.env.SMOKE_URL ?? 'http://localhost:4321';
+  if (values.save) {
+    await mkdir(values.save, { recursive: true });
+  }
   for (const page of PAGES) {
     const { status, html } = await fetchPage(origin, page.path);
     check(status === 200, `${page.path}: HTTP ${status}`);
+    if (values.save) {
+      await savePage(values.save, page.path, html);
+    }
     for (const [needle, label] of page.expect) {
       check(html.includes(needle), `${page.path}: ${label}`);
     }