@crombolo/cli 0.13.2 → 0.14.4

package/src/remote/transfer.ts

@@ -0,0 +1,337 @@
+/**
+ * `crombolo pull` and `crombolo push` as functions (story 108): they take a
+ * signed-in client and options, and return what happened; the binary prints.
+ *
+ * A pull is a job on the remote ending in a sealed file, opened here with a
+ * passphrase made for that one pull and kept in memory. A push exports this
+ * site sealed the same way, checks with the remote what it would replace, and
+ * uploads it as a job — which the remote may hold for approval.
+ */
+import { randomBytes } from 'node:crypto'
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import {
+  bootstrapDatabase,
+  type CromboloConfig,
+  exportSite,
+  extractTarball,
+  type MediaStore,
+  mediaStoreFor,
+} from '@crombolo/server'
+import { type BundleBlob, loadBundle } from '@crombolo/transfer'
+import { type Job, problemOf, type RemoteClient, RemoteError, sha256File } from './client.ts'
+
+export type MediaMode = 'included' | 'omitted' | 'referenced'
+
+/** A passphrase for one transfer: never shown, never stored. */
+export const transferPassphrase = () => randomBytes(24).toString('base64url')
+
+/** Refused before anything was sent: the CLI's own guard, exit code 2. */
+export class RefusedBeforeSending extends Error {}
+
+export interface PullResult {
+  job: Job
+  file: string
+  passphrase: string
+  size: number
+}
+
+/** Asks the remote for its site and downloads the file to `out`. */
+export async function pullFile(
+  client: RemoteClient,
+  options: {
+    environment: string
+    out: string
+    full?: boolean
+    media?: MediaMode
+    onProgress?(job: Job): void
+    timeoutMs?: number
+    sleep?: (ms: number) => Promise<void>
+  },
+): Promise<PullResult> {
+  const environments = await client.environments()
+  const env = environments.find((e) => e.id === options.environment)
+  if (!env)
+    throw new RefusedBeforeSending(
+      `${client.discovery.name} has no environment "${options.environment}": it has ${environments.map((e) => e.id).join(', ')}.`,
+    )
+  if (!env.may.pull) throw new RefusedBeforeSending(`You may not pull from ${env.name}.`)
+  if (options.full && !env.may.pullFull)
+    throw new RefusedBeforeSending(
+      `You may not pull a full copy of ${env.name}; pull without --full for an anonymised one.`,
+    )
+  if (options.media === 'referenced' && !client.discovery.capabilities.includes('media-fetch'))
+    throw new RefusedBeforeSending(
+      `${client.discovery.name} does not serve media by reference; pull without --media-info-only.`,
+    )
+  const passphrase = transferPassphrase()
+  const started = await client.post<Job>(`environments/${env.id}/pulls`, {
+    passphrase,
+    full: options.full === true,
+    media: options.media ?? 'included',
+  })
+  const job = await client.settle(started.id, {
+    onProgress: options.onProgress,
+    timeoutMs: options.timeoutMs,
+    sleep: options.sleep,
+  })
+  if (job.state !== 'done') throw new RemoteError(job.problem ?? jobProblem(job))
+  const { size } = await client.download(job, options.out)
+  return { job, file: options.out, passphrase, size }
+}
+
+const jobProblem = (job: Job) => ({
+  status: 0,
+  code: job.state,
+  title: `The ${job.kind} ended ${job.state}`,
+})
+
+/** What the local site has already fetched of a remote's media, by key: the remote's etag. */
+const FETCHED_FILE = join('.crombolo', 'remote-media-fetched.json')
+
+function readFetched(siteDir: string): Record<string, string> {
+  const file = join(siteDir, FETCHED_FILE)
+  if (!existsSync(file)) return {}
+  try {
+    return JSON.parse(readFileSync(file, 'utf8')) as Record<string, string>
+  } catch {
+    return {}
+  }
+}
+
+export interface MediaFetchResult {
+  fetched: number
+  skipped: number
+  bytes: number
+  failed: string[]
+}
+
+/**
+ * Fetches the media files a file names by reference into the local store, a
+ * batch of links at a time and a few files at once. A file already fetched at
+ * the same etag is skipped, so a second pull downloads only what changed.
+ */
+export async function fetchReferencedMedia(
+  client: RemoteClient,
+  options: {
+    environment: string
+    blobs: readonly Pick<BundleBlob, 'key' | 'etag'>[]
+    store: MediaStore
+    siteDir: string
+    onProgress?(done: number, total: number): void
+    http?: typeof fetch
+  },
+): Promise<MediaFetchResult> {
+  const fetched = readFetched(options.siteDir)
+  const result: MediaFetchResult = { fetched: 0, skipped: 0, bytes: 0, failed: [] }
+  const wanted: string[] = []
+  for (const blob of options.blobs) {
+    const have = await options.store.get(blob.key)
+    if (have && blob.etag && fetched[blob.key] === blob.etag) result.skipped++
+    else wanted.push(blob.key)
+  }
+  let done = 0
+  for (let at = 0; at < wanted.length; at += 200) {
+    const batch = wanted.slice(at, at + 200)
+    const { links } = await client.post<{
+      links: Record<string, { url: string; etag: string; size: number } | null>
+    }>(`environments/${options.environment}/media/links`, { keys: batch })
+    const queue = [...batch]
+    const worker = async () => {
+      for (;;) {
+        const key = queue.shift()
+        if (!key) return
+        const link = links[key]
+        try {
+          if (!link) throw new Error('not on the remote')
+          // A link through the API needs the sign-in; a presigned one does not.
+          const response = link.url.startsWith(client.discovery.api)
+            ? await client.call(link.url)
+            : await (options.http ?? fetch)(link.url)
+          if (!response.ok) throw await problemOf(response)
+          const body = await response.blob()
+          await options.store.put(key, body)
+          fetched[key] = link.etag
+          result.fetched++
+          result.bytes += body.size
+        } catch {
+          result.failed.push(key)
+        }
+        options.onProgress?.(++done, wanted.length)
+      }
+    }
+    await Promise.all(Array.from({ length: 6 }, worker))
+  }
+  mkdirSync(join(options.siteDir, '.crombolo'), { recursive: true })
+  writeFileSync(join(options.siteDir, FETCHED_FILE), `${JSON.stringify(fetched, null, 2)}\n`)
+  return result
+}
+
+/** The blobs a file names by reference, read from its manifest. */
+export async function referencedBlobs(file: string): Promise<BundleBlob[]> {
+  const dir = mkdtempSync(join(tmpdir(), 'crombolo-pulled-'))
+  try {
+    await extractTarball(file, dir)
+    const manifest = loadBundle(join(dir, 'content')).set?.manifest
+    return manifest?.media === 'referenced' ? manifest.blobs : []
+  } finally {
+    rmSync(dir, { recursive: true, force: true })
+  }
+}
+
+export interface PushCheck {
+  environment: string
+  recycles: Record<string, number>
+  removesFiles: string[]
+  keepsPeople: boolean
+  /** Whether the remote answered it, or it is only what this side is sending. */
+  fromRemote: boolean
+  sending: { counts: Record<string, number>; files: number; media: number }
+}
+
+export interface PushResult {
+  check: PushCheck
+  job: Job
+}
+
+/**
+ * Exports this site and sends it. `confirm` is asked with the check for a
+ * protected environment unless `autoApprove`; `onApproval` hears the job when
+ * the remote holds it for approval. The flags are this side's guard against a
+ * slip, never authority: the remote decides.
+ */
+export async function pushSite(
+  config: CromboloConfig,
+  client: RemoteClient,
+  options: {
+    environment: string
+    force?: boolean
+    autoApprove?: boolean
+    mediaInfoOnly?: boolean
+    confirm?(check: PushCheck): Promise<boolean>
+    onCheck?(check: PushCheck): void
+    onProgress?(job: Job): void
+    onApproval?(job: Job): void | Promise<void>
+    onMedia?(done: number, total: number): void
+    timeoutMs?: number
+    sleep?: (ms: number) => Promise<void>
+  },
+): Promise<PushResult> {
+  const environments = await client.environments()
+  const env = environments.find((e) => e.id === options.environment)
+  if (!env)
+    throw new RefusedBeforeSending(
+      `${client.discovery.name} has no environment "${options.environment}".`,
+    )
+  if (!env.may.push) throw new RefusedBeforeSending(`You may not push to ${env.name}.`)
+  if (env.protected && !options.force)
+    throw new RefusedBeforeSending(
+      `${env.name} is protected: pushing replaces what people rely on. Pass --force to go on.`,
+    )
+  if (options.mediaInfoOnly && !client.discovery.capabilities.includes('media-fetch'))
+    throw new RefusedBeforeSending(
+      `${client.discovery.name} does not take media by reference; push without --media-info-only.`,
+    )
+
+  const work = mkdtempSync(join(tmpdir(), 'crombolo-push-'))
+  try {
+    const out = join(work, 'site.tgz')
+    const passphrase = transferPassphrase()
+    const { db } = await bootstrapDatabase(config)
+    try {
+      await exportSite(db, config, {
+        out,
+        sealContent: true,
+        passphrase,
+        media: options.mediaInfoOnly ? 'referenced' : true,
+      })
+    } finally {
+      await db.close()
+    }
+    const unpacked = join(work, 'unpacked')
+    await extractTarball(out, unpacked)
+    const loaded = loadBundle(join(unpacked, 'content'))
+    const manifest = loaded.set?.manifest
+    if (!manifest) throw new Error('The export could not be read back.')
+    const sending = {
+      counts: { ...manifest.counts } as Record<string, number>,
+      files: loaded.files.size,
+      media: manifest.blobs.length,
+    }
+    const check: PushCheck = client.discovery.capabilities.includes('push-check')
+      ? {
+          ...(await client.post<Omit<PushCheck, 'fromRemote' | 'sending'>>(
+            `environments/${env.id}/pushes/check`,
+            {
+              nodes: (loaded.set?.nodes ?? []).map((node) => node.key),
+              files: [...loaded.files.keys()],
+            },
+          )),
+          fromRemote: true,
+          sending,
+        }
+      : {
+          environment: env.id,
+          recycles: {},
+          removesFiles: [],
+          keepsPeople: true,
+          fromRemote: false,
+          sending,
+        }
+    options.onCheck?.(check)
+    if (env.protected && !options.autoApprove) {
+      const yes = (await options.confirm?.(check)) ?? false
+      if (!yes) throw new RefusedBeforeSending('Nothing was sent.')
+    }
+
+    if (options.mediaInfoOnly && manifest.blobs.length > 0) {
+      const store = await mediaStoreFor(config)
+      const { have } = await client.post<{ have: string[] }>(`environments/${env.id}/media/have`, {
+        blobs: manifest.blobs.map((blob) => ({ key: blob.key, etag: blob.etag ?? undefined })),
+      })
+      const present = new Set(have)
+      const missing = manifest.blobs.filter((blob) => !present.has(blob.key))
+      let done = 0
+      for (const blob of missing) {
+        const file = await store.get(blob.key)
+        if (file) {
+          const bytes = await file.bytes()
+          const response = await client.api(
+            `environments/${env.id}/media/${blob.key.split('/').map(encodeURIComponent).join('/')}`,
+            {
+              method: 'PUT',
+              headers: {
+                'crombolo-sha256': new Bun.CryptoHasher('sha256').update(bytes).digest('hex'),
+              },
+              body: bytes as BodyInit,
+            },
+          )
+          if (!response.ok) throw await problemOf(response)
+        }
+        options.onMedia?.(++done, missing.length)
+      }
+    }
+
+    const response = await client.api(`environments/${env.id}/pushes`, {
+      method: 'PUT',
+      headers: {
+        'content-type': 'application/gzip',
+        'crombolo-sha256': await sha256File(out),
+        'crombolo-passphrase': passphrase,
+      },
+      body: Bun.file(out),
+    })
+    if (!response.ok) throw await problemOf(response)
+    const started = (await response.json()) as Job
+    const job = await client.settle(started.id, {
+      onProgress: options.onProgress,
+      onApproval: options.onApproval,
+      timeoutMs: options.timeoutMs,
+      sleep: options.sleep,
+    })
+    return { check, job }
+  } finally {
+    rmSync(work, { recursive: true, force: true })
+  }
+}

package/templates/basic/bundle/package.json

@@ -6,7 +6,7 @@
     "content"
   ],
   "peerDependencies": {
-    "@crombolo/crombolo": ">=0.13.2"
+    "@crombolo/crombolo": ">=0.14.4"
   },
   "crombolo": {
     "id": "basic",

package/templates/demo/harbourstone/bundle/content/bundle.json

@@ -3,13 +3,13 @@
   "id": "af030aee-ba70-4bed-8a27-a3806a79100d",
   "createdAt": "2026-10-01T00:00:00.000Z",
   "createdBy": "bun run build:template",
-  "integrity": "sha256-dda870bf7e8fadb391bcadc9e75a5721cfe209a9e8aa794df2801e0b17d8e958",
+  "integrity": "sha256-d5915f80be85d8eb3aca0488721ea3b7884e6649049cd6261ced0e7f58d44c80",
   "snapshot": "published",
   "provenance": {
     "siteName": "Harbourstone Distillery",
     "schemaVersion": "1.0.0",
     "schemaRevision": "0",
-    "schemaHash": "9c729977ea20bcaca36022f92fd57b76321825b82d817f7e688bd8dc6b7ba289"
+    "schemaHash": "ac93ee497083881b983ce2f3e305e8d450a31ca1d27203223d8472fd247cae66"
   },
   "selector": {
     "roots": [
@@ -47,7 +47,8 @@
       "a562f0a2-cb6e-4ead-a1ce-4d180632419f",
       "cd681b4b-9c4e-4624-9c13-450aa6a9c135",
       "40107d59-f3e9-4592-aee8-5b03d4586cbd",
-      "3d312cc8-2937-40e1-ba14-fd781c9a862d"
+      "3d312cc8-2937-40e1-ba14-fd781c9a862d",
+      "9468d1b4-f521-48ef-b044-2ab07b461b96"
     ],
     "descendants": true,
     "asGiven": [
@@ -57,7 +58,7 @@
   "counts": {
     "media": 7,
     "element": 20,
-    "document": 8
+    "document": 9
   },
   "dependencies": {
     "carried": [
@@ -83,6 +84,7 @@
       "6d8d6398-738f-4f38-ad3c-e5d6e9a971fe",
       "7b4713f8-d917-457d-a9d8-7dd01a3617b4",
       "88ec1185-53b5-439a-a2ce-3f56603be4ff",
+      "9468d1b4-f521-48ef-b044-2ab07b461b96",
       "97469df2-107c-4010-8c14-4de88f735529",
       "a2eb439f-71d9-4ea6-89d7-c5bc72618281",
       "a562f0a2-cb6e-4ead-a1ce-4d180632419f",
@@ -124,6 +126,10 @@
           "key": "cc07b313-0843-4aa8-bbda-871c8da728c8",
           "alias": "Image"
         },
+        {
+          "key": "4308082e-5024-47fc-950d-1ee93a3e304f",
+          "alias": "searchPage"
+        },
         {
           "key": "55138391-faf2-4795-9ed6-43d581d350a4",
           "alias": "tastingNote"
@@ -138,6 +144,7 @@
         "pages/contentPage",
         "pages/expression",
         "pages/homePage",
+        "pages/searchPage",
         "pages/whiskyList"
       ],
       "languages": []

package/templates/demo/harbourstone/bundle/content/nodes/40107d59-f3e9-4592-aee8-5b03d4586cbd.json

@@ -67,6 +67,13 @@
       "editor": "Umbraco.TextBox",
       "value": "Cask strength"
     },
+    {
+      "property": "productType",
+      "culture": null,
+      "segment": null,
+      "editor": "Umbraco.TextBox",
+      "value": "Whisky"
+    },
     {
       "property": "summary",
       "culture": null,

package/templates/demo/harbourstone/bundle/content/nodes/6ad8f0f3-9334-47f1-b805-80c4b4b4fa88.json

@@ -67,6 +67,13 @@
       "editor": "Umbraco.TextBox",
       "value": "House malt"
     },
+    {
+      "property": "productType",
+      "culture": null,
+      "segment": null,
+      "editor": "Umbraco.TextBox",
+      "value": "Whisky"
+    },
     {
       "property": "summary",
       "culture": null,

package/templates/demo/harbourstone/bundle/content/nodes/9468d1b4-f521-48ef-b044-2ab07b461b96.json

@@ -0,0 +1,42 @@
+{
+  "key": "9468d1b4-f521-48ef-b044-2ab07b461b96",
+  "kind": "document",
+  "contentType": {
+    "key": "4308082e-5024-47fc-950d-1ee93a3e304f",
+    "alias": "searchPage"
+  },
+  "parent": "d60ef12b-0f0a-4db5-9763-21fa52b8d4da",
+  "sortOrder": 3,
+  "template": "pages/searchPage",
+  "variants": [
+    {
+      "culture": null,
+      "segment": null,
+      "name": "Search",
+      "published": true
+    }
+  ],
+  "values": [
+    {
+      "property": "eyebrow",
+      "culture": null,
+      "segment": null,
+      "editor": "Umbraco.TextBox",
+      "value": "Search"
+    },
+    {
+      "property": "standfirst",
+      "culture": null,
+      "segment": null,
+      "editor": "Umbraco.TextArea",
+      "value": "Search the range, the tasting notes and the story of the distillery."
+    },
+    {
+      "property": "title",
+      "culture": null,
+      "segment": null,
+      "editor": "Umbraco.TextBox",
+      "value": "Find it in the cove."
+    }
+  ]
+}

package/templates/demo/harbourstone/bundle/content/nodes/a562f0a2-cb6e-4ead-a1ce-4d180632419f.json

@@ -67,6 +67,13 @@
       "editor": "Umbraco.TextBox",
       "value": "Lightly peated"
     },
+    {
+      "property": "productType",
+      "culture": null,
+      "segment": null,
+      "editor": "Umbraco.TextBox",
+      "value": "Whisky"
+    },
     {
       "property": "summary",
       "culture": null,

package/templates/demo/harbourstone/bundle/content/nodes/cd681b4b-9c4e-4624-9c13-450aa6a9c135.json

@@ -67,6 +67,13 @@
       "editor": "Umbraco.TextBox",
       "value": "Sherry matured"
     },
+    {
+      "property": "productType",
+      "culture": null,
+      "segment": null,
+      "editor": "Umbraco.TextBox",
+      "value": "Whisky"
+    },
     {
       "property": "summary",
       "culture": null,

package/templates/demo/harbourstone/bundle/package.json

@@ -6,7 +6,7 @@
     "content"
   ],
   "peerDependencies": {
-    "@crombolo/crombolo": ">=0.13.2"
+    "@crombolo/crombolo": ">=0.14.4"
   },
   "crombolo": {
     "id": "demo-harbourstone",

package/templates/demo/harbourstone/files/components/pages/searchPage.tsx

@@ -0,0 +1,76 @@
+import { type PageProps, search } from '@crombolo/crombolo'
+import { Layout } from '../shared/layout.tsx'
+import { Eyebrow } from '../shared/section-intro.tsx'
+
+const PER_PAGE = 10
+
+/**
+ * The pages matching `?q=`, best first, ten at a time. Each result reads its
+ * eyebrow and title by the aliases every page type here shares, so a whisky
+ * shows as "Single malt — The Harbour 12" rather than by its node name.
+ */
+export default async function SearchPage({ model, nav, request }: PageProps) {
+  const url = new URL(request.url)
+  const query = url.searchParams.get('q')?.trim() ?? ''
+  const page = Math.max(1, Number.parseInt(url.searchParams.get('page') ?? '1', 10) || 1)
+  const results = query
+    ? await search(query, { skip: (page - 1) * PER_PAGE, take: PER_PAGE })
+    : { total: 0, items: [] }
+  const pages = Math.ceil(results.total / PER_PAGE)
+  const link = (n: number) => `${model.url}?q=${encodeURIComponent(query)}&page=${n}`
+
+  return (
+    <Layout model={model} nav={nav}>
+      <header class="page-header">
+        <Eyebrow>{model.text('eyebrow')}</Eyebrow>
+        <h1>{model.text('title')}</h1>
+        <search>
+          <form class="search-form" action={model.url} method="get">
+            <input type="search" name="q" value={query} aria-label="Search the site" />
+            <button class="button" type="submit">
+              Search
+            </button>
+          </form>
+        </search>
+        {query ? (
+          <p class="standfirst">
+            {results.total === 0
+              ? `Nothing matches “${query}”.`
+              : `${results.total} ${results.total === 1 ? 'page matches' : 'pages match'} “${query}”.`}
+          </p>
+        ) : model.hasValue('standfirst') ? (
+          <p class="standfirst">{model.text('standfirst')}</p>
+        ) : null}
+      </header>
+
+      {results.items.length > 0 ? (
+        <section>
+          <ol class="search-results">
+            {results.items.map((item) => (
+              <li>
+                <Eyebrow>{item.content.text('eyebrow')}</Eyebrow>
+                <h2>
+                  <a href={item.url}>{item.content.text('title') || item.name}</a>
+                </h2>
+                <p>
+                  {item.excerpt.map((segment) =>
+                    segment.match ? <mark>{segment.text}</mark> : segment.text,
+                  )}
+                </p>
+              </li>
+            ))}
+          </ol>
+          {pages > 1 ? (
+            <nav class="search-pages" aria-label="Search results pages">
+              {page > 1 ? <a href={link(page - 1)}>Previous</a> : <span />}
+              <span>
+                Page {page} of {pages}
+              </span>
+              {page < pages ? <a href={link(page + 1)}>Next</a> : <span />}
+            </nav>
+          ) : null}
+        </section>
+      ) : null}
+    </Layout>
+  )
+}

package/templates/demo/harbourstone/files/components/shared/site-header.tsx

@@ -11,12 +11,17 @@ import type { Navigation, PublishedContent } from '@crombolo/crombolo'
  * There is no hamburger because there is no client-side JavaScript on this
  * site: below the breakpoint the links wrap under the wordmark instead, which
  * `site.css` does with flex-wrap alone.
+ *
+ * A Search Page among the children is the search box at the end instead of a
+ * link: a plain GET form, which needs no script either.
  */
 export function SiteHeader({ model, nav }: { model: PublishedContent; nav: Navigation }) {
   const home = nav.root()[0]
   if (!home) return null
   const ancestry = model.path.split(',')
-  const links = [home, ...nav.children(home)]
+  const children = nav.children(home)
+  const searchPage = children.find((child) => child.contentType.alias === 'searchPage')
+  const links = [home, ...children.filter((child) => child !== searchPage)]
   return (
     <header class="masthead">
       <div class="masthead__inner">
@@ -38,6 +43,13 @@ export function SiteHeader({ model, nav }: { model: PublishedContent; nav: Navig
             )
           })}
         </nav>
+        {searchPage ? (
+          <search class="masthead__search">
+            <form action={searchPage.url} method="get">
+              <input type="search" name="q" placeholder="Search" aria-label="Search the site" />
+            </form>
+          </search>
+        ) : null}
       </div>
     </header>
   )