@crombolo/server 0.14.1 → 0.14.4

package/src/server.ts

@@ -31,7 +31,9 @@ import {
   createExtensionRegistry,
   isSpaRoute,
   renderBackOfficeShell,
+  renderDeviceShell,
   renderLoginShell,
+  renderRemoteApprovalShell,
   resolveGraphic,
   resolveLoginAsset,
   resolveStaticFile,
@@ -95,7 +97,13 @@ import { CommandError, installBundleContent } from './bundle-install.ts'
 import { bundleChains, bundleTablesReport } from './bundle-tables.ts'
 import { createBundleRedirects, createServerBundles, validateServerBundles } from './bundles.ts'
 import { busyResponse } from './busy.ts'
-import { type CromboloConfig, loadConfig, type ServerRole, VERSION } from './config.ts'
+import {
+  type CromboloConfig,
+  listeningBodyLimit,
+  loadConfig,
+  type ServerRole,
+  VERSION,
+} from './config.ts'
 import { attachDatabase, bootstrapDatabase } from './database.ts'
 import { createDeliveryMemberSecurity } from './delivery-members.ts'
 import { createDiagnostics } from './diagnostics.ts'
@@ -136,6 +144,8 @@ import { noNodesPage } from './no-nodes.ts'
 import { NotImplementedLog } from './not-implemented.ts'
 import { createDeps, PREVIEW_COOKIE, resolveUserLinkSender } from './ports.ts'
 import { createRedirectTracker } from './redirects.ts'
+import { createRemote } from './remote.ts'
+import { withRemoteMedia } from './remote-media.ts'
 import { handleTransferAction, pendingTransfers } from './transfer-dashboard.ts'
 import {
   changeUrlOptions,
@@ -656,6 +666,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
           },
         )
       : undefined
+  const remote = createRemote({ db, config, authenticate })
   const members = createMemberAuth(db, config, {
     nodeState: schema.nodeState,
     nodeId: schema.nodeId,
@@ -728,7 +739,11 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
   })
   // The site's config may name a store; without one the environment decides,
   // and that falls back to the media directory on this disk.
-  const mediaStore = config.mediaStore ?? (await mediaStoreFromEnvironment(config.mediaDir))
+  // A site pulled with `--lazy-media` reads files through from its remote (story 108).
+  const mediaStore = withRemoteMedia(
+    config.mediaStore ?? (await mediaStoreFromEnvironment(config.mediaDir)),
+    config.siteDir,
+  )
   // Resolved once: constructing it twice would warn twice about a provider that
   // is half-configured, and the adapters are cheap but not free of side effects.
   const email = resolveEmailPort(config)
@@ -968,6 +983,12 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
       if (response) return response
     }
 
+    // The remote protocol (story 107), where the backoffice is: it acts as a person.
+    if (servesBackOffice) {
+      const response = await remote.handle(request, url)
+      if (response) return response
+    }
+
     if (servesBackOffice) {
       const response = await serveBackOffice(request, url)
       if (response) {
@@ -1698,6 +1719,51 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
       return api.dispatch(request)
     }
 
+    // Approving a command-line sign-in (story 107): signed in first, then back here.
+    if (pathname === `${paths.backOfficePath}/device` && request.method === 'GET') {
+      const sessionId = readCookie(request, names.session)
+      if (!sessionId || !(await auth.resolveSession(sessionId))) {
+        const back = `${pathname}${url.search}`
+        return new Response(null, {
+          status: 303,
+          headers: {
+            location: `${paths.backOfficePath}/login?returnUrl=${encodeURIComponent(back)}`,
+          },
+        })
+      }
+      const { importmap } = collectManifests(paths, extensions.list())
+      return html(
+        renderDeviceShell(paths, importmap, {
+          title: config.siteName,
+          userCode: url.searchParams.get('user_code') ?? undefined,
+        }),
+      )
+    }
+
+    // Approving a push to a protected environment (story 107): the page reads
+    // and decides the job through the remote API, as the signed-in person.
+    if (
+      pathname === `${paths.backOfficePath}/remote-approval` &&
+      request.method === 'GET' &&
+      config.remote.enabled
+    ) {
+      const sessionId = readCookie(request, names.session)
+      if (!sessionId || !(await auth.resolveSession(sessionId)))
+        return new Response(null, {
+          status: 303,
+          headers: {
+            location: `${paths.backOfficePath}/login?returnUrl=${encodeURIComponent(`${pathname}${url.search}`)}`,
+          },
+        })
+      const { importmap } = collectManifests(paths, extensions.list())
+      return html(
+        renderRemoteApprovalShell(paths, importmap, {
+          title: config.siteName,
+          job: url.searchParams.get('job') ?? '',
+        }),
+      )
+    }
+
     if (pathname === `${paths.backOfficePath}/login`) {
       const { importmap } = collectManifests(paths, extensions.list())
       return html(
@@ -2108,11 +2174,24 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     jobs,
     serveOptions: {
       port: config.port,
-      maxRequestBodySize: config.maxRequestBodyBytes,
+      maxRequestBodySize: listeningBodyLimit(config),
       fetch: diagnostics.wrap(async (request, bunServer: Server<HubSocketData>) => {
         const peer = bunServer.requestIP(request)?.address
         if (peer) peers.set(request, peer)
-        const pathname = new URL(request.url).pathname
+        const requestUrl = new URL(request.url)
+        const pathname = requestUrl.pathname
+        // Listening with the remote's larger limit (story 107), so every other
+        // request is held to the usual one here.
+        if (
+          listeningBodyLimit(config) > config.maxRequestBodyBytes &&
+          !remote.isUpload(request, requestUrl)
+        ) {
+          const length = Number(request.headers.get('content-length') ?? 0)
+          if (length > config.maxRequestBodyBytes)
+            return new Response('Payload Too Large', { status: 413 })
+          if (!request.headers.has('content-length') && request.headers.has('transfer-encoding'))
+            return new Response('Length Required', { status: 411 })
+        }
         if (pathname === hubPath || pathname === previewHubPath) {
           const principal = await authenticate(request)
           const hub = pathname === hubPath ? events : previewHub

package/src/site-file.ts

@@ -27,6 +27,7 @@ import {
   packageNameFor,
   planStreamedBundle,
   sanitise,
+  sealBytes,
   writeManifest,
 } from '@crombolo/transfer'
 import { createFileSystemPort } from './adapters/file-system.ts'
@@ -40,15 +41,25 @@ import { TarballWriter } from './tar.ts'
 export interface SiteExportOptions {
   /** The `.tgz` to write. */
   out: string
-  /** False for `--no-media`: the media nodes travel, their files do not. */
-  media?: boolean
+  /**
+   * False for `--no-media`: the media nodes travel, their files do not.
+   * `referenced`: likewise, for a remote and its CLI to move the files one at a
+   * time (story 107).
+   */
+  media?: boolean | 'referenced'
   /** Carry the latest drafts rather than what is published. */
   drafts?: boolean
   withMembers?: boolean
   withUsers?: boolean
   /** Replace every personal value before anything is written (`sanitise`). */
   anonymise?: boolean
-  /** Seals the people; needed when any travel. */
+  /**
+   * Seal the content data — property values and media files — as well as any
+   * people, under `passphrase`: the bundle rule, metadata readable and content
+   * not (story 107). Each media file is sealed whole, one at a time.
+   */
+  sealContent?: boolean
+  /** Seals the people, and the content with `sealContent`; needed when either travels sealed. */
   passphrase?: string
   package?: { name?: string; version?: string }
 }
@@ -101,12 +112,14 @@ export async function exportSite(
   config: CromboloConfig,
   options: SiteExportOptions,
 ): Promise<SiteExportResult> {
-  const withMedia = options.media !== false
+  const withMedia = options.media !== false && options.media !== 'referenced'
   const carryMembers = Boolean(options.withMembers)
   // Anonymised users are removed, so there are none to carry.
   const carryUsers = Boolean(options.withUsers) && !options.anonymise
   if ((carryMembers || carryUsers) && !options.passphrase)
     throw new Error('Members and users travel sealed: give a passphrase.')
+  if (options.sealContent && !options.passphrase)
+    throw new Error('Sealed content needs a passphrase.')
 
   const exported = await exportBundle(db, {
     roots: await topLevelNodes(db),
@@ -158,30 +171,43 @@ export async function exportSite(
   const store = await mediaStoreFor(config)
   const stored = new Map<string, StoredFile>()
   const missing: string[] = []
-  if (withMedia)
+  // Referenced, each file is named with its size and etag, so the far side can
+  // fetch it — or skip it, holding one with the same etag already.
+  const described = new Map<string, { size: number; etag: string }>()
+  if (withMedia || options.media === 'referenced')
     for (const key of exported.blobKeys) {
       const file = await store.get(key)
-      if (file) stored.set(key, file)
-      else missing.push(key)
+      if (!file) missing.push(key)
+      else if (withMedia) stored.set(key, file)
+      else described.set(key, { size: file.size, etag: file.etag })
     }
 
   const site = {
     ...set,
     manifest: {
       ...set.manifest,
+      blobs: set.manifest.blobs.map((blob) => {
+        const found = described.get(blob.key)
+        return found ? { ...blob, size: found.size, etag: found.etag } : blob
+      }),
       label: `${config.siteName} (whole site)`,
       provenance: { ...set.manifest.provenance, cmsVersion: VERSION },
       scope: 'site' as const,
-      ...(withMedia ? {} : { media: 'omitted' as const }),
+      ...(withMedia
+        ? {}
+        : {
+            media: options.media === 'referenced' ? ('referenced' as const) : ('omitted' as const),
+          }),
       requires: requiredPackages(config),
     },
   }
-  const seal = people && options.passphrase ? newSeal(options.passphrase) : undefined
+  const seal =
+    (people || options.sealContent) && options.passphrase ? newSeal(options.passphrase) : undefined
   const planned = planStreamedBundle(
     site,
     new Map([...stored].map(([key, file]) => [key, file.size])),
     sections,
-    { people, seal },
+    { people, seal, content: options.sealContent ? 'sealed' : 'open' },
   )
 
   const name = options.package?.name ?? packageNameFor(`${config.siteName}-site`)
@@ -213,6 +239,14 @@ export async function exportSite(
         await writer.add(`${PAYLOAD_DIR}/${path}`, body)
       } else {
         const media = stored.get(blobs.get(path) as string) as StoredFile
+        if (planned.sealing && seal) {
+          const sealed = sealBytes(seal.key, path, await media.bytes())
+          hasher.update(sealed)
+          hasher.update('\n')
+          await writer.add(`${PAYLOAD_DIR}/${path}`, sealed)
+          bytes += media.size
+          continue
+        }
         async function* hashed() {
           for await (const chunk of streamOf(media)) {
             hasher.update(chunk)
@@ -233,7 +267,7 @@ export async function exportSite(
     await writer.close()
     await rename(partial, options.out)
   } catch (error) {
-    await writer.close().catch(() => {})
+    await writer.abort()
     await rm(partial, { force: true })
     throw error
   }

package/src/tar.ts

@@ -148,6 +148,15 @@ export class TarballWriter {
     this.#gzip.end()
     await this.#done
   }
+
+  /**
+   * Gives up on a half-written file: the stream ends now, and its error is
+   * swallowed, so nothing surfaces later after the caller has cleaned up.
+   */
+  async abort(): Promise<void> {
+    this.#gzip.destroy()
+    await this.#done.catch(() => {})
+  }
 }
 
 const decoder = new TextDecoder()