@crombolo/server 0.13.2 → 0.14.4

package/src/schema-store.ts

@@ -111,6 +111,22 @@ export async function materialiseSchema(store: SchemaStore, dir: string): Promis
   return keys
 }
 
+/**
+ * Points a config at its store's schema: materialises the store into
+ * `schemaCacheDir` and makes that the schema directory. A no-op without a store;
+ * answers how many files were copied.
+ */
+export async function useSchemaStore(config: {
+  schemaStore: SchemaStore | undefined
+  schemaCacheDir: string
+  schemaDir: string
+}): Promise<number | undefined> {
+  if (!config.schemaStore) return undefined
+  const copied = await materialiseSchema(config.schemaStore, config.schemaCacheDir)
+  config.schemaDir = config.schemaCacheDir
+  return copied.length
+}
+
 export interface PublishReport {
   written: string[]
   removed: string[]

package/src/schema.ts

@@ -43,7 +43,7 @@ import { listComponents } from './components.ts'
 import { type CromboloConfig, VERSION } from './config.ts'
 import { formTypesFor } from './form-types.ts'
 import { logger } from './logging.ts'
-import { materialiseSchema } from './schema-store.ts'
+import { useSchemaStore } from './schema-store.ts'
 
 export interface SchemaBoot {
   /** The state this node runs at: the files' version, the deploy's revision. */
@@ -397,13 +397,11 @@ export async function importSchema(
   db: Db,
   config: CromboloConfig,
   boot: { nodeId: string; nodeState: NodeSchemaState },
-  options: { dryRun?: boolean } = {},
+  options: { dryRun?: boolean; materialise?: boolean } = {},
 ): Promise<SchemaImportResult> {
-  let materialised: number | undefined
-  if (config.schemaStore) {
-    materialised = (await materialiseSchema(config.schemaStore, config.schemaCacheDir)).length
-    config.schemaDir = config.schemaCacheDir
-  }
+  // A caller that has just written into the directory says not to, or the store's
+  // older copy would be laid over its write.
+  const materialised = options.materialise === false ? undefined : await useSchemaStore(config)
   if (!existsSync(config.schemaDir)) {
     return { materialised, classification: 'none', findings: [], report: undefined }
   }

package/src/server.ts

@@ -31,7 +31,9 @@ import {
   createExtensionRegistry,
   isSpaRoute,
   renderBackOfficeShell,
+  renderDeviceShell,
   renderLoginShell,
+  renderRemoteApprovalShell,
   resolveGraphic,
   resolveLoginAsset,
   resolveStaticFile,
@@ -46,6 +48,7 @@ import {
   hasAccessToSensitiveData,
   localReturnUrl,
   MANAGEMENT_API_PREFIX,
+  ObjectTypes,
   PREVIEW_HUB_PATH,
   SERVER_EVENT_HUB_PATH,
 } from '@crombolo/core'
@@ -53,6 +56,7 @@ import {
   appendCacheInstruction,
   createTablesCapability,
   type Db,
+  ensureKeyValue,
   FormEntryRepository,
   FormWorkflowRepository,
   PublicAccessRepository,
@@ -60,7 +64,9 @@ import {
   readLedger,
   readReport,
   readUrlOptions,
+  SiteSearch,
   TransferRunRepository,
+  UserGroupRepository,
   type WritesPaused,
 } from '@crombolo/data'
 import {
@@ -69,6 +75,7 @@ import {
   FORM_SCRIPT,
   FORM_SCRIPT_PATH,
   FORM_SUBMIT_PATH,
+  MEMBER_EXTERNAL_SIGN_IN_PATH,
   PublishedCache,
   Renderer,
   type SnapshotStatus,
@@ -90,7 +97,13 @@ import { CommandError, installBundleContent } from './bundle-install.ts'
 import { bundleChains, bundleTablesReport } from './bundle-tables.ts'
 import { createBundleRedirects, createServerBundles, validateServerBundles } from './bundles.ts'
 import { busyResponse } from './busy.ts'
-import { type CromboloConfig, loadConfig, type ServerRole, VERSION } from './config.ts'
+import {
+  type CromboloConfig,
+  listeningBodyLimit,
+  loadConfig,
+  type ServerRole,
+  VERSION,
+} from './config.ts'
 import { attachDatabase, bootstrapDatabase } from './database.ts'
 import { createDeliveryMemberSecurity } from './delivery-members.ts'
 import { createDiagnostics } from './diagnostics.ts'
@@ -131,6 +144,8 @@ import { noNodesPage } from './no-nodes.ts'
 import { NotImplementedLog } from './not-implemented.ts'
 import { createDeps, PREVIEW_COOKIE, resolveUserLinkSender } from './ports.ts'
 import { createRedirectTracker } from './redirects.ts'
+import { createRemote } from './remote.ts'
+import { withRemoteMedia } from './remote-media.ts'
 import { handleTransferAction, pendingTransfers } from './transfer-dashboard.ts'
 import {
   changeUrlOptions,
@@ -141,6 +156,12 @@ import {
 
 export { PREVIEW_COOKIE }
 
+import { BACKOFFICE_EXTERNAL_PATH, backofficeAudience } from './backoffice-external.ts'
+import { createExternalSignIn } from './external-sign-in.ts'
+import {
+  createBundleIdentityProviders,
+  IdentityProviderOverrides,
+} from './identity-provider-overrides.ts'
 import { syncConfiguredRedirects } from './redirects.ts'
 import {
   bootSchema,
@@ -149,7 +170,7 @@ import {
   type SchemaBoot,
   watchSchema,
 } from './schema.ts'
-import { materialiseSchema, publishSchema, type SchemaStore } from './schema-store.ts'
+import { publishSchema, type SchemaStore, useSchemaStore } from './schema-store.ts'
 import { createSeoHost, originOf } from './seo.ts'
 import {
   combineHubs,
@@ -158,6 +179,8 @@ import {
   type ServerEventHub,
 } from './server-events.ts'
 import { createSetupRoutes } from './setup.ts'
+import { siteSearchFor } from './site-search.ts'
+import { TwoFactorService, USER_TWO_FACTOR_PROVIDER } from './two-factor.ts'
 import { summariseUpgrades } from './upgrade-summary.ts'
 import { assertViewRuntime } from './view-runtime.ts'
 import { bundleEventProblems, createWebhookService, webhookStoreView } from './webhooks.ts'
@@ -305,15 +328,13 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
   // A configured schema store is copied down before anything reads `schema/`, and
   // becomes the schema directory for this boot. Everything after here — loading,
   // validation, hashing, sync, the writer — works on files exactly as it always has.
-  if (config.schemaStore) {
-    const copied = await materialiseSchema(config.schemaStore, config.schemaCacheDir)
-    config.schemaDir = config.schemaCacheDir
+  const copied = await useSchemaStore(config)
+  if (config.schemaStore)
     schemaLog.info('Schema read from {store}: {count} files in {dir}', {
       store: config.schemaStore.description,
-      count: copied.length,
+      count: copied,
       dir: config.schemaCacheDir,
     })
-  }
   // A node that does not own the schema still loads it — the renderer needs the
   // property definitions — but writes none of it.
   const schema = await bootSchema(db, owns ? config : { ...config, syncSchemaAtBoot: false })
@@ -338,10 +359,29 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     clientKey: (request) =>
       peers.get(request) ?? request.headers.get('x-forwarded-for') ?? 'unknown',
   })
+  // Story 032: a user with the authenticator on is asked for a code after the password.
+  const twoFactor = new TwoFactorService(db, {
+    secretKey: config.secretKey,
+    issuer: config.siteName,
+  })
+  let twoFactorKey: Promise<string> | undefined
   const authRoutes = createAuthRoutes({
     backOfficePath: paths.backOfficePath,
     service: auth,
     cookies,
+    twoFactor: {
+      providersFor: (userKey) => twoFactor.providers(userKey),
+      verify: (userKey, provider, code) =>
+        provider === USER_TWO_FACTOR_PROVIDER
+          ? twoFactor.verify(userKey, provider, code)
+          : Promise.resolve(false),
+      secret: () => {
+        twoFactorKey ??= ensureKeyValue(db, 'crombolo/twoFactorPendingKey', () =>
+          crypto.randomUUID(),
+        )
+        return twoFactorKey
+      },
+    },
     onAudit: (event, request) =>
       recordAudit(db, {
         category: 'sign-in',
@@ -356,6 +396,47 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
         ...clientAddress(request),
       }),
   })
+  // Story 031: backoffice users signing in through an identity provider, ending
+  // in the session the login endpoint would have set.
+  const backofficeExternal =
+    config.backofficeIdentityProviders.length === 0
+      ? undefined
+      : createExternalSignIn(
+          db,
+          backofficeAudience(db, {
+            auth,
+            providers: config.backofficeIdentityProviders,
+            backOfficePath: paths.backOfficePath,
+            sessionCookie: (sessionId) =>
+              serializeCookie({
+                name: names.session,
+                value: sessionId,
+                secure: config.secureCookies,
+                maxAgeSeconds: auth.settings.refreshTokenSeconds,
+              }),
+            clientAddress: (request) => clientAddress(request).ipAddress ?? undefined,
+            onSignIn: (user, request) =>
+              recordAudit(db, {
+                category: 'sign-in',
+                action: 'SignIn',
+                source: 'request',
+                userKey: user.key,
+                userName: user.userName,
+                entityType: 'User',
+                entityKey: user.key,
+                entityName: user.userName,
+                detail: 'Through an identity provider',
+                ...clientAddress(request),
+              }),
+          }),
+          {
+            applicationUrl: config.applicationUrl,
+            secureCookies: config.secureCookies,
+            log: (message) => log.warning('Backoffice sign-in: {message}', { message }),
+            resolve: (provider) =>
+              new IdentityProviderOverrides(db, config.secretKey).apply('user', provider),
+          },
+        )
 
   /**
    * Resolves the caller from the reference access token. The client sends
@@ -506,6 +587,25 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
             },
           },
           log: bundleLog('bundle'),
+          identityProviders: createBundleIdentityProviders({
+            overrides: new IdentityProviderOverrides(db, config.secretKey),
+            members: config.memberIdentityProviders,
+            backoffice: config.backofficeIdentityProviders,
+            memberCallbackPath: MEMBER_EXTERNAL_SIGN_IN_PATH,
+            backofficeCallbackPath: BACKOFFICE_EXTERNAL_PATH,
+            groups: async (audience) =>
+              audience === 'user'
+                ? (await new UserGroupRepository(db).all()).map((g) => ({
+                    value: g.alias,
+                    name: g.name,
+                  }))
+                : (
+                    await db.query<{ text: string }>(
+                      'SELECT text FROM node WHERE node_object_type = ? ORDER BY text',
+                      [ObjectTypes.MemberGroup],
+                    )
+                  ).map((g) => ({ value: String(g.text), name: String(g.text) })),
+          }),
         },
         log: (id) => bundleLog(id),
       })
@@ -566,6 +666,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
           },
         )
       : undefined
+  const remote = createRemote({ db, config, authenticate })
   const members = createMemberAuth(db, config, {
     nodeState: schema.nodeState,
     nodeId: schema.nodeId,
@@ -589,6 +690,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
           protection: () => new PublicAccessRepository(db).allRules(),
           member: (token) => deliveryMembers.member(token),
           security: (request, action) => deliveryMembers.handle(request, action),
+          search: (term, options) => new SiteSearch(db).query(term, options),
         })
 
   /** A document's latest saved version, as preview shows it, in the published tree's context. */
@@ -637,7 +739,11 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
   })
   // The site's config may name a store; without one the environment decides,
   // and that falls back to the media directory on this disk.
-  const mediaStore = config.mediaStore ?? (await mediaStoreFromEnvironment(config.mediaDir))
+  // A site pulled with `--lazy-media` reads files through from its remote (story 108).
+  const mediaStore = withRemoteMedia(
+    config.mediaStore ?? (await mediaStoreFromEnvironment(config.mediaDir)),
+    config.siteDir,
+  )
   // Resolved once: constructing it twice would warn twice about a provider that
   // is half-configured, and the adapters are cheap but not free of side effects.
   const email = resolveEmailPort(config)
@@ -877,6 +983,12 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
       if (response) return response
     }
 
+    // The remote protocol (story 107), where the backoffice is: it acts as a person.
+    if (servesBackOffice) {
+      const response = await remote.handle(request, url)
+      if (response) return response
+    }
+
     if (servesBackOffice) {
       const response = await serveBackOffice(request, url)
       if (response) {
@@ -1593,6 +1705,8 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
       if (response) return response
     }
 
+    const externalResponse = await backofficeExternal?.handle(request, pathname)
+    if (externalResponse) return externalResponse
     const authResponse = await authRoutes.handle(request, pathname)
     if (authResponse) return authResponse
 
@@ -1605,6 +1719,51 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
       return api.dispatch(request)
     }
 
+    // Approving a command-line sign-in (story 107): signed in first, then back here.
+    if (pathname === `${paths.backOfficePath}/device` && request.method === 'GET') {
+      const sessionId = readCookie(request, names.session)
+      if (!sessionId || !(await auth.resolveSession(sessionId))) {
+        const back = `${pathname}${url.search}`
+        return new Response(null, {
+          status: 303,
+          headers: {
+            location: `${paths.backOfficePath}/login?returnUrl=${encodeURIComponent(back)}`,
+          },
+        })
+      }
+      const { importmap } = collectManifests(paths, extensions.list())
+      return html(
+        renderDeviceShell(paths, importmap, {
+          title: config.siteName,
+          userCode: url.searchParams.get('user_code') ?? undefined,
+        }),
+      )
+    }
+
+    // Approving a push to a protected environment (story 107): the page reads
+    // and decides the job through the remote API, as the signed-in person.
+    if (
+      pathname === `${paths.backOfficePath}/remote-approval` &&
+      request.method === 'GET' &&
+      config.remote.enabled
+    ) {
+      const sessionId = readCookie(request, names.session)
+      if (!sessionId || !(await auth.resolveSession(sessionId)))
+        return new Response(null, {
+          status: 303,
+          headers: {
+            location: `${paths.backOfficePath}/login?returnUrl=${encodeURIComponent(`${pathname}${url.search}`)}`,
+          },
+        })
+      const { importmap } = collectManifests(paths, extensions.list())
+      return html(
+        renderRemoteApprovalShell(paths, importmap, {
+          title: config.siteName,
+          job: url.searchParams.get('job') ?? '',
+        }),
+      )
+    }
+
     if (pathname === `${paths.backOfficePath}/login`) {
       const { importmap } = collectManifests(paths, extensions.list())
       return html(
@@ -1623,6 +1782,14 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
           // the session exists, so anything else is an open redirect at the worst
           // possible moment, and `javascript:` there would run signed in.
           returnUrl: localReturnUrl(url.searchParams.get('returnUrl')) ?? paths.backOfficePath,
+          externalProviders: config.backofficeIdentityProviders.map((provider) => ({
+            name: provider.oidc.name,
+            url:
+              backofficeExternal?.startUrl(
+                provider.oidc.id,
+                localReturnUrl(url.searchParams.get('returnUrl')) ?? paths.backOfficePath,
+              ) ?? '',
+          })),
         }),
       )
     }
@@ -1731,7 +1898,10 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     if (!key) return undefined
     const draft = await draftModel(key, new URL(request.url).searchParams.get('culture'))
     if (!draft) return undefined
-    return renderer.renderModel(draft.content, draft.culture, await pageRequest(request))
+    return renderer.renderModel(draft.content, draft.culture, {
+      ...(await pageRequest(request)),
+      search: siteSearchFor(db, cache, undefined),
+    })
   }
 
   /** What every rendered page is told about the request and the site's search defaults. */
@@ -1880,6 +2050,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
         member,
         submission,
         access: (content) => members.decide(content, member),
+        search: siteSearchFor(db, cache, member),
         ...(await pageRequest(request)),
       }))
     const cleared: Record<string, string> = cookie
@@ -2003,11 +2174,24 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     jobs,
     serveOptions: {
       port: config.port,
-      maxRequestBodySize: config.maxRequestBodyBytes,
+      maxRequestBodySize: listeningBodyLimit(config),
       fetch: diagnostics.wrap(async (request, bunServer: Server<HubSocketData>) => {
         const peer = bunServer.requestIP(request)?.address
         if (peer) peers.set(request, peer)
-        const pathname = new URL(request.url).pathname
+        const requestUrl = new URL(request.url)
+        const pathname = requestUrl.pathname
+        // Listening with the remote's larger limit (story 107), so every other
+        // request is held to the usual one here.
+        if (
+          listeningBodyLimit(config) > config.maxRequestBodyBytes &&
+          !remote.isUpload(request, requestUrl)
+        ) {
+          const length = Number(request.headers.get('content-length') ?? 0)
+          if (length > config.maxRequestBodyBytes)
+            return new Response('Payload Too Large', { status: 413 })
+          if (!request.headers.has('content-length') && request.headers.has('transfer-encoding'))
+            return new Response('Length Required', { status: 411 })
+        }
         if (pathname === hubPath || pathname === previewHubPath) {
           const principal = await authenticate(request)
           const hub = pathname === hubPath ? events : previewHub