@crombolo/server 0.13.2 → 0.14.1

package/src/ports.ts

@@ -12,6 +12,7 @@ import type {
   ServerPort,
   TemporaryFilePort,
   UserItemPort,
+  UserTwoFactorPort,
 } from '@crombolo/api-management'
 import { AuthStore, expireCookie, serializeCookie } from '@crombolo/auth'
 import {
@@ -84,6 +85,7 @@ import { createOEmbedService } from './oembed.ts'
 import { createRedirectTracker } from './redirects.ts'
 import { componentAliasesIn, type SchemaBoot } from './schema.ts'
 import { publishSchema } from './schema-store.ts'
+import { TwoFactorService, USER_TWO_FACTOR_PROVIDER } from './two-factor.ts'
 import { createWebhookService, type WebhookService } from './webhooks.ts'
 
 /** Re-exported for tests; the canonical list lives in @crombolo/core. */
@@ -492,7 +494,22 @@ export function createDeps(options: DepsOptions): ManagementApiDeps {
         await store.endAllSessionsForUser(userId)
       },
     })
-    deps.userManagement = userPorts.users
+    // Story 032: the authenticator provider, offered only where secrets can be sealed.
+    const twoFactor = new TwoFactorService(options.db, {
+      secretKey: options.config.secretKey,
+      issuer: options.config.siteName,
+    })
+    const configuration = userPorts.users.configuration
+    const currentConfiguration = userPorts.users.currentConfiguration
+    deps.userManagement = {
+      ...userPorts.users,
+      configuration: () => ({ ...configuration(), allowTwoFactor: twoFactor.available }),
+      currentConfiguration: () => ({
+        ...currentConfiguration(),
+        allowTwoFactor: twoFactor.available,
+      }),
+      twoFactor: userTwoFactorPort(twoFactor),
+    }
     deps.userGroups = userPorts.groups
     deps.userData = userPorts.userData
     deps.media = createMediaPort(options.db, mediaFiles, {
@@ -621,3 +638,40 @@ export function createDeps(options: DepsOptions): ManagementApiDeps {
   }
   return deps
 }
+
+/** The user operations' view of two-factor sign-in: the one authenticator provider. */
+function userTwoFactorPort(service: TwoFactorService): UserTwoFactorPort {
+  const provider = USER_TWO_FACTOR_PROVIDER
+  return {
+    providerName: provider,
+    async list(userKey) {
+      return [
+        {
+          providerName: provider,
+          isEnabledOnUser: (await service.providers(userKey)).includes(provider),
+        },
+      ]
+    },
+    setup(account) {
+      const setup = service.setup(account)
+      return setup.ok
+        ? {
+            ok: true,
+            secret: setup.value.secret,
+            qrCodeSetupImageUrl: setup.value.qrCodeSetupImageUrl,
+          }
+        : { ok: false, message: setup.message }
+    },
+    async enable(userKey, code, secret) {
+      const enabled = await service.enable(userKey, provider, secret, code)
+      return enabled.ok
+        ? { ok: true, recoveryCodes: enabled.value.recoveryCodes }
+        : { ok: false, message: enabled.message }
+    },
+    async disable(userKey, code) {
+      const disabled = await service.disable(userKey, provider, code)
+      return disabled.ok ? { ok: true } : { ok: false, message: disabled.message }
+    },
+    reset: (userKey) => service.reset(userKey, provider),
+  }
+}

package/src/schema-store.ts

@@ -111,6 +111,22 @@ export async function materialiseSchema(store: SchemaStore, dir: string): Promis
   return keys
 }
 
+/**
+ * Points a config at its store's schema: materialises the store into
+ * `schemaCacheDir` and makes that the schema directory. A no-op without a store;
+ * answers how many files were copied.
+ */
+export async function useSchemaStore(config: {
+  schemaStore: SchemaStore | undefined
+  schemaCacheDir: string
+  schemaDir: string
+}): Promise<number | undefined> {
+  if (!config.schemaStore) return undefined
+  const copied = await materialiseSchema(config.schemaStore, config.schemaCacheDir)
+  config.schemaDir = config.schemaCacheDir
+  return copied.length
+}
+
 export interface PublishReport {
   written: string[]
   removed: string[]

package/src/schema.ts

@@ -43,7 +43,7 @@ import { listComponents } from './components.ts'
 import { type CromboloConfig, VERSION } from './config.ts'
 import { formTypesFor } from './form-types.ts'
 import { logger } from './logging.ts'
-import { materialiseSchema } from './schema-store.ts'
+import { useSchemaStore } from './schema-store.ts'
 
 export interface SchemaBoot {
   /** The state this node runs at: the files' version, the deploy's revision. */
@@ -397,13 +397,11 @@ export async function importSchema(
   db: Db,
   config: CromboloConfig,
   boot: { nodeId: string; nodeState: NodeSchemaState },
-  options: { dryRun?: boolean } = {},
+  options: { dryRun?: boolean; materialise?: boolean } = {},
 ): Promise<SchemaImportResult> {
-  let materialised: number | undefined
-  if (config.schemaStore) {
-    materialised = (await materialiseSchema(config.schemaStore, config.schemaCacheDir)).length
-    config.schemaDir = config.schemaCacheDir
-  }
+  // A caller that has just written into the directory says not to, or the store's
+  // older copy would be laid over its write.
+  const materialised = options.materialise === false ? undefined : await useSchemaStore(config)
   if (!existsSync(config.schemaDir)) {
     return { materialised, classification: 'none', findings: [], report: undefined }
   }

package/src/server.ts

@@ -46,6 +46,7 @@ import {
   hasAccessToSensitiveData,
   localReturnUrl,
   MANAGEMENT_API_PREFIX,
+  ObjectTypes,
   PREVIEW_HUB_PATH,
   SERVER_EVENT_HUB_PATH,
 } from '@crombolo/core'
@@ -53,6 +54,7 @@ import {
   appendCacheInstruction,
   createTablesCapability,
   type Db,
+  ensureKeyValue,
   FormEntryRepository,
   FormWorkflowRepository,
   PublicAccessRepository,
@@ -60,7 +62,9 @@ import {
   readLedger,
   readReport,
   readUrlOptions,
+  SiteSearch,
   TransferRunRepository,
+  UserGroupRepository,
   type WritesPaused,
 } from '@crombolo/data'
 import {
@@ -69,6 +73,7 @@ import {
   FORM_SCRIPT,
   FORM_SCRIPT_PATH,
   FORM_SUBMIT_PATH,
+  MEMBER_EXTERNAL_SIGN_IN_PATH,
   PublishedCache,
   Renderer,
   type SnapshotStatus,
@@ -141,6 +146,12 @@ import {
 
 export { PREVIEW_COOKIE }
 
+import { BACKOFFICE_EXTERNAL_PATH, backofficeAudience } from './backoffice-external.ts'
+import { createExternalSignIn } from './external-sign-in.ts'
+import {
+  createBundleIdentityProviders,
+  IdentityProviderOverrides,
+} from './identity-provider-overrides.ts'
 import { syncConfiguredRedirects } from './redirects.ts'
 import {
   bootSchema,
@@ -149,7 +160,7 @@ import {
   type SchemaBoot,
   watchSchema,
 } from './schema.ts'
-import { materialiseSchema, publishSchema, type SchemaStore } from './schema-store.ts'
+import { publishSchema, type SchemaStore, useSchemaStore } from './schema-store.ts'
 import { createSeoHost, originOf } from './seo.ts'
 import {
   combineHubs,
@@ -158,6 +169,8 @@ import {
   type ServerEventHub,
 } from './server-events.ts'
 import { createSetupRoutes } from './setup.ts'
+import { siteSearchFor } from './site-search.ts'
+import { TwoFactorService, USER_TWO_FACTOR_PROVIDER } from './two-factor.ts'
 import { summariseUpgrades } from './upgrade-summary.ts'
 import { assertViewRuntime } from './view-runtime.ts'
 import { bundleEventProblems, createWebhookService, webhookStoreView } from './webhooks.ts'
@@ -305,15 +318,13 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
   // A configured schema store is copied down before anything reads `schema/`, and
   // becomes the schema directory for this boot. Everything after here — loading,
   // validation, hashing, sync, the writer — works on files exactly as it always has.
-  if (config.schemaStore) {
-    const copied = await materialiseSchema(config.schemaStore, config.schemaCacheDir)
-    config.schemaDir = config.schemaCacheDir
+  const copied = await useSchemaStore(config)
+  if (config.schemaStore)
     schemaLog.info('Schema read from {store}: {count} files in {dir}', {
       store: config.schemaStore.description,
-      count: copied.length,
+      count: copied,
       dir: config.schemaCacheDir,
     })
-  }
   // A node that does not own the schema still loads it — the renderer needs the
   // property definitions — but writes none of it.
   const schema = await bootSchema(db, owns ? config : { ...config, syncSchemaAtBoot: false })
@@ -338,10 +349,29 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     clientKey: (request) =>
       peers.get(request) ?? request.headers.get('x-forwarded-for') ?? 'unknown',
   })
+  // Story 032: a user with the authenticator on is asked for a code after the password.
+  const twoFactor = new TwoFactorService(db, {
+    secretKey: config.secretKey,
+    issuer: config.siteName,
+  })
+  let twoFactorKey: Promise<string> | undefined
   const authRoutes = createAuthRoutes({
     backOfficePath: paths.backOfficePath,
     service: auth,
     cookies,
+    twoFactor: {
+      providersFor: (userKey) => twoFactor.providers(userKey),
+      verify: (userKey, provider, code) =>
+        provider === USER_TWO_FACTOR_PROVIDER
+          ? twoFactor.verify(userKey, provider, code)
+          : Promise.resolve(false),
+      secret: () => {
+        twoFactorKey ??= ensureKeyValue(db, 'crombolo/twoFactorPendingKey', () =>
+          crypto.randomUUID(),
+        )
+        return twoFactorKey
+      },
+    },
     onAudit: (event, request) =>
       recordAudit(db, {
         category: 'sign-in',
@@ -356,6 +386,47 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
         ...clientAddress(request),
       }),
   })
+  // Story 031: backoffice users signing in through an identity provider, ending
+  // in the session the login endpoint would have set.
+  const backofficeExternal =
+    config.backofficeIdentityProviders.length === 0
+      ? undefined
+      : createExternalSignIn(
+          db,
+          backofficeAudience(db, {
+            auth,
+            providers: config.backofficeIdentityProviders,
+            backOfficePath: paths.backOfficePath,
+            sessionCookie: (sessionId) =>
+              serializeCookie({
+                name: names.session,
+                value: sessionId,
+                secure: config.secureCookies,
+                maxAgeSeconds: auth.settings.refreshTokenSeconds,
+              }),
+            clientAddress: (request) => clientAddress(request).ipAddress ?? undefined,
+            onSignIn: (user, request) =>
+              recordAudit(db, {
+                category: 'sign-in',
+                action: 'SignIn',
+                source: 'request',
+                userKey: user.key,
+                userName: user.userName,
+                entityType: 'User',
+                entityKey: user.key,
+                entityName: user.userName,
+                detail: 'Through an identity provider',
+                ...clientAddress(request),
+              }),
+          }),
+          {
+            applicationUrl: config.applicationUrl,
+            secureCookies: config.secureCookies,
+            log: (message) => log.warning('Backoffice sign-in: {message}', { message }),
+            resolve: (provider) =>
+              new IdentityProviderOverrides(db, config.secretKey).apply('user', provider),
+          },
+        )
 
   /**
    * Resolves the caller from the reference access token. The client sends
@@ -506,6 +577,25 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
             },
           },
           log: bundleLog('bundle'),
+          identityProviders: createBundleIdentityProviders({
+            overrides: new IdentityProviderOverrides(db, config.secretKey),
+            members: config.memberIdentityProviders,
+            backoffice: config.backofficeIdentityProviders,
+            memberCallbackPath: MEMBER_EXTERNAL_SIGN_IN_PATH,
+            backofficeCallbackPath: BACKOFFICE_EXTERNAL_PATH,
+            groups: async (audience) =>
+              audience === 'user'
+                ? (await new UserGroupRepository(db).all()).map((g) => ({
+                    value: g.alias,
+                    name: g.name,
+                  }))
+                : (
+                    await db.query<{ text: string }>(
+                      'SELECT text FROM node WHERE node_object_type = ? ORDER BY text',
+                      [ObjectTypes.MemberGroup],
+                    )
+                  ).map((g) => ({ value: String(g.text), name: String(g.text) })),
+          }),
         },
         log: (id) => bundleLog(id),
       })
@@ -589,6 +679,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
           protection: () => new PublicAccessRepository(db).allRules(),
           member: (token) => deliveryMembers.member(token),
           security: (request, action) => deliveryMembers.handle(request, action),
+          search: (term, options) => new SiteSearch(db).query(term, options),
         })
 
   /** A document's latest saved version, as preview shows it, in the published tree's context. */
@@ -1593,6 +1684,8 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
       if (response) return response
     }
 
+    const externalResponse = await backofficeExternal?.handle(request, pathname)
+    if (externalResponse) return externalResponse
     const authResponse = await authRoutes.handle(request, pathname)
     if (authResponse) return authResponse
 
@@ -1623,6 +1716,14 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
           // the session exists, so anything else is an open redirect at the worst
           // possible moment, and `javascript:` there would run signed in.
           returnUrl: localReturnUrl(url.searchParams.get('returnUrl')) ?? paths.backOfficePath,
+          externalProviders: config.backofficeIdentityProviders.map((provider) => ({
+            name: provider.oidc.name,
+            url:
+              backofficeExternal?.startUrl(
+                provider.oidc.id,
+                localReturnUrl(url.searchParams.get('returnUrl')) ?? paths.backOfficePath,
+              ) ?? '',
+          })),
         }),
       )
     }
@@ -1731,7 +1832,10 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
     if (!key) return undefined
     const draft = await draftModel(key, new URL(request.url).searchParams.get('culture'))
     if (!draft) return undefined
-    return renderer.renderModel(draft.content, draft.culture, await pageRequest(request))
+    return renderer.renderModel(draft.content, draft.culture, {
+      ...(await pageRequest(request)),
+      search: siteSearchFor(db, cache, undefined),
+    })
   }
 
   /** What every rendered page is told about the request and the site's search defaults. */
@@ -1880,6 +1984,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
         member,
         submission,
         access: (content) => members.decide(content, member),
+        search: siteSearchFor(db, cache, member),
         ...(await pageRequest(request)),
       }))
     const cleared: Record<string, string> = cookie

package/src/site-file.ts

@@ -0,0 +1,258 @@
+/**
+ * A whole site in one file (story 104): every content node, media file, schema
+ * file, component, stylesheet, script and dictionary item, as one private npm
+ * package tarball — the bundle format with `scope: 'site'`. `docs/bundles.md`.
+ *
+ * Written as it goes. The media files stream from the store into the gzip, and
+ * the integrity hash is computed as each file passes, in the path order
+ * `bundleIntegrity` defines; only the small files are held. The tarball is
+ * written beside its destination and renamed into place once complete, so a
+ * failed export never leaves a file that looks whole.
+ */
+import { rename, rm } from 'node:fs/promises'
+import { discoverBundles } from '@crombolo/backoffice-host'
+import { type Db, NodeRepository, type NodeRow, ObjectTypes } from '@crombolo/data'
+import {
+  BUNDLE_SECTIONS,
+  type BundleFile,
+  type BundlePackageInfo,
+  blobPath,
+  bundleIdFor,
+  bundlePackageJson,
+  exportBundle,
+  exportPeople,
+  MANIFEST_FILE,
+  newSeal,
+  PAYLOAD_DIR,
+  packageNameFor,
+  planStreamedBundle,
+  sanitise,
+  writeManifest,
+} from '@crombolo/transfer'
+import { createFileSystemPort } from './adapters/file-system.ts'
+import { createTemplateFileStore } from './adapters/template-files.ts'
+import { type CromboloConfig, VERSION } from './config.ts'
+import { mediaStoreFor } from './media-files.ts'
+import { type StoredFile, streamOf } from './media-store.ts'
+import { siteSections } from './site-sections.ts'
+import { TarballWriter } from './tar.ts'
+
+export interface SiteExportOptions {
+  /** The `.tgz` to write. */
+  out: string
+  /** False for `--no-media`: the media nodes travel, their files do not. */
+  media?: boolean
+  /** Carry the latest drafts rather than what is published. */
+  drafts?: boolean
+  withMembers?: boolean
+  withUsers?: boolean
+  /** Replace every personal value before anything is written (`sanitise`). */
+  anonymise?: boolean
+  /** Seals the people; needed when any travel. */
+  passphrase?: string
+  package?: { name?: string; version?: string }
+}
+
+export interface SiteExportResult {
+  out: string
+  bundleId: string
+  package: { name: string; version: string }
+  counts: Record<string, number>
+  sections: Partial<Record<keyof typeof BUNDLE_SECTIONS, number>>
+  media: { carried: number; bytes: number; missing: string[]; omitted: boolean }
+  people: { members: number; users: number }
+  /** How many distinct email addresses `--anonymise` replaced. */
+  anonymised?: number
+}
+
+/** The content kinds a whole site carries, each from its tree's top. */
+const SITE_TREES = [
+  ObjectTypes.Document,
+  ObjectTypes.Media,
+  ObjectTypes.Element,
+  ObjectTypes.DocumentBlueprint,
+]
+
+async function topLevelNodes(db: Db): Promise<NodeRow[]> {
+  const nodes = new NodeRepository(db)
+  const roots: NodeRow[] = []
+  for (const tree of SITE_TREES) {
+    for (let skip = 0; ; skip += 500) {
+      const page = await nodes.roots(tree, skip, 500)
+      roots.push(...page.items)
+      if (skip + 500 >= page.total) break
+    }
+  }
+  return roots
+}
+
+/** The server bundles the site wires, by package name and version. */
+function requiredPackages(config: CromboloConfig): Record<string, string> {
+  if (config.bundles.length === 0) return {}
+  const wired = new Set(config.bundles.map((bundle) => bundle.id))
+  const required: Record<string, string> = {}
+  for (const found of discoverBundles(config.siteDir).bundles)
+    if (wired.has(found.id)) required[found.packageName] = found.version
+  return required
+}
+
+export async function exportSite(
+  db: Db,
+  config: CromboloConfig,
+  options: SiteExportOptions,
+): Promise<SiteExportResult> {
+  const withMedia = options.media !== false
+  const carryMembers = Boolean(options.withMembers)
+  // Anonymised users are removed, so there are none to carry.
+  const carryUsers = Boolean(options.withUsers) && !options.anonymise
+  if ((carryMembers || carryUsers) && !options.passphrase)
+    throw new Error('Members and users travel sealed: give a passphrase.')
+
+  const exported = await exportBundle(db, {
+    roots: await topLevelNodes(db),
+    asGiven: [],
+    descendants: true,
+    snapshot: options.drafts ? 'drafts' : 'published',
+    blueprints: true,
+    withBlobs: withMedia,
+    siteName: config.siteName,
+    nodeId: config.nodeId,
+  })
+
+  let sections: BundleFile[] = await siteSections(
+    {
+      db,
+      componentFiles: createTemplateFileStore(config.componentsDir),
+      components: createFileSystemPort(config.componentsDir, {
+        extension: '.tsx',
+        rewrites: ['.cshtml'],
+      }),
+      stylesheets: createFileSystemPort(config.stylesheetsDir, { extension: '.css' }),
+      scripts: createFileSystemPort(config.scriptsDir, { extension: '.js' }),
+      schemaDir: config.schemaDir,
+    },
+    'all',
+  )
+
+  let set = exported.set
+  let people =
+    carryMembers || carryUsers
+      ? await exportPeople(db, { members: carryMembers, users: carryUsers })
+      : undefined
+  let anonymised: number | undefined
+  if (options.anonymise) {
+    const dictionaryPath = `${BUNDLE_SECTIONS.dictionary}.udt`
+    const dictionary = sections.find((file) => file.path === dictionaryPath)
+    const clean = sanitise({ set, people, dictionary: dictionary?.text })
+    set = clean.set
+    people = clean.people
+    anonymised = clean.emails
+    if (dictionary && clean.dictionary !== undefined)
+      sections = sections.map((file) =>
+        file.path === dictionaryPath ? { ...file, text: clean.dictionary } : file,
+      )
+  }
+
+  // Each media file is looked up once: its size goes in the manifest now, its
+  // bytes stream into the tarball later.
+  const store = await mediaStoreFor(config)
+  const stored = new Map<string, StoredFile>()
+  const missing: string[] = []
+  if (withMedia)
+    for (const key of exported.blobKeys) {
+      const file = await store.get(key)
+      if (file) stored.set(key, file)
+      else missing.push(key)
+    }
+
+  const site = {
+    ...set,
+    manifest: {
+      ...set.manifest,
+      label: `${config.siteName} (whole site)`,
+      provenance: { ...set.manifest.provenance, cmsVersion: VERSION },
+      scope: 'site' as const,
+      ...(withMedia ? {} : { media: 'omitted' as const }),
+      requires: requiredPackages(config),
+    },
+  }
+  const seal = people && options.passphrase ? newSeal(options.passphrase) : undefined
+  const planned = planStreamedBundle(
+    site,
+    new Map([...stored].map(([key, file]) => [key, file.size])),
+    sections,
+    { people, seal },
+  )
+
+  const name = options.package?.name ?? packageNameFor(`${config.siteName}-site`)
+  const info: BundlePackageInfo = {
+    name,
+    version: options.package?.version ?? '1.0.0',
+    id: bundleIdFor(name.replace(/^@[^/]+\//, '')),
+    label: site.manifest.label,
+    private: true,
+    peerRange: `>=${VERSION}`,
+  }
+
+  const partial = `${options.out}.partial`
+  const writer = TarballWriter.create(partial)
+  const encoder = new TextEncoder()
+  const hasher = new Bun.CryptoHasher('sha256')
+  let bytes = 0
+  try {
+    const small = new Map(planned.files.map((file) => [file.path, file]))
+    const blobs = new Map(planned.blobs.map((key) => [blobPath(key), key]))
+    const paths = [...small.keys(), ...blobs.keys()].sort((a, b) => a.localeCompare(b))
+    for (const path of paths) {
+      hasher.update(path)
+      hasher.update('\0')
+      const file = small.get(path)
+      if (file) {
+        const body = file.bytes ?? encoder.encode(file.text ?? '')
+        hasher.update(body)
+        await writer.add(`${PAYLOAD_DIR}/${path}`, body)
+      } else {
+        const media = stored.get(blobs.get(path) as string) as StoredFile
+        async function* hashed() {
+          for await (const chunk of streamOf(media)) {
+            hasher.update(chunk)
+            yield chunk
+          }
+        }
+        await writer.addStream(`${PAYLOAD_DIR}/${path}`, media.size, hashed())
+        bytes += media.size
+      }
+      hasher.update('\n')
+    }
+    const manifest = { ...planned.manifest, integrity: `sha256-${hasher.digest('hex')}` }
+    await writer.add(`${PAYLOAD_DIR}/${MANIFEST_FILE}`, encoder.encode(writeManifest(manifest)))
+    await writer.add(
+      'package.json',
+      encoder.encode(`${JSON.stringify(bundlePackageJson(info), null, 2)}\n`),
+    )
+    await writer.close()
+    await rename(partial, options.out)
+  } catch (error) {
+    await writer.close().catch(() => {})
+    await rm(partial, { force: true })
+    throw error
+  }
+
+  const counted: SiteExportResult['sections'] = {}
+  for (const file of sections) {
+    const section = (Object.keys(BUNDLE_SECTIONS) as (keyof typeof BUNDLE_SECTIONS)[]).find(
+      (s) => file.path === `${BUNDLE_SECTIONS[s]}.udt` || file.path.startsWith(`${s}/`),
+    )
+    if (section) counted[section] = (counted[section] ?? 0) + 1
+  }
+  return {
+    out: options.out,
+    bundleId: site.manifest.id,
+    package: { name: info.name, version: info.version },
+    counts: { ...site.manifest.counts },
+    sections: counted,
+    media: { carried: stored.size, bytes, missing, omitted: !withMedia },
+    people: { members: people?.members.length ?? 0, users: people?.users.length ?? 0 },
+    ...(anonymised !== undefined ? { anonymised } : {}),
+  }
+}