@crombolo/server 0.13.2 → 0.14.4

package/src/identity-provider-overrides.ts

@@ -0,0 +1,283 @@
+/**
+ * Settings an administrator changed for an identity provider the config
+ * declares (story 033): kept in `key_value`, one row per provider and audience,
+ * with the client secret sealed by `CROMBOLO_SECRET_KEY`. The config's preset is
+ * the provider; these are only the fields changed, put on top of it at every
+ * sign-in, so a change reaches every node with nothing to invalidate.
+ */
+import type { Db, ExternalLoginAudience } from '@crombolo/data'
+import { entraIssuer, type IdentityProvider } from './identity-providers.ts'
+import { openSecret, SECRET_KEY_VARIABLE, sealSecret, usableSecretKey } from './secrets.ts'
+
+/** The fields an override may change. */
+export interface ProviderOverrides {
+  name?: string
+  tenantId?: string
+  issuer?: string
+  clientId?: string
+  autoCreate?: boolean
+  linkByEmail?: boolean
+  emailsVerified?: boolean
+  groupMap?: Record<string, string>
+  defaultGroups?: string[]
+}
+
+/** What an administrator sends: the fields to change, and a client secret to set (`null` clears it). */
+export interface ProviderOverrideInput extends ProviderOverrides {
+  clientSecret?: string | null
+}
+
+interface Stored extends ProviderOverrides {
+  clientSecretSealed?: string
+}
+
+const OVERRIDABLE: readonly (keyof ProviderOverrides)[] = [
+  'name',
+  'tenantId',
+  'issuer',
+  'clientId',
+  'autoCreate',
+  'linkByEmail',
+  'emailsVerified',
+  'groupMap',
+  'defaultGroups',
+]
+
+const keyOf = (audience: ExternalLoginAudience, id: string) =>
+  `crombolo:identity-provider:${audience}:${id}`
+
+/** The provider the config declares, with an administrator's changes on top. Pure. */
+export function effectiveProvider(
+  code: IdentityProvider,
+  overrides: ProviderOverrides & { clientSecret?: string },
+): IdentityProvider {
+  const tenantId = overrides.tenantId ?? code.tenantId
+  const issuer =
+    overrides.issuer ??
+    (code.kind === 'entra' && overrides.tenantId
+      ? entraIssuer(overrides.tenantId)
+      : code.oidc.issuer)
+  return {
+    ...code,
+    ...(tenantId ? { tenantId } : {}),
+    oidc: {
+      ...code.oidc,
+      name: overrides.name ?? code.oidc.name,
+      issuer,
+      clientId: overrides.clientId ?? code.oidc.clientId,
+      clientSecret: overrides.clientSecret ?? code.oidc.clientSecret,
+    },
+    autoCreate: overrides.autoCreate ?? code.autoCreate,
+    linkByEmail: overrides.linkByEmail ?? code.linkByEmail,
+    emailsVerified: overrides.emailsVerified ?? code.emailsVerified,
+    groupMap: overrides.groupMap ?? code.groupMap,
+    defaultGroups: overrides.defaultGroups ?? code.defaultGroups,
+  }
+}
+
+export class IdentityProviderOverrides {
+  #db: Db
+  #secretKey: string | undefined
+
+  constructor(db: Db, secretKey: string | undefined) {
+    this.#db = db
+    this.#secretKey = usableSecretKey(secretKey)
+  }
+
+  async #read(audience: ExternalLoginAudience, id: string): Promise<Stored> {
+    const rows = await this.#db.query<{ value: string | null }>(
+      'SELECT value FROM key_value WHERE key = ?',
+      [keyOf(audience, id)],
+    )
+    try {
+      return rows[0]?.value ? (JSON.parse(rows[0].value) as Stored) : {}
+    } catch {
+      return {}
+    }
+  }
+
+  /** The fields changed, with whether a client secret is kept here; never the secret itself. */
+  async get(
+    audience: ExternalLoginAudience,
+    id: string,
+  ): Promise<{ overrides: ProviderOverrides; hasClientSecret: boolean }> {
+    const { clientSecretSealed, ...overrides } = await this.#read(audience, id)
+    return { overrides, hasClientSecret: Boolean(clientSecretSealed) }
+  }
+
+  /** A provider as a sign-in uses it: the config's, with the changes and a kept secret on top. */
+  async apply(audience: ExternalLoginAudience, code: IdentityProvider): Promise<IdentityProvider> {
+    const { clientSecretSealed, ...overrides } = await this.#read(audience, code.oidc.id)
+    if (Object.keys(overrides).length === 0 && !clientSecretSealed) return code
+    let clientSecret: string | undefined
+    if (clientSecretSealed && this.#secretKey) {
+      try {
+        clientSecret = openSecret(clientSecretSealed, this.#secretKey)
+      } catch {
+        clientSecret = undefined
+      }
+    }
+    return effectiveProvider(code, { ...overrides, clientSecret })
+  }
+
+  /** Changes some fields. An empty string or an empty value clears a field back to the config's. */
+  async update(
+    audience: ExternalLoginAudience,
+    id: string,
+    input: ProviderOverrideInput,
+  ): Promise<{ ok: true } | { ok: false; error: string }> {
+    const stored = await this.#read(audience, id)
+    for (const field of OVERRIDABLE) {
+      if (!(field in input)) continue
+      const value = input[field]
+      if (value === undefined || value === null || value === '') delete stored[field]
+      else (stored as Record<string, unknown>)[field] = value
+    }
+    if (input.clientSecret === null || input.clientSecret === '') delete stored.clientSecretSealed
+    else if (typeof input.clientSecret === 'string') {
+      if (!this.#secretKey)
+        return {
+          ok: false,
+          error: `Keeping a client secret here needs ${SECRET_KEY_VARIABLE} set on the server.`,
+        }
+      stored.clientSecretSealed = sealSecret(input.clientSecret, this.#secretKey)
+    }
+    await this.#write(audience, id, stored)
+    return { ok: true }
+  }
+
+  /** Puts a provider back to what the config says. */
+  async reset(audience: ExternalLoginAudience, id: string): Promise<void> {
+    await this.#db.exec('DELETE FROM key_value WHERE key = ?', [keyOf(audience, id)])
+  }
+
+  async #write(audience: ExternalLoginAudience, id: string, stored: Stored): Promise<void> {
+    const key = keyOf(audience, id)
+    const now = this.#db.dialect.now()
+    if (Object.keys(stored).length === 0) {
+      await this.#db.exec('DELETE FROM key_value WHERE key = ?', [key])
+      return
+    }
+    const value = JSON.stringify(stored)
+    const updated = await this.#db.query<{ key: string }>(
+      `UPDATE key_value SET value = ?, update_date = ${now} WHERE key = ? RETURNING key`,
+      [value, key],
+    )
+    if (updated.length === 0)
+      await this.#db.exec(`INSERT INTO key_value (key, value, update_date) VALUES (?, ?, ${now})`, [
+        key,
+        value,
+      ])
+  }
+}
+
+/** A provider as a bundle sees it: its effective settings and which of them were changed. */
+export interface BundleIdentityProvider {
+  audience: ExternalLoginAudience
+  id: string
+  kind: IdentityProvider['kind']
+  name: string
+  tenantId: string | null
+  issuer: string
+  clientId: string
+  /** Whether there is a client secret, from the config or kept here. Never the secret. */
+  hasClientSecret: boolean
+  autoCreate: boolean
+  linkByEmail: boolean
+  emailsVerified: boolean
+  groupMap: Record<string, string>
+  defaultGroups: string[]
+  /** The fields an administrator changed, `clientSecret` among them when one is kept here. */
+  overridden: string[]
+  /** The path to register with the provider as the redirect URL, after the site's address. */
+  callbackPath: string
+}
+
+/** The `identityProviders` bundle capability (story 033): edit, never declare. */
+export interface BundleIdentityProviders {
+  list(): Promise<BundleIdentityProvider[]>
+  update(
+    audience: ExternalLoginAudience,
+    id: string,
+    input: ProviderOverrideInput,
+  ): Promise<{ ok: true } | { ok: false; error: string }>
+  reset(audience: ExternalLoginAudience, id: string): Promise<boolean>
+  /** Whether the provider's discovery document answers, with what it says if not. */
+  test(audience: ExternalLoginAudience, id: string): Promise<{ ok: boolean; message: string }>
+  /** The groups a mapping can name: user groups by alias, member groups by name. */
+  groups(audience: ExternalLoginAudience): Promise<Array<{ value: string; name: string }>>
+}
+
+export function createBundleIdentityProviders(options: {
+  overrides: IdentityProviderOverrides
+  members: readonly IdentityProvider[]
+  backoffice: readonly IdentityProvider[]
+  memberCallbackPath: string
+  backofficeCallbackPath: string
+  groups(audience: ExternalLoginAudience): Promise<Array<{ value: string; name: string }>>
+}): BundleIdentityProviders {
+  const declared = (audience: ExternalLoginAudience, id: string) =>
+    (audience === 'member' ? options.members : options.backoffice).find((p) => p.oidc.id === id)
+
+  const describe = async (
+    audience: ExternalLoginAudience,
+    code: IdentityProvider,
+  ): Promise<BundleIdentityProvider> => {
+    const effective = await options.overrides.apply(audience, code)
+    const { overrides, hasClientSecret } = await options.overrides.get(audience, code.oidc.id)
+    const base = audience === 'member' ? options.memberCallbackPath : options.backofficeCallbackPath
+    return {
+      audience,
+      id: code.oidc.id,
+      kind: code.kind,
+      name: effective.oidc.name,
+      tenantId: effective.tenantId ?? null,
+      issuer: effective.oidc.issuer,
+      clientId: effective.oidc.clientId,
+      hasClientSecret: Boolean(effective.oidc.clientSecret),
+      autoCreate: effective.autoCreate,
+      linkByEmail: effective.linkByEmail,
+      emailsVerified: effective.emailsVerified,
+      groupMap: { ...effective.groupMap },
+      defaultGroups: [...effective.defaultGroups],
+      overridden: [...Object.keys(overrides), ...(hasClientSecret ? ['clientSecret'] : [])],
+      callbackPath: `${base}/${code.oidc.id}/callback`,
+    }
+  }
+
+  return {
+    async list() {
+      return Promise.all([
+        ...options.members.map((p) => describe('member', p)),
+        ...options.backoffice.map((p) => describe('user', p)),
+      ])
+    },
+    async update(audience, id, input) {
+      if (!declared(audience, id))
+        return { ok: false, error: 'The config declares no such identity provider.' }
+      return options.overrides.update(audience, id, input)
+    },
+    async reset(audience, id) {
+      if (!declared(audience, id)) return false
+      await options.overrides.reset(audience, id)
+      return true
+    },
+    groups: (audience) => options.groups(audience),
+    async test(audience, id) {
+      const code = declared(audience, id)
+      if (!code) return { ok: false, message: 'The config declares no such identity provider.' }
+      const effective = await options.overrides.apply(audience, code)
+      const url = `${effective.oidc.issuer.replace(/\/$/, '')}/.well-known/openid-configuration`
+      try {
+        const response = await fetch(url, { headers: { accept: 'application/json' } })
+        if (!response.ok) return { ok: false, message: `${url} answered ${response.status}.` }
+        const document = (await response.json()) as { authorization_endpoint?: unknown }
+        return typeof document.authorization_endpoint === 'string'
+          ? { ok: true, message: `${effective.oidc.name} answered at ${effective.oidc.issuer}.` }
+          : { ok: false, message: `${url} is not an OpenID Connect discovery document.` }
+      } catch (error) {
+        return { ok: false, message: `${url} could not be reached: ${(error as Error).message}` }
+      }
+    },
+  }
+}

package/src/identity-providers.ts

@@ -0,0 +1,185 @@
+/**
+ * External identity providers (story 031): typed presets a site's config lists,
+ * one list for members and one for backoffice users. Each preset knows its
+ * provider's issuer, scopes and claims; the rest says what a first sign-in may
+ * do and how groups follow the account.
+ *
+ * ```ts
+ * defineConfig({
+ *   backofficeIdentityProviders: [entra({ tenantId, clientId, clientSecret, groups: { map: { … } } })],
+ *   memberIdentityProviders: [awsIdentityCenter({ issuer, clientId, clientSecret })],
+ * })
+ * ```
+ */
+import type { OidcClaims, OidcProviderOptions } from '@crombolo/auth'
+
+/** What every preset takes besides the provider's own settings. */
+export interface IdentityProviderOptions {
+  /** The provider's id, in its URLs and in `external_login`; the preset's name by default. */
+  id?: string
+  /** What its sign-in button says: "Sign in with {name}". */
+  name?: string
+  /** Create a member or user for an account nobody has yet. On by default. */
+  autoCreate?: boolean
+  /**
+   * Link a first sign-in to an existing member or user with the same email,
+   * when the provider says the email is verified. Off by default: without it, an
+   * email already taken is refused rather than merged.
+   */
+  linkByEmail?: boolean
+  /**
+   * Treat every email this provider sends as verified. For a provider whose
+   * emails are the organisation's own, such as one Entra tenant, which sends no
+   * `email_verified` claim.
+   */
+  emailsVerified?: boolean
+  /**
+   * Groups that follow the account: claim values mapped to member groups (for
+   * members) or user group aliases (for users), made to match the claim on
+   * every sign-in. Groups outside the map are left as they are.
+   */
+  groups?: { claim?: string; map: Record<string, string> }
+  /** Groups a new member or user joins, besides the mapped ones. */
+  defaultGroups?: readonly string[]
+  /** Members only: the member type a new member is created as, by alias; `Member` by default. */
+  memberType?: string
+}
+
+/** Which preset made a provider: what the Entra bundle looks for. */
+export type IdentityProviderKind = 'entra' | 'aws' | 'oidc'
+
+/** A provider, ready for the config. */
+export interface IdentityProvider {
+  kind: IdentityProviderKind
+  /** Entra only: the tenant its issuer is for, which an override may change. */
+  tenantId?: string
+  oidc: OidcProviderOptions
+  autoCreate: boolean
+  linkByEmail: boolean
+  emailsVerified: boolean
+  groupMap: Readonly<Record<string, string>>
+  defaultGroups: readonly string[]
+  memberType: string | undefined
+}
+
+const ID = /^[a-z0-9][a-z0-9-]*$/
+
+/** Entra's issuer for a tenant. */
+export const entraIssuer = (tenantId: string) =>
+  `https://login.microsoftonline.com/${tenantId}/v2.0`
+
+function preset(
+  kind: IdentityProviderKind,
+  defaults: { id: string; name: string },
+  provider: { issuer: string; clientId: string; clientSecret?: string; scopes?: readonly string[] },
+  claims: OidcClaims,
+  options: IdentityProviderOptions,
+): IdentityProvider {
+  const id = options.id ?? defaults.id
+  if (!ID.test(id))
+    throw new Error(`An identity provider's id is letters, digits and dashes: "${id}" is not.`)
+  if (!provider.issuer) throw new Error(`The ${defaults.name} identity provider needs an issuer.`)
+  if (!provider.clientId)
+    throw new Error(`The ${defaults.name} identity provider needs a clientId.`)
+  return {
+    kind,
+    oidc: {
+      id,
+      name: options.name ?? defaults.name,
+      issuer: provider.issuer,
+      clientId: provider.clientId,
+      clientSecret: provider.clientSecret,
+      scopes: provider.scopes ?? ['openid', 'profile', 'email'],
+      // A group map with no claim named reads the usual `groups` claim.
+      claims: {
+        ...claims,
+        groups: options.groups?.claim ?? claims.groups ?? (options.groups ? 'groups' : undefined),
+      },
+    },
+    autoCreate: options.autoCreate ?? true,
+    linkByEmail: options.linkByEmail ?? false,
+    emailsVerified: options.emailsVerified ?? false,
+    groupMap: options.groups?.map ?? {},
+    defaultGroups: options.defaultGroups ?? [],
+    memberType: options.memberType,
+  }
+}
+
+/**
+ * Microsoft Entra ID, for one tenant. Register a web app with a client secret
+ * and the redirect URLs in `docs/operations.md`; for groups, add the `groups`
+ * claim to its token configuration.
+ */
+export function entra(
+  options: IdentityProviderOptions & { tenantId: string; clientId: string; clientSecret?: string },
+): IdentityProvider {
+  if (!options.tenantId) throw new Error('The Entra identity provider needs a tenantId.')
+  const provider = preset(
+    'entra',
+    { id: 'entra', name: 'Microsoft' },
+    {
+      issuer: entraIssuer(options.tenantId),
+      clientId: options.clientId,
+      clientSecret: options.clientSecret,
+    },
+    { email: 'email', name: 'name', groups: 'groups' },
+    options,
+  )
+  return { ...provider, tenantId: options.tenantId }
+}
+
+/**
+ * AWS IAM Identity Center, through the OpenID Connect application its
+ * instance issues; `issuer` is that instance's issuer URL.
+ */
+export function awsIdentityCenter(
+  options: IdentityProviderOptions & { issuer: string; clientId: string; clientSecret?: string },
+): IdentityProvider {
+  return preset(
+    'aws',
+    { id: 'aws', name: 'AWS' },
+    { issuer: options.issuer, clientId: options.clientId, clientSecret: options.clientSecret },
+    { email: 'email', name: 'name', groups: 'groups' },
+    options,
+  )
+}
+
+/** Any OpenID Connect provider, by its issuer: Google, Okta, Auth0, Keycloak and the like. */
+export function oidc(
+  options: IdentityProviderOptions & {
+    id: string
+    name: string
+    issuer: string
+    clientId: string
+    clientSecret?: string
+    scopes?: readonly string[]
+    claims?: Partial<OidcClaims>
+  },
+): IdentityProvider {
+  return preset(
+    'oidc',
+    { id: options.id, name: options.name },
+    {
+      issuer: options.issuer,
+      clientId: options.clientId,
+      clientSecret: options.clientSecret,
+      scopes: options.scopes,
+    },
+    { email: 'email', name: 'name', ...options.claims },
+    options,
+  )
+}
+
+/** Refuses two providers with one id in a list, which would share one set of URLs. */
+export function checkIdentityProviders(
+  list: readonly IdentityProvider[],
+  label: string,
+): readonly IdentityProvider[] {
+  const seen = new Set<string>()
+  for (const provider of list) {
+    if (seen.has(provider.oidc.id))
+      throw new Error(`Two ${label} identity providers have the id "${provider.oidc.id}".`)
+    seen.add(provider.oidc.id)
+  }
+  return list
+}

package/src/index.ts

@@ -12,6 +12,7 @@ export * from './adapters/public-access.ts'
 export * from './adapters/redirects.ts'
 export * from './adapters/references.ts'
 export * from './adapters/template-files.ts'
+export * from './api-client.ts'
 export * from './backup.ts'
 export * from './boot-failure.ts'
 export * from './bundle-install.ts'
@@ -34,6 +35,7 @@ export * from './email-postmark.ts'
 export * from './email-resend.ts'
 export * from './email-ses.ts'
 export * from './extensions-notice.ts'
+export * from './external-sign-in.ts'
 export * from './file-intake.ts'
 export * from './form-files.ts'
 export * from './form-submit.ts'
@@ -43,6 +45,8 @@ export * from './forms.ts'
 export * from './git.ts'
 export * from './git-github.ts'
 export * from './git-routes.ts'
+export * from './identity-provider-overrides.ts'
+export * from './identity-providers.ts'
 export * from './imaging.ts'
 export * from './jobs.ts'
 export * from './listen.ts'
@@ -60,6 +64,9 @@ export * from './no-nodes.ts'
 export * from './oembed.ts'
 export * from './ports.ts'
 export * from './redirects.ts'
+export * from './remote.ts'
+export * from './remote-credentials.ts'
+export * from './remote-media.ts'
 export * from './schema.ts'
 export * from './schema-store.ts'
 export * from './secrets.ts'
@@ -67,8 +74,13 @@ export * from './seo.ts'
 export * from './server.ts'
 export * from './server-events.ts'
 export * from './setup.ts'
+export * from './site-file.ts'
+export * from './site-replace.ts'
+export * from './site-sections.ts'
 export * from './status.ts'
+export * from './tar.ts'
 export * from './transfer-dashboard.ts'
+export * from './two-factor.ts'
 export * from './udt.ts'
 export * from './upgrade-summary.ts'
 export * from './url-settings.ts'

package/src/media-store-azure.ts

@@ -156,13 +156,19 @@ export function azureMediaStore(options: AzureMediaStoreOptions): MediaStore {
           if (!response.ok) throw new Error(`Azure GET ${key} failed with ${response.status}.`)
           return new Uint8Array(await response.arrayBuffer())
         },
+        stream: async () => {
+          const response = await send('GET', url)
+          if (!response.ok || !response.body)
+            throw new Error(`Azure GET ${key} failed with ${response.status}.`)
+          return response.body
+        },
       }
     },
 
     async put(key, body, putOptions) {
       if (!isSafeMediaKey(key)) throw new Error(`"${key}" is not a usable media key.`)
       // Copied into a plain ArrayBuffer so `fetch` and the signature agree on its length.
-      const bytes = new Uint8Array(body instanceof File ? await body.arrayBuffer() : body)
+      const bytes = new Uint8Array(body instanceof Blob ? await body.arrayBuffer() : body)
       const response = await send('PUT', blobUrl(key), {
         body: bytes,
         headers: {

package/src/media-store-s3.ts

@@ -61,6 +61,7 @@ export function s3MediaStore(options: S3MediaStoreOptions): MediaStore {
           etag: stat.etag.replaceAll('"', ''),
           contentType: stat.type || contentTypeFor(key),
           bytes: async () => new Uint8Array(await file.arrayBuffer()),
+          stream: async () => file.stream(),
         }
       } catch {
         // A missing object is a thrown NoSuchKey, not a null stat.

package/src/media-store.ts

@@ -13,6 +13,7 @@
  *   `ab12cd34/photo.jpg`        a placed file, served at `/media/ab12cd34/photo.jpg`
  *   `.temp/<uuid>/photo.jpg`    an upload not yet placed, with `meta.json` beside it
  *   `.cache/ab/<hash>.webp`     a processed image variant
+ *   `.remote/<job>/site.tgz`    a remote pull's or push's file (story 107), never public
  *
  * A site picks a store in `crombolo.config.ts` — `fileSystemMediaStore`,
  * `s3MediaStore` or `azureMediaStore` — or leaves it to the environment, which
@@ -31,6 +32,8 @@ export interface StoredFile {
   etag: string
   contentType: string
   bytes(): Promise<Uint8Array>
+  /** The bytes as a stream, for a file too big to hold (story 104); `streamOf` falls back to `bytes()`. */
+  stream?(): Promise<ReadableStream<Uint8Array>>
   /** A path on this machine, when the store has one; lets the server stream it. */
   localPath?: string
 }
@@ -39,7 +42,7 @@ export interface MediaStore {
   /** Named in the boot banner and in logs: `media/`, `s3://bucket`, … */
   readonly description: string
   get(key: string): Promise<StoredFile | undefined>
-  put(key: string, body: Uint8Array | File, options?: { contentType?: string }): Promise<void>
+  put(key: string, body: Uint8Array | Blob, options?: { contentType?: string }): Promise<void>
   delete(key: string): Promise<void>
   /** Every key under a prefix, which is how temporary uploads are swept. */
   list(prefix: string): Promise<string[]>
@@ -51,6 +54,24 @@ export interface MediaStore {
   publicUrl?(key: string): string | undefined
 }
 
+/** A stored file's bytes in chunks: streamed where the store can, whole where it cannot. */
+export async function* streamOf(file: StoredFile): AsyncIterable<Uint8Array> {
+  if (!file.stream) {
+    yield await file.bytes()
+    return
+  }
+  const reader = (await file.stream()).getReader()
+  try {
+    for (;;) {
+      const { done, value } = await reader.read()
+      if (done) return
+      yield value
+    }
+  } finally {
+    reader.releaseLock()
+  }
+}
+
 /** Content types by extension, for stores that do not infer one. */
 const CONTENT_TYPES: Record<string, string> = {
   avif: 'image/avif',
@@ -91,7 +112,8 @@ export function isSafeMediaKey(key: string): boolean {
       part !== '' &&
       part !== '.' &&
       part !== '..' &&
-      (!part.startsWith('.') || (index === 0 && (part === '.temp' || part === '.cache'))),
+      (!part.startsWith('.') ||
+        (index === 0 && (part === '.temp' || part === '.cache' || part === '.remote'))),
   )
 }
 
@@ -139,6 +161,7 @@ export function fileSystemMediaStore(
         etag: `${stat.size.toString(16)}-${Math.trunc(stat.mtimeMs).toString(16)}`,
         contentType: file.type || contentTypeFor(key),
         bytes: () => file.bytes(),
+        stream: async () => file.stream(),
         localPath: path,
       }
     },