@crombolo/transfer 0.13.2 → 0.14.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crombolo/transfer",
-  "version": "0.13.2",
+  "version": "0.14.4",
   "description": "Content transfer for crombolo: bundles of content moved between environments.",
   "license": "MIT",
   "author": "Crombolo",
@@ -28,7 +28,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@crombolo/core": "0.13.2",
-    "@crombolo/data": "0.13.2"
+    "@crombolo/core": "0.14.4",
+    "@crombolo/data": "0.14.4"
   }
 }

package/src/import.ts

@@ -71,6 +71,11 @@ export interface ImportOptions extends CheckOptions {
   label?: string | undefined
   /** The npm package the bundle arrived in, recorded on the run. */
   package?: { name: string; version: string } | undefined
+  /**
+   * A node's values become exactly the bundle's, rather than the bundle's laid
+   * over what is here: a whole-site replace (story 105).
+   */
+  exact?: boolean
 }
 
 export interface ImportResult {
@@ -342,7 +347,7 @@ async function applyBundle(tx: Db, set: ContentSet, options: ImportOptions): Pro
           ? ((await templates.byAlias(node.template))?.key ?? current.componentKey)
           : current.componentKey,
         parentKey: current.parentKey,
-        values: overlay(current.values, taken),
+        values: overlay(options.exact ? [] : current.values, taken),
         variants: keepNames
           ? current.variants.map((v) => ({
               culture: v.culture,

package/src/index.ts

@@ -9,5 +9,6 @@ export * from './people-apply.ts'
 export * from './references.ts'
 export * from './resolutions.ts'
 export * from './revert.ts'
+export * from './sanitise.ts'
 export * from './seal.ts'
 export * from './write.ts'

package/src/load.ts

@@ -257,6 +257,7 @@ function parseManifest(text: string, problems: BundleProblem[]): BundleManifest
       schemaVersion: str(provenance.schemaVersion) ?? '',
       schemaRevision: str(provenance.schemaRevision) ?? '',
       schemaHash: nullableStr(provenance.schemaHash) ?? null,
+      ...(str(provenance.cmsVersion) ? { cmsVersion: str(provenance.cmsVersion) as string } : {}),
     },
     selector: {
       roots: keys(selector.roots),
@@ -313,9 +314,20 @@ function parseManifest(text: string, problems: BundleProblem[]): BundleManifest
       : {}),
     ...(parseSealParameters(raw.seal) ? { seal: parseSealParameters(raw.seal) } : {}),
     ...(raw.content === 'sealed' || raw.content === 'omitted' ? { content: raw.content } : {}),
+    ...(raw.scope === 'site' ? { scope: 'site' as const } : {}),
+    ...(raw.media === 'omitted' || raw.media === 'referenced' ? { media: raw.media } : {}),
+    ...(isRecord(raw.requires) ? { requires: parseRequires(raw.requires) } : {}),
   }
 }
 
+/** Package names to versions, keeping only string pairs. */
+function parseRequires(raw: Record<string, unknown>): Record<string, string> {
+  const out: Record<string, string> = {}
+  for (const [name, version] of Object.entries(raw))
+    if (typeof version === 'string') out[name] = version
+  return out
+}
+
 /**
  * The declared sections, dropping any name this version does not know and any
  * path that tries to climb out of the bundle — a manifest is a file that

package/src/model.ts

@@ -172,6 +172,8 @@ export interface BundleProvenance {
   schemaRevision: string
   schemaHash: string | null
   siteName: string
+  /** The CMS version that wrote the bundle (story 104). Absent on bundles written before it. */
+  cmsVersion?: string
 }
 
 export interface BundleSelector {
@@ -228,6 +230,25 @@ export interface BundleManifest {
    * Absent: in the open.
    */
   content?: BundleContentMode
+  /**
+   * `site`: a whole site (story 104) — every node, the whole schema and every
+   * file — which an import replaces a site with rather than overlays. Absent: a
+   * selection, overlaid as always.
+   */
+  scope?: 'site'
+  /**
+   * `omitted`: a whole-site file written without its media files (`--no-media`).
+   * `referenced`: without them because they move one at a time between the
+   * stores of a remote and the CLI (story 107): every blob is named, none is in
+   * the file, and the file is no use away from its remote.
+   */
+  media?: 'omitted' | 'referenced'
+  /**
+   * The server bundles the site wires, by package name, at the versions that
+   * wrote it. An import refuses when the destination lacks one or has an older
+   * version, naming it.
+   */
+  requires?: Record<string, string>
 }
 
 export type BundleContentMode = 'sealed' | 'omitted'

package/src/sanitise.ts

@@ -0,0 +1,121 @@
+/**
+ * Anonymising a bundle before it is written (story 104, reused by 100's clone):
+ * the structure travels, the people in it do not. It runs on what the export
+ * gathered and before anything is serialised, so an unsanitised copy never
+ * exists in the output.
+ *
+ * - Back-office users are removed; the destination makes one fresh local
+ *   administrator. User groups stay, as structure.
+ * - Members stay as placeholders — type, groups and approval kept — with name,
+ *   username and email replaced, the password and every property value gone.
+ * - Every email address in content, people and the dictionary is replaced, the
+ *   same address by the same stand-in each time so references still agree.
+ *
+ * Tokens, sessions, external logins and form entries are never exported, so
+ * there is nothing of theirs to remove. Code and schema are left alone: an
+ * address in a template is the site's, not a person's.
+ */
+import type { BundleNode, BundleValue, ContentSet } from './model.ts'
+import type { BundleMember, BundlePeople } from './people.ts'
+
+const EMAIL = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g
+
+/** The domain stand-ins use: reserved, so nothing is ever delivered to it. */
+export const ANONYMISED_DOMAIN = 'example.invalid'
+
+export interface SanitiseInput {
+  set: ContentSet
+  people?: BundlePeople
+  /** The dictionary `.udt`, whose translations may hold addresses. */
+  dictionary?: string
+}
+
+export interface SanitiseResult {
+  set: ContentSet
+  people: BundlePeople | undefined
+  dictionary: string | undefined
+  /** How many distinct email addresses were replaced. */
+  emails: number
+}
+
+export function sanitise(input: SanitiseInput): SanitiseResult {
+  const standIns = new Map<string, string>()
+  const standInFor = (address: string): string => {
+    const known = address.toLowerCase()
+    let replacement = standIns.get(known)
+    if (!replacement) {
+      replacement = `person-${standIns.size + 1}@${ANONYMISED_DOMAIN}`
+      standIns.set(known, replacement)
+    }
+    return replacement
+  }
+  const scrub = (text: string) =>
+    text.replace(EMAIL, (address) =>
+      address.toLowerCase().endsWith(`@${ANONYMISED_DOMAIN}`) ? address : standInFor(address),
+    )
+  const deep = (value: unknown): unknown => {
+    if (typeof value === 'string') return scrub(value)
+    if (Array.isArray(value)) return value.map(deep)
+    if (value && typeof value === 'object')
+      return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, deep(v)]))
+    return value
+  }
+
+  const usernames = new Map<string, string>()
+  const node = (n: BundleNode): BundleNode => ({
+    ...n,
+    variants: deep(n.variants) as BundleNode['variants'],
+    values: n.values.map((v: BundleValue) => ({ ...v, value: deep(v.value) })),
+    ...(n.access
+      ? {
+          access: {
+            ...n.access,
+            // A protected page naming a member names their stand-in; one naming
+            // someone who did not travel names nobody.
+            ...(n.access.memberUserNames
+              ? {
+                  memberUserNames: n.access.memberUserNames.flatMap((name) => {
+                    const standIn = usernames.get(name.toLowerCase())
+                    return standIn ? [standIn] : []
+                  }),
+                }
+              : {}),
+          },
+        }
+      : {}),
+  })
+
+  const member = (m: BundleMember, index: number): BundleMember => {
+    const n = index + 1
+    standIns.set(m.email.toLowerCase(), `member-${n}@${ANONYMISED_DOMAIN}`)
+    usernames.set(m.username.toLowerCase(), `member-${n}`)
+    return {
+      ...m,
+      name: `Member ${n}`,
+      username: `member-${n}`,
+      email: `member-${n}@${ANONYMISED_DOMAIN}`,
+      passwordHash: null,
+      passwordConfig: null,
+      values: [],
+    }
+  }
+
+  // Members first, so an address that belongs to one maps to their stand-in
+  // wherever content mentions it.
+  const people = input.people
+    ? {
+        memberGroups: input.people.memberGroups,
+        members: input.people.members.map(member),
+        userGroups: input.people.userGroups,
+        users: [],
+      }
+    : undefined
+  for (const user of input.people?.users ?? []) standInFor(user.email)
+
+  return {
+    set: { ...input.set, nodes: input.set.nodes.map(node) },
+    people,
+    dictionary: input.dictionary === undefined ? undefined : scrub(input.dictionary),
+    emails: standIns.size,
+  }
+}

package/src/write.ts

@@ -157,6 +157,7 @@ export function writeManifest(manifest: BundleManifest): string {
       schemaVersion: manifest.provenance.schemaVersion,
       schemaRevision: manifest.provenance.schemaRevision,
       schemaHash: manifest.provenance.schemaHash,
+      ...(manifest.provenance.cmsVersion ? { cmsVersion: manifest.provenance.cmsVersion } : {}),
     },
     selector: {
       roots: manifest.selector.roots,
@@ -193,6 +194,11 @@ export function writeManifest(manifest: BundleManifest): string {
     ...(manifest.people ? { people: manifest.people } : {}),
     ...(manifest.seal ? { seal: manifest.seal } : {}),
     ...(manifest.content ? { content: manifest.content } : {}),
+    ...(manifest.scope ? { scope: manifest.scope } : {}),
+    ...(manifest.media ? { media: manifest.media } : {}),
+    ...(manifest.requires && Object.keys(manifest.requires).length > 0
+      ? { requires: Object.fromEntries(Object.entries(manifest.requires).sort()) }
+      : {}),
   })
 }
 
@@ -289,13 +295,52 @@ export function writeBundle(
   sections: readonly BundleFile[] = [],
   options: WriteOptions = {},
 ): BundleFile[] {
+  const sizes = new Map([...blobs].map(([key, bytes]) => [key, bytes.length]))
+  const planned = planBundle(set, sizes, sections, options)
+  const carried: BundleFile[] = planned.blobs.map((key) => {
+    const path = blobPath(key)
+    const bytes = blobs.get(key) as Uint8Array
+    return {
+      path,
+      bytes: planned.sealing && options.seal ? sealBytes(options.seal.key, path, bytes) : bytes,
+    }
+  })
+  const manifest = {
+    ...planned.manifest,
+    integrity: bundleIntegrity([...planned.files, ...carried]),
+  }
+  return [{ path: MANIFEST_FILE, text: writeManifest(manifest) }, ...planned.files, ...carried]
+}
+
+/**
+ * A bundle whose media files the caller streams (story 104): the manifest and
+ * every other file, from each carried file's size alone. The caller writes
+ * `blobs` at `blobPath` — sealed with `sealBytes` under the seal when `sealing`
+ * — computes `bundleIntegrity` over those and `files` in path order, and writes
+ * the manifest with it.
+ */
+export function planStreamedBundle(
+  set: ContentSet,
+  blobSizes: ReadonlyMap<string, number>,
+  sections: readonly BundleFile[] = [],
+  options: WriteOptions = {},
+): { manifest: BundleManifest; files: BundleFile[]; blobs: string[]; sealing: boolean } {
+  return planBundle(set, blobSizes, sections, { ...options, content: options.content ?? 'open' })
+}
+
+function planBundle(
+  set: ContentSet,
+  blobSizes: ReadonlyMap<string, number>,
+  sections: readonly BundleFile[],
+  options: WriteOptions,
+): { manifest: BundleManifest; files: BundleFile[]; blobs: string[]; sealing: boolean } {
   const seal = options.seal
   const content = options.content ?? (seal ? 'sealed' : 'open')
   if ((content === 'sealed' || options.people) && !seal)
     throw new Error('Sealed content and people need a seal: give a passphrase.')
   const sealing = content === 'sealed'
   const source = content === 'omitted' ? set.nodes.map(withoutContent) : set.nodes
-  const carriedBlobs: ReadonlyMap<string, Uint8Array> = content === 'omitted' ? new Map() : blobs
+  const carriedSizes: ReadonlyMap<string, number> = content === 'omitted' ? new Map() : blobSizes
 
   const sorted = [...source].sort((a, b) => a.key.localeCompare(b.key))
   const nodes: BundleFile[] = sorted.map((node) => ({
@@ -309,12 +354,6 @@ export function writeBundle(
           return { path, bytes: sealBytes(seal.key, path, Buffer.from(json(sealedValues(node)))) }
         })
       : []
-  const carried: BundleFile[] = [...carriedBlobs]
-    .sort(([a], [b]) => a.localeCompare(b))
-    .map(([key, bytes]) => {
-      const path = blobPath(key)
-      return { path, bytes: sealing && seal ? sealBytes(seal.key, path, bytes) : bytes }
-    })
 
   // A file in no known section would travel without being declared, so it
   // would not be hashed and a reader would never look for it. Refused here
@@ -345,27 +384,19 @@ export function writeBundle(
           : CONTENT_ONLY_FORMAT_VERSION,
     // Without values nothing names a media file, so none is expected either.
     blobs: (content === 'omitted' ? [] : set.manifest.blobs).map((blob) => {
-      const bytes = carriedBlobs.get(blob.key)
-      return bytes ? { ...blob, included: true, size: bytes.length } : { ...blob, included: false }
+      const size = carriedSizes.get(blob.key)
+      return size !== undefined ? { ...blob, included: true, size } : { ...blob, included: false }
     }),
     carries: extra.length > 0 ? carries : undefined,
     people: people?.block,
     seal: seal && (sealing || people) ? seal.params : undefined,
     content: content === 'open' ? undefined : content,
-    integrity: bundleIntegrity([
-      ...nodes,
-      ...sealedFiles,
-      ...carried,
-      ...extra,
-      ...(people?.files ?? []),
-    ]),
+    integrity: '',
+  }
+  return {
+    manifest,
+    files: [...nodes, ...sealedFiles, ...extra, ...(people?.files ?? [])],
+    blobs: [...carriedSizes.keys()].sort((a, b) => a.localeCompare(b)),
+    sealing,
   }
-  return [
-    { path: MANIFEST_FILE, text: writeManifest(manifest) },
-    ...nodes,
-    ...sealedFiles,
-    ...carried,
-    ...extra,
-    ...(people?.files ?? []),
-  ]
 }