@crombolo/transfer 0.0.0-stage → 0.9.1

package/src/load.ts

@@ -0,0 +1,360 @@
+/**
+ * A bundle on disk → `ContentSet`, validated.
+ *
+ * Reading is strict about shape and about integrity, and says which file was
+ * wrong, because everything downstream assumes a bundle is well formed: the
+ * check reports on what it would do, and by then a malformed node file would be
+ * a crash rather than a finding.
+ */
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import {
+  BUNDLE_FORMAT_VERSION,
+  BUNDLE_SECTIONS,
+  type BundleKind,
+  type BundleManifest,
+  type BundleNode,
+  type BundleProblem,
+  type BundleSection,
+  type BundleValue,
+  type BundleVariant,
+  type ContentSet,
+} from './model.ts'
+import { blobPath, bundleIntegrity, MANIFEST_FILE, nodePath, sectionOf } from './write.ts'
+
+export interface LoadedBundle {
+  set: ContentSet | undefined
+  problems: BundleProblem[]
+  /**
+   * The carried section files, by their path inside the bundle, as paths on
+   * disk — the structure and views an install applies. Empty for a
+   * content-only bundle, which is every version 1 bundle.
+   */
+  files: Map<string, string>
+  /**
+   * The media files the bundle carries, by store key, as paths on disk.
+   *
+   * Paths rather than bytes: a bundle of a media library is as large as the
+   * library, and nothing here needs it in memory at once.
+   */
+  blobs: Map<string, string>
+}
+
+const KINDS: readonly BundleKind[] = ['document', 'media', 'element', 'blueprint']
+
+const isRecord = (v: unknown): v is Record<string, unknown> =>
+  typeof v === 'object' && v !== null && !Array.isArray(v)
+
+const str = (v: unknown): string | undefined => (typeof v === 'string' ? v : undefined)
+const nullableStr = (v: unknown): string | null | undefined =>
+  v === null ? null : typeof v === 'string' ? v : undefined
+
+function parseVariant(
+  raw: unknown,
+  file: string,
+  problems: BundleProblem[],
+): BundleVariant | undefined {
+  if (!isRecord(raw)) {
+    problems.push({ file, message: 'a variant is not an object' })
+    return undefined
+  }
+  const name = str(raw.name)
+  const culture = nullableStr(raw.culture)
+  const segment = nullableStr(raw.segment)
+  if (name === undefined || culture === undefined || segment === undefined) {
+    problems.push({ file, message: 'a variant needs culture, segment and name' })
+    return undefined
+  }
+  return { culture, segment, name, published: raw.published === true }
+}
+
+function parseValue(
+  raw: unknown,
+  file: string,
+  problems: BundleProblem[],
+): BundleValue | undefined {
+  if (!isRecord(raw)) {
+    problems.push({ file, message: 'a value is not an object' })
+    return undefined
+  }
+  const property = str(raw.property)
+  const editor = str(raw.editor)
+  const culture = nullableStr(raw.culture)
+  const segment = nullableStr(raw.segment)
+  if (
+    property === undefined ||
+    editor === undefined ||
+    culture === undefined ||
+    segment === undefined
+  ) {
+    problems.push({ file, message: 'a value needs property, editor, culture and segment' })
+    return undefined
+  }
+  const references = Array.isArray(raw.references)
+    ? raw.references.flatMap((r) => (typeof r === 'string' ? [r] : []))
+    : undefined
+  return { property, culture, segment, editor, value: raw.value, references }
+}
+
+export function parseNode(
+  text: string,
+  file: string,
+  problems: BundleProblem[],
+): BundleNode | undefined {
+  let raw: unknown
+  try {
+    raw = JSON.parse(text)
+  } catch (error) {
+    problems.push({ file, message: `not JSON: ${(error as Error).message}` })
+    return undefined
+  }
+  if (!isRecord(raw)) {
+    problems.push({ file, message: 'not an object' })
+    return undefined
+  }
+  const key = str(raw.key)
+  const kind = str(raw.kind) as BundleKind | undefined
+  const contentType = isRecord(raw.contentType) ? raw.contentType : undefined
+  const typeKey = str(contentType?.key)
+  const typeAlias = str(contentType?.alias)
+  if (!key) problems.push({ file, message: 'no key' })
+  if (!kind || !KINDS.includes(kind))
+    problems.push({ file, message: `kind must be one of ${KINDS.join(', ')}` })
+  if (!typeAlias) problems.push({ file, message: 'no contentType.alias' })
+  const parent = nullableStr(raw.parent)
+  if (parent === undefined) problems.push({ file, message: 'parent must be a key or null' })
+  if (!key || !kind || !KINDS.includes(kind) || !typeAlias || parent === undefined) return undefined
+
+  const variants = Array.isArray(raw.variants)
+    ? raw.variants.flatMap((v) => {
+        const parsed = parseVariant(v, file, problems)
+        return parsed ? [parsed] : []
+      })
+    : []
+  const values = Array.isArray(raw.values)
+    ? raw.values.flatMap((v) => {
+        const parsed = parseValue(v, file, problems)
+        return parsed ? [parsed] : []
+      })
+    : []
+  if (variants.length === 0) problems.push({ file, message: 'no variants: a node needs a name' })
+
+  return {
+    key,
+    kind,
+    contentType: { key: typeKey ?? '', alias: typeAlias },
+    parent,
+    sortOrder: typeof raw.sortOrder === 'number' ? raw.sortOrder : 0,
+    template: nullableStr(raw.template) ?? null,
+    variants,
+    values,
+  }
+}
+
+function parseManifest(text: string, problems: BundleProblem[]): BundleManifest | undefined {
+  let raw: unknown
+  try {
+    raw = JSON.parse(text)
+  } catch (error) {
+    problems.push({ file: MANIFEST_FILE, message: `not JSON: ${(error as Error).message}` })
+    return undefined
+  }
+  if (!isRecord(raw)) {
+    problems.push({ file: MANIFEST_FILE, message: 'not an object' })
+    return undefined
+  }
+  const formatVersion = typeof raw.formatVersion === 'number' ? raw.formatVersion : undefined
+  if (formatVersion === undefined) {
+    problems.push({ file: MANIFEST_FILE, message: 'no formatVersion' })
+    return undefined
+  }
+  if (formatVersion > BUNDLE_FORMAT_VERSION) {
+    problems.push({
+      file: MANIFEST_FILE,
+      message: `format ${formatVersion} is newer than this version understands (${BUNDLE_FORMAT_VERSION}) — upgrade crombolo to import it`,
+    })
+    return undefined
+  }
+  const provenance = isRecord(raw.provenance) ? raw.provenance : {}
+  const selector = isRecord(raw.selector) ? raw.selector : {}
+  const dependencies = isRecord(raw.dependencies) ? raw.dependencies : {}
+  const schema = isRecord(dependencies.schema) ? dependencies.schema : {}
+  const keys = (v: unknown): string[] =>
+    Array.isArray(v) ? v.flatMap((k) => (typeof k === 'string' ? [k] : [])) : []
+
+  return {
+    formatVersion,
+    id: str(raw.id) ?? '',
+    ...(str(raw.label) ? { label: str(raw.label) as string } : {}),
+    createdAt: str(raw.createdAt) ?? '',
+    createdBy: str(raw.createdBy) ?? '',
+    integrity: str(raw.integrity) ?? '',
+    snapshot: raw.snapshot === 'drafts' ? 'drafts' : 'published',
+    provenance: {
+      siteName: str(provenance.siteName) ?? '',
+      schemaVersion: str(provenance.schemaVersion) ?? '',
+      schemaRevision: str(provenance.schemaRevision) ?? '',
+      schemaHash: nullableStr(provenance.schemaHash) ?? null,
+    },
+    selector: {
+      roots: keys(selector.roots),
+      descendants: selector.descendants !== false,
+      asGiven: keys(selector.asGiven),
+    },
+    counts: isRecord(raw.counts) ? (raw.counts as BundleManifest['counts']) : {},
+    dependencies: {
+      carried: keys(dependencies.carried),
+      expected: Array.isArray(dependencies.expected)
+        ? dependencies.expected.flatMap((e) =>
+            isRecord(e) && typeof e.key === 'string'
+              ? [
+                  {
+                    key: e.key,
+                    objectType: str(e.objectType) ?? '',
+                    name: str(e.name) ?? '',
+                    why: str(e.why) ?? '',
+                  },
+                ]
+              : [],
+          )
+        : [],
+      schema: {
+        contentTypes: Array.isArray(schema.contentTypes)
+          ? schema.contentTypes.flatMap((t) =>
+              isRecord(t) && typeof t.alias === 'string'
+                ? [{ key: str(t.key) ?? '', alias: t.alias }]
+                : [],
+            )
+          : [],
+        components: keys(schema.components),
+        languages: keys(schema.languages),
+      },
+    },
+    blobs: Array.isArray(raw.blobs)
+      ? raw.blobs.flatMap((b) =>
+          isRecord(b) && typeof b.key === 'string'
+            ? [
+                {
+                  key: b.key,
+                  node: str(b.node) ?? '',
+                  etag: nullableStr(b.etag) ?? null,
+                  size: typeof b.size === 'number' ? b.size : null,
+                  included: b.included === true,
+                },
+              ]
+            : [],
+        )
+      : [],
+    ...(isRecord(raw.carries) ? { carries: parseCarries(raw.carries) } : {}),
+  }
+}
+
+/**
+ * The declared sections, dropping any name this version does not know and any
+ * path that tries to climb out of the bundle — a manifest is a file that
+ * travelled, so what it names is read, never trusted.
+ */
+function parseCarries(raw: Record<string, unknown>): Partial<Record<BundleSection, string[]>> {
+  const out: Partial<Record<BundleSection, string[]>> = {}
+  for (const section of Object.keys(BUNDLE_SECTIONS) as BundleSection[]) {
+    const paths = raw[section]
+    if (!Array.isArray(paths)) continue
+    const safe = paths.flatMap((path) =>
+      typeof path === 'string' && sectionOf(path) === section && !path.split('/').includes('..')
+        ? [path]
+        : [],
+    )
+    if (safe.length > 0) out[section] = safe.sort()
+  }
+  return out
+}
+
+/** Reads a bundle directory. `set` is undefined when it cannot be trusted at all. */
+export function loadBundle(dir: string): LoadedBundle {
+  const problems: BundleProblem[] = []
+  const blobs = new Map<string, string>()
+  const files = new Map<string, string>()
+  const manifestFile = join(dir, MANIFEST_FILE)
+  if (!existsSync(manifestFile)) {
+    problems.push({ file: MANIFEST_FILE, message: `no bundle at ${dir}` })
+    return { set: undefined, problems, blobs, files }
+  }
+  const manifest = parseManifest(readFileSync(manifestFile, 'utf8'), problems)
+  if (!manifest) return { set: undefined, problems, blobs, files }
+
+  const nodesDir = join(dir, 'nodes')
+  const nodeFiles = existsSync(nodesDir)
+    ? readdirSync(nodesDir)
+        .filter((name) => name.endsWith('.json'))
+        .sort()
+    : []
+  const read: Array<{ path: string; text?: string; bytes?: Uint8Array }> = nodeFiles.map(
+    (name) => ({
+      path: `nodes/${name}`,
+      text: readFileSync(join(nodesDir, name), 'utf8'),
+    }),
+  )
+
+  // A blob the manifest says it carries has to be here, and has to be the file
+  // the hash was taken over: `included` is a claim, and this is what checks it.
+  for (const blob of manifest.blobs) {
+    if (!blob.included) continue
+    const path = blobPath(blob.key)
+    const file = join(dir, path)
+    if (!existsSync(file)) {
+      problems.push({
+        file: path,
+        message: 'the manifest says this file is carried, and it is not',
+      })
+      continue
+    }
+    blobs.set(blob.key, file)
+    read.push({ path, bytes: new Uint8Array(readFileSync(file)) })
+  }
+
+  // Every file the manifest says it carries has to be here, and has to be the
+  // file the hash was taken over — the same rule as a blob's `included`.
+  for (const [, paths] of Object.entries(manifest.carries ?? {})) {
+    for (const path of paths as string[]) {
+      const file = join(dir, path)
+      if (!existsSync(file)) {
+        problems.push({
+          file: path,
+          message: 'the manifest says this file is carried, and it is not',
+        })
+        continue
+      }
+      files.set(path, file)
+      read.push({ path, text: readFileSync(file, 'utf8') })
+    }
+  }
+
+  // Before shape: a truncated copy produces confusing field errors otherwise.
+  const integrity = bundleIntegrity(read)
+  if (manifest.integrity && manifest.integrity !== integrity)
+    problems.push({
+      file: MANIFEST_FILE,
+      message: `integrity does not match the node files (${manifest.integrity} vs ${integrity}) — the bundle is incomplete or was edited`,
+    })
+
+  const nodes = read.flatMap((file) => {
+    // Only `nodes/` holds nodes. Blobs have no text, and a carried section is a
+    // TOML or TSX file that has no business being parsed as one.
+    if (file.text === undefined || !file.path.startsWith('nodes/')) return []
+    const node = parseNode(file.text, file.path, problems)
+    if (node && nodePath(node.key) !== file.path)
+      problems.push({
+        file: file.path,
+        message: `holds node ${node.key}, so it should be named after it`,
+      })
+    return node ? [node] : []
+  })
+
+  const seen = new Set<string>()
+  for (const node of nodes) {
+    if (seen.has(node.key)) problems.push({ file: nodePath(node.key), message: 'appears twice' })
+    seen.add(node.key)
+  }
+
+  return { set: { manifest, nodes }, problems, blobs, files }
+}

package/src/model.ts

@@ -0,0 +1,194 @@
+/**
+ * The bundle: content selected out of one environment and reviewable on its way
+ * to another. `docs/13-content-transfer.md`.
+ *
+ * Everything travels by uuid and alias, never by integer id, because the ids
+ * differ between environments and the keys do not. A content type is matched by
+ * key first — tooling writes keys into `schema/*.toml`, so they are stable
+ * across environments — with the alias as the readable fallback.
+ */
+
+/** Bumped when a change to the shapes below stops an older reader understanding a bundle. */
+export const BUNDLE_FORMAT_VERSION = 2
+
+/**
+ * The version a bundle carrying only content is still written as.
+ *
+ * A content-only bundle is byte-for-byte what version 1 was, so it keeps saying
+ * 1 and an older node can still read it. Only a bundle that carries a section
+ * beyond its content needs a reader that knows about `carries`, and that one
+ * says 2 — which an older node refuses with a message naming the upgrade,
+ * rather than failing an integrity check it cannot explain.
+ */
+export const CONTENT_ONLY_FORMAT_VERSION = 1
+
+/**
+ * The sections a bundle can carry beyond its content, and the directory each
+ * one occupies.
+ *
+ * Logical names, not the destination's directory names: a site may hold its
+ * components anywhere (`componentsDir`), so the artifact names what a file *is*
+ * and the importer decides where it goes. A bundle written against one site's
+ * layout therefore installs into another's.
+ *
+ * There is one section for components, not two. Umbraco's split of templates
+ * and partial views was a Razor concern (`docs/05-rendering.md`), and a bundle
+ * that carried it would push the distinction back into every site that
+ * installed one.
+ */
+export const BUNDLE_SECTIONS = {
+  schema: 'schema',
+  components: 'components',
+  styles: 'styles',
+  scripts: 'scripts',
+  dictionary: 'dictionary',
+} as const
+
+export type BundleSection = keyof typeof BUNDLE_SECTIONS
+
+/**
+ * The kinds that travel. `member` is deliberately absent: members are personal
+ * data, and a bundle is a file that gets copied around and committed.
+ */
+export type BundleKind = 'document' | 'media' | 'element' | 'blueprint'
+
+/** Which snapshot of a node's values the bundle carries. */
+export type BundleSnapshot = 'published' | 'drafts'
+
+export interface BundleValue {
+  property: string
+  culture: string | null
+  segment: string | null
+  /** The editor that wrote it, so the destination can refuse a property whose editor changed. */
+  editor: string
+  value: unknown
+  /** Node keys this value refers to, extracted at export so the importer need not re-parse. */
+  references?: string[]
+}
+
+/**
+ * One culture (or the invariant `null`) of a node: its name and whether it was
+ * published at the source. Shaped to feed the repository's own variant input.
+ */
+export interface BundleVariant {
+  culture: string | null
+  segment: string | null
+  name: string
+  published: boolean
+}
+
+export interface BundleNode {
+  key: string
+  kind: BundleKind
+  contentType: { key: string; alias: string }
+  /** The parent's key, or null for a node at the tree root. */
+  parent: string | null
+  sortOrder: number
+  /** A component alias — the path under `components/`, so aliases travel, not keys. */
+  template: string | null
+  variants: BundleVariant[]
+  values: BundleValue[]
+}
+
+/** Something the bundle refers to but does not carry: it must already exist at the destination. */
+export interface BundleDependency {
+  key: string
+  objectType: string
+  name: string
+  /** Why it is needed, in words, so a blocking finding can explain itself. */
+  why: string
+}
+
+/** A media file the bundle's media nodes name. Blobs travel only when asked for. */
+export interface BundleBlob {
+  /** The media store key, as the value names it. */
+  key: string
+  etag: string | null
+  size: number | null
+  included: boolean
+  /** The media node that names it. */
+  node: string
+}
+
+export interface BundleDependencies {
+  /** Keys carried in the bundle. */
+  carried: string[]
+  /** Keys that must already exist at the destination. */
+  expected: BundleDependency[]
+  /** What the destination's schema and views must provide. */
+  schema: {
+    contentTypes: Array<{ key: string; alias: string }>
+    components: string[]
+    languages: string[]
+  }
+}
+
+/**
+ * Where a bundle came from. Recorded and printed, never gated on: development
+ * runs at revision 0 by design, so refusing a bundle whose revision is higher
+ * than the destination's would refuse every transfer out of production. What
+ * the destination actually has to satisfy is checked element by element.
+ */
+export interface BundleProvenance {
+  schemaVersion: string
+  schemaRevision: string
+  schemaHash: string | null
+  siteName: string
+}
+
+export interface BundleSelector {
+  /** The resolved subtree roots. */
+  roots: string[]
+  descendants: boolean
+  /** What was typed on the command line, for readability when a path was given. */
+  asGiven: string[]
+}
+
+export interface BundleManifest {
+  formatVersion: number
+  id: string
+  /**
+   * What a person calls this bundle — a created package's name, when that is
+   * where it came from.
+   *
+   * It travels with the artifact rather than being supplied at import time, so
+   * a downloaded bundle still knows what it is: two files are otherwise
+   * distinguishable only by filename, which is the first thing to be lost.
+   */
+  label?: string
+  createdAt: string
+  createdBy: string
+  /** `sha256-…` over the node files, so a half-copied bundle is refused rather than imported. */
+  integrity: string
+  snapshot: BundleSnapshot
+  provenance: BundleProvenance
+  selector: BundleSelector
+  counts: Partial<Record<BundleKind, number>>
+  dependencies: BundleDependencies
+  blobs: BundleBlob[]
+  /**
+   * The files this bundle carries beyond its content and media, by section, in
+   * path order — the structure and the views that make a bundle installable
+   * into a site that has neither.
+   *
+   * A claim the reader checks, exactly as `blobs.included` is: every path named
+   * here has to be present, and the integrity hash covers it. Absent on a
+   * content-only bundle, which is what version 1 always was.
+   */
+  carries?: Partial<Record<BundleSection, string[]>>
+}
+
+/**
+ * A bundle read into memory: the manifest and its nodes. Named for
+ * `SchemaSet` in `@crombolo/schema`, which plays the same part for structure.
+ */
+export interface ContentSet {
+  manifest: BundleManifest
+  nodes: BundleNode[]
+}
+
+/** A problem with the bundle itself, as opposed to a finding about applying it. */
+export interface BundleProblem {
+  file: string
+  message: string
+}

package/src/references.ts

@@ -0,0 +1,71 @@
+/**
+ * What a stored value refers to.
+ *
+ * `ReferenceRepository` answers the other direction — what refers to *this* node
+ * — by scanning for a known key. Exporting needs the outbound direction, from a
+ * value to the keys inside it, and nothing knew how to do that.
+ *
+ * Two spellings travel, as `references.ts` in `@crombolo/data` documents: a
+ * picker stores `umb://document/<hex>` or a bare uuid, dashed or not. Element
+ * pickers store a bare `Guid[]`, and rich text and block editors hold both
+ * inside JSON, so the scan goes through structures and through JSON held in a
+ * string.
+ *
+ * **These are candidates, not references.** An undashed 32-hex run is
+ * indistinguishable from any other 32 hex characters, so the caller resolves
+ * each against `node.unique_id` and discards what does not exist. Being wrong
+ * in this direction is cheap; missing a reference is not.
+ */
+import { keyFromReference } from '@crombolo/core'
+
+/** Dashed uuids, and 32-hex runs that are not part of a longer hex string. */
+const DASHED = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
+const PLAIN = /(?<![0-9a-f])[0-9a-f]{32}(?![0-9a-f])/gi
+
+function fromString(text: string, into: Set<string>): void {
+  for (const match of text.matchAll(DASHED)) {
+    const key = keyFromReference(match[0])
+    if (key) into.add(key)
+  }
+  for (const match of text.matchAll(PLAIN)) {
+    const key = keyFromReference(match[0])
+    if (key) into.add(key)
+  }
+}
+
+function walk(value: unknown, into: Set<string>, depth: number): void {
+  // Block editors nest, but not deeply; the guard is against a cycle in a
+  // structure we did not build, not against legitimate nesting.
+  if (depth > 32) return
+  if (typeof value === 'string') {
+    fromString(value, into)
+    // A value is often JSON in a string. Scanning the text already found every
+    // key in it, so parsing adds nothing — except where a key is escaped or
+    // split across the encoding, which is why the parse is attempted at all.
+    if (/^\s*[[{]/.test(value)) {
+      try {
+        walk(JSON.parse(value), into, depth + 1)
+      } catch {
+        // Not JSON after all; the text scan above stands.
+      }
+    }
+    return
+  }
+  if (Array.isArray(value)) {
+    for (const item of value) walk(item, into, depth + 1)
+    return
+  }
+  if (typeof value === 'object' && value !== null) {
+    for (const item of Object.values(value)) walk(item, into, depth + 1)
+  }
+}
+
+/**
+ * Every key-shaped candidate inside a value, dashed and lowercased. Resolve
+ * them against real nodes before treating any of them as a reference.
+ */
+export function referenceCandidates(value: unknown): string[] {
+  const found = new Set<string>()
+  walk(value, found, 0)
+  return [...found].sort()
+}