@crombolo/transfer 0.49.0 → 0.51.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crombolo/transfer",
3
- "version": "0.49.0",
3
+ "version": "0.51.0",
4
4
  "description": "Internal to crombolo, with no compatibility promise: use @crombolo/crombolo. Content transfer for crombolo: bundles of content moved between environments.",
5
5
  "license": "MIT",
6
6
  "author": "Crombolo",
@@ -28,7 +28,7 @@
28
28
  "access": "public"
29
29
  },
30
30
  "dependencies": {
31
- "@crombolo/core": "0.49.0",
32
- "@crombolo/data": "0.49.0"
31
+ "@crombolo/core": "0.51.0",
32
+ "@crombolo/data": "0.51.0"
33
33
  }
34
34
  }
package/src/check.ts CHANGED
@@ -305,7 +305,10 @@ export async function checkBundle(
305
305
  const repoFor = (kind: BundleNode['kind']): DocumentRepository => {
306
306
  const found = byKind.get(kind)
307
307
  if (found) return found
308
- const repo = new DocumentRepository(db, { nodeState, kind })
308
+ const repo = new DocumentRepository(db, {
309
+ nodeState,
310
+ kind: kind === 'element-folder' ? 'element' : kind,
311
+ })
309
312
  byKind.set(kind, repo)
310
313
  return repo
311
314
  }
@@ -463,6 +466,78 @@ export async function checkBundle(
463
466
  return editors
464
467
  }
465
468
 
469
+ /**
470
+ * A Library folder is made if it is missing and otherwise left alone, so it
471
+ * takes none of the comparing a node with values does (story 283).
472
+ */
473
+ const carriedNodes = new Map(set.nodes.map((n) => [n.key, n]))
474
+ const checkFolder = async (
475
+ node: BundleNode,
476
+ name: string,
477
+ existing: NodeRow | undefined,
478
+ resolution: Resolution | undefined,
479
+ ): Promise<NodePlan> => {
480
+ const planned = (action: NodePlan['action']): NodePlan => ({
481
+ key: node.key,
482
+ name,
483
+ kind: node.kind,
484
+ action,
485
+ changing: [],
486
+ resolution,
487
+ })
488
+ const finding = (kind: Finding['kind'], code: string, message: string) =>
489
+ findings.push({
490
+ kind,
491
+ code,
492
+ subjectType: 'document',
493
+ subjectKey: node.key,
494
+ subjectName: name,
495
+ propertyAlias: null,
496
+ culture: null,
497
+ message,
498
+ link: null,
499
+ })
500
+ if (resolution === 'skip') return planned('skip')
501
+ if (existing && existing.objectType !== ObjectTypes.ElementContainer) {
502
+ finding('blocking', 'folder-key-taken', `the key of the folder "${name}" is in use here`)
503
+ return planned('skip')
504
+ }
505
+ const parent = node.parent ? carriedNodes.get(node.parent) : undefined
506
+ if (parent && parent.kind !== 'element-folder') {
507
+ finding('blocking', 'folder-parent-invalid', `"${name}" sits under something not a folder`)
508
+ return planned('skip')
509
+ }
510
+ if (existing && !existing.trashed) return planned('unchanged')
511
+ const atTop = !parent
512
+ if (atTop && options.under && options.under.objectType !== ObjectTypes.ElementContainer) {
513
+ finding(
514
+ 'blocking',
515
+ 'folder-under-invalid',
516
+ `the folder "${name}" can only be placed under another Library folder`,
517
+ )
518
+ return planned('skip')
519
+ }
520
+ if (existing) {
521
+ finding('auto', 'folder-restored', `"${name}" comes back out of the recycle bin`)
522
+ return planned('create')
523
+ }
524
+ const parentHere = atTop
525
+ ? (options.under ?? (node.parent ? await nodes.byKey(node.parent) : undefined))
526
+ : await nodes.byKey(parent.key)
527
+ const namesakes =
528
+ atTop && !parentHere && node.parent
529
+ ? []
530
+ : await nodes.childrenNamed(parentHere?.id ?? -1, [ObjectTypes.ElementContainer], name)
531
+ if (namesakes.length > 0)
532
+ finding(
533
+ 'auto',
534
+ 'folder-beside-namesake',
535
+ `a folder called "${name}" is already here; this one keeps its own key and sits beside it`,
536
+ )
537
+ finding('auto', 'new-node', `the folder "${name}" is new here`)
538
+ return planned('create')
539
+ }
540
+
466
541
  const siblingNames = new Map<string, Set<string>>()
467
542
  // What earlier imports applied: the third side of every comparison below.
468
543
  const appliedBefore = await new ContentAppliedRepository(db).byKeys(set.nodes.map((n) => n.key))
@@ -472,6 +547,10 @@ export async function checkBundle(
472
547
  const answers = resolutions[node.key]
473
548
  const resolution = answers?.[WHOLE_NODE] ?? options.resolveAll
474
549
  const existing = await nodes.byKey(node.key)
550
+ if (node.kind === 'element-folder') {
551
+ plan.push(await checkFolder(node, name, existing, resolution))
552
+ continue
553
+ }
475
554
  const editors = await editorsFor(node.contentType.alias)
476
555
  const typeMissing = !(await typeFor(node.contentType))
477
556
 
package/src/code.ts CHANGED
@@ -22,6 +22,7 @@ const SITE_FOLDERS = new Set([
22
22
  'components',
23
23
  'css',
24
24
  'scripts',
25
+ 'public',
25
26
  'content',
26
27
  'media',
27
28
  'logs',
package/src/export.ts CHANGED
@@ -34,6 +34,7 @@ import {
34
34
  type BundleSnapshot,
35
35
  type BundleValue,
36
36
  type ContentSet,
37
+ ELEMENT_FOLDER_TYPE,
37
38
  } from './model.ts'
38
39
  import { referenceCandidates } from './references.ts'
39
40
 
@@ -43,6 +44,7 @@ const KINDS: Record<BundleKind, { objectType: string; contentKind: ContentKind }
43
44
  media: { objectType: ObjectTypes.Media, contentKind: 'media' },
44
45
  element: { objectType: ObjectTypes.Element, contentKind: 'element' },
45
46
  blueprint: { objectType: ObjectTypes.DocumentBlueprint, contentKind: 'blueprint' },
47
+ 'element-folder': { objectType: ObjectTypes.ElementContainer, contentKind: 'element' },
46
48
  }
47
49
 
48
50
  /** Editors that hold a file rather than a reference to one. */
@@ -168,6 +170,20 @@ function toBundleNode(
168
170
  }
169
171
  }
170
172
 
173
+ /** A Library folder: a name and a place, with no type, values or versions. */
174
+ function toFolderNode(node: NodeRow, parent: string | null): BundleNode {
175
+ return {
176
+ key: node.key,
177
+ kind: 'element-folder',
178
+ contentType: { ...ELEMENT_FOLDER_TYPE },
179
+ parent,
180
+ sortOrder: node.sortOrder,
181
+ template: null,
182
+ variants: [{ culture: null, segment: null, name: node.text ?? '', published: false }],
183
+ values: [],
184
+ }
185
+ }
186
+
171
187
  /** Every node of the selection, roots first and parents before children. */
172
188
  async function selected(
173
189
  db: Db,
@@ -176,7 +192,12 @@ async function selected(
176
192
  const byKind = new Map<string, BundleKind>(
177
193
  Object.entries(KINDS).map(([kind, v]) => [v.objectType, kind as BundleKind]),
178
194
  )
179
- const wanted: string[] = [ObjectTypes.Document, ObjectTypes.Media, ObjectTypes.Element]
195
+ const wanted: string[] = [
196
+ ObjectTypes.Document,
197
+ ObjectTypes.Media,
198
+ ObjectTypes.Element,
199
+ ObjectTypes.ElementContainer,
200
+ ]
180
201
  if (options.blueprints) wanted.push(ObjectTypes.DocumentBlueprint)
181
202
 
182
203
  const found = new Map<number, { node: NodeRow; kind: BundleKind }>()
@@ -190,6 +211,13 @@ async function selected(
190
211
  if (k) found.set(descendant.id, { node: descendant, kind: k })
191
212
  }
192
213
  }
214
+ // The folders above a Library item come with it, so it has somewhere to land.
215
+ for (const { node, kind } of [...found.values()]) {
216
+ if (kind !== 'element' && kind !== 'element-folder') continue
217
+ for (const ancestor of await nodes.ancestors(node.key))
218
+ if (ancestor.objectType === ObjectTypes.ElementContainer)
219
+ found.set(ancestor.id, { node: ancestor, kind: 'element-folder' })
220
+ }
193
221
  return [...found.values()].sort((a, b) => a.node.level - b.node.level || a.node.id - b.node.id)
194
222
  }
195
223
 
@@ -233,6 +261,21 @@ export async function exportBundle(db: Db, options: ExportOptions): Promise<Expo
233
261
  const counts: Partial<Record<BundleKind, number>> = {}
234
262
 
235
263
  for (const { node, kind } of chosen) {
264
+ // The parent travels as a key only when it is in the bundle; otherwise it is
265
+ // an expected dependency, which the destination must already have. The
266
+ // system containers — the tree root and the recycle bins — have real `node`
267
+ // rows with keys of their own, and a node sitting under one has no parent as
268
+ // far as a bundle is concerned.
269
+ const parentKey =
270
+ node.parentId <= 0
271
+ ? null
272
+ : (keyOfNodeId.get(node.parentId) ?? (await nodes.byId(node.parentId))?.key ?? null)
273
+ if (kind === 'element-folder') {
274
+ bundleNodes.push(toFolderNode(node, parentKey))
275
+ counts[kind] = (counts[kind] ?? 0) + 1
276
+ continue
277
+ }
278
+
236
279
  const repo = repoFor(kind)
237
280
  // Media is never published, so there is no published snapshot to prefer.
238
281
  const published =
@@ -249,15 +292,6 @@ export async function exportBundle(db: Db, options: ExportOptions): Promise<Expo
249
292
  for (const key of keys) referenced.add(key)
250
293
  }
251
294
 
252
- // The parent travels as a key only when it is in the bundle; otherwise it is
253
- // an expected dependency, which the destination must already have. The
254
- // system containers — the tree root and the recycle bins — have real `node`
255
- // rows with keys of their own, and a node sitting under one has no parent as
256
- // far as a bundle is concerned.
257
- const parentKey =
258
- node.parentId <= 0
259
- ? null
260
- : (keyOfNodeId.get(node.parentId) ?? (await nodes.byId(node.parentId))?.key ?? null)
261
295
  // A template's definition is its file, so the alias travels and the key does
262
296
  // not: the same view has a different key in every environment.
263
297
  const componentAlias = aggregate.componentKey
package/src/import.ts CHANGED
@@ -34,6 +34,7 @@
34
34
  * follow the bundle under the same rule.
35
35
  */
36
36
  import type { DocumentValue } from '@crombolo/core'
37
+ import { ObjectTypes } from '@crombolo/core'
37
38
  import {
38
39
  type AppliedState,
39
40
  appendCacheInstruction,
@@ -45,6 +46,7 @@ import {
45
46
  currentSchemaState,
46
47
  type Db,
47
48
  DocumentRepository,
49
+ FolderRepository,
48
50
  insideTransaction,
49
51
  Locks,
50
52
  MediaAccessRepository,
@@ -69,6 +71,7 @@ const CONTENT_KIND: Record<BundleKind, ContentKind> = {
69
71
  media: 'media',
70
72
  element: 'element',
71
73
  blueprint: 'blueprint',
74
+ 'element-folder': 'element',
72
75
  }
73
76
 
74
77
  export interface ImportOptions extends CheckOptions {
@@ -241,10 +244,42 @@ async function applyBundle(tx: Db, set: ContentSet, options: ImportOptions): Pro
241
244
  })
242
245
 
243
246
  const planByKey = new Map(check.plan.map((p) => [p.key, p]))
244
- const ordered = inTreeOrder(set.nodes).filter(
247
+ const ordered = inTreeOrder(set.nodes.filter((node) => node.kind !== 'element-folder')).filter(
245
248
  (node) => planByKey.get(node.key)?.action !== 'skip',
246
249
  )
247
250
 
251
+ // Library folders first, in the source's order within a parent, so an element
252
+ // lands in its folder. One that is here stays as it is (story 283).
253
+ const folders = new FolderRepository(tx, ObjectTypes.ElementContainer, ObjectTypes.Element)
254
+ const foldersHere = new Set<string>()
255
+ const madeFolders: string[] = []
256
+ const folderPlans: NodePlan[] = []
257
+ const carriedFolders = set.nodes
258
+ .filter((node) => node.kind === 'element-folder')
259
+ .sort((a, b) => a.sortOrder - b.sortOrder)
260
+ for (const node of inTreeOrder(carriedFolders)) {
261
+ const plan = planByKey.get(node.key)
262
+ if (!plan || plan.action === 'skip') continue
263
+ foldersHere.add(node.key)
264
+ folderPlans.push(plan)
265
+ if (plan.action !== 'create') continue
266
+ const parentKey =
267
+ node.parent && foldersHere.has(node.parent)
268
+ ? node.parent
269
+ : (options.under?.key ??
270
+ (node.parent && (await repoFor('element').nodes.byKey(node.parent))
271
+ ? node.parent
272
+ : null))
273
+ if (await folders.byKey(node.key)) await folders.restore(node.key, parentKey)
274
+ else
275
+ await folders.create({
276
+ key: node.key,
277
+ name: node.variants[0]?.name ?? '',
278
+ parentKey,
279
+ })
280
+ madeFolders.push(node.key)
281
+ }
282
+
248
283
  // What each node was before this run, read before anything is written:
249
284
  // the two numbers a revert needs to put the site back.
250
285
  const before = new Map<string, { head: number | undefined; published: number | undefined }>()
@@ -274,7 +309,10 @@ async function applyBundle(tx: Db, set: ContentSet, options: ImportOptions): Pro
274
309
  `No content type "${node.contentType.alias}" for "${node.key}", which the check should have refused.`,
275
310
  )
276
311
  const parentKey =
277
- node.parent && !created.has(node.parent) && !before.has(node.parent)
312
+ node.parent &&
313
+ !created.has(node.parent) &&
314
+ !before.has(node.parent) &&
315
+ !foldersHere.has(node.parent)
278
316
  ? (options.under?.key ?? null)
279
317
  : (node.parent ?? options.under?.key ?? null)
280
318
  await repo.create({
@@ -324,7 +362,7 @@ async function applyBundle(tx: Db, set: ContentSet, options: ImportOptions): Pro
324
362
  }
325
363
 
326
364
  // Pass two: the values, overlaid onto whatever is here.
327
- const applied: NodePlan[] = []
365
+ const applied: NodePlan[] = [...folderPlans]
328
366
  for (const node of ordered) {
329
367
  const plan = planByKey.get(node.key)
330
368
  const kept = new Set(plan?.keepLocal ?? [])
@@ -521,6 +559,16 @@ async function applyBundle(tx: Db, set: ContentSet, options: ImportOptions): Pro
521
559
  eventId: markers?.head ?? null,
522
560
  })
523
561
  }
562
+ for (const key of madeFolders)
563
+ await runs.record({
564
+ runId,
565
+ nodeKey: key,
566
+ kind: 'element-folder',
567
+ action: 'create',
568
+ beforeEventId: null,
569
+ beforePublishedEventId: null,
570
+ eventId: null,
571
+ })
524
572
  // What this run applied, for the next import to compare against. A node
525
573
  // that kept its own order keeps the order recorded before.
526
574
  const appliedRecords = new ContentAppliedRepository(tx)
@@ -548,7 +596,7 @@ async function applyBundle(tx: Db, set: ContentSet, options: ImportOptions): Pro
548
596
  await appliedRecords.record(node.key, runId, state)
549
597
  }
550
598
 
551
- await runs.finish(runId, ordered.length + people.length)
599
+ await runs.finish(runId, ordered.length + madeFolders.length + people.length)
552
600
 
553
601
  // Other nodes drop their published cache; this one already has.
554
602
  await appendCacheInstruction(tx, {
package/src/index.ts CHANGED
@@ -7,6 +7,7 @@ export * from './model.ts'
7
7
  export * from './package.ts'
8
8
  export * from './people.ts'
9
9
  export * from './people-apply.ts'
10
+ export * from './public.ts'
10
11
  export * from './references.ts'
11
12
  export * from './resolutions.ts'
12
13
  export * from './revert.ts'
package/src/load.ts CHANGED
@@ -7,7 +7,7 @@
7
7
  * a crash rather than a finding.
8
8
  */
9
9
 
10
- import { existsSync, readdirSync, readFileSync } from 'node:fs'
10
+ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
11
11
  import { join, relative, sep } from 'node:path'
12
12
  import {
13
13
  BUNDLE_FORMAT_VERSION,
@@ -25,9 +25,11 @@ import {
25
25
  type BundleValue,
26
26
  type BundleVariant,
27
27
  type ContentSet,
28
+ ELEMENT_FOLDER_TYPE,
28
29
  OLDEST_BUNDLE_FORMAT_VERSION,
29
30
  } from './model.ts'
30
31
  import { type BundlePeople, openPeople, type PeopleBlock, parsePeopleBlock } from './people.ts'
32
+ import { hashFile, isPublicPath, PUBLIC_MANIFEST, type PublicFile } from './public.ts'
31
33
  import { openBytes, parseSealParameters, unlock } from './seal.ts'
32
34
  import {
33
35
  openSiteData,
@@ -72,7 +74,7 @@ export interface LoadedBundle {
72
74
  blobs: Map<string, string>
73
75
  }
74
76
 
75
- const KINDS: readonly BundleKind[] = ['document', 'media', 'element', 'blueprint']
77
+ const KINDS: readonly BundleKind[] = ['document', 'media', 'element', 'blueprint', 'element-folder']
76
78
 
77
79
  const isRecord = (v: unknown): v is Record<string, unknown> =>
78
80
  typeof v === 'object' && v !== null && !Array.isArray(v)
@@ -147,8 +149,9 @@ export function parseNode(
147
149
  const key = str(raw.key)
148
150
  const kind = str(raw.kind) as BundleKind | undefined
149
151
  const contentType = isRecord(raw.contentType) ? raw.contentType : undefined
150
- const typeKey = str(contentType?.key)
151
- const typeAlias = str(contentType?.alias)
152
+ const isFolder = kind === 'element-folder'
153
+ const typeKey = isFolder ? ELEMENT_FOLDER_TYPE.key : str(contentType?.key)
154
+ const typeAlias = isFolder ? ELEMENT_FOLDER_TYPE.alias : str(contentType?.alias)
152
155
  if (!key) problems.push({ file, message: 'no key' })
153
156
  if (!kind || !KINDS.includes(kind))
154
157
  problems.push({ file, message: `kind must be one of ${KINDS.join(', ')}` })
@@ -170,6 +173,8 @@ export function parseNode(
170
173
  })
171
174
  : []
172
175
  if (variants.length === 0) problems.push({ file, message: 'no variants: a node needs a name' })
176
+ if (isFolder && values.length > 0)
177
+ problems.push({ file, message: 'a Library folder holds no values' })
173
178
 
174
179
  const access = parseAccess(raw.access)
175
180
  const mediaAccess = kind === 'media' ? parseMediaAccess(raw.mediaAccess) : undefined
@@ -371,6 +376,7 @@ function parseManifest(text: string, problems: BundleProblem[]): BundleManifest
371
376
  // Filled from `media.json` by the loader, which the hash covers (story 195).
372
377
  blobs: [],
373
378
  ...(raw.mediaManifest === MEDIA_MANIFEST ? { mediaManifest: MEDIA_MANIFEST } : {}),
379
+ ...(raw.publicManifest === PUBLIC_MANIFEST ? { publicManifest: PUBLIC_MANIFEST } : {}),
374
380
  ...(isRecord(raw.carries) ? { carries: parseCarries(raw.carries) } : {}),
375
381
  ...(raw.people !== undefined && parsePeopleBlock(raw.people)
376
382
  ? { people: parsePeopleBlock(raw.people) as PeopleBlock }
@@ -402,6 +408,8 @@ function parseRequires(raw: Record<string, unknown>): Record<string, string> {
402
408
  function parseCarries(raw: Record<string, unknown>): Partial<Record<BundleSection, string[]>> {
403
409
  const out: Partial<Record<BundleSection, string[]>> = {}
404
410
  for (const section of Object.keys(BUNDLE_SECTIONS) as BundleSection[]) {
411
+ // Only ever from `public.json`, whose list is what the hash covers.
412
+ if (section === 'public') continue
405
413
  const paths = raw[section]
406
414
  if (!Array.isArray(paths)) continue
407
415
  const safe = paths.flatMap((path) =>
@@ -477,6 +485,53 @@ function readMediaManifest(
477
485
  return { blobs, text }
478
486
  }
479
487
 
488
+ /**
489
+ * `public.json`, read and checked: undefined when the bundle has none, and a
490
+ * problem when `bundle.json` says it has and it is missing or malformed.
491
+ */
492
+ function readPublicManifest(
493
+ dir: string,
494
+ manifest: BundleManifest,
495
+ problems: BundleProblem[],
496
+ ): { files: PublicFile[]; text: string } | undefined {
497
+ const file = join(dir, PUBLIC_MANIFEST)
498
+ if (!manifest.publicManifest) {
499
+ if (existsSync(file))
500
+ problems.push({ file: PUBLIC_MANIFEST, message: 'bundle.json does not declare this file' })
501
+ return undefined
502
+ }
503
+ if (!existsSync(file)) {
504
+ problems.push({ file: PUBLIC_MANIFEST, message: 'bundle.json declares it, and it is missing' })
505
+ return undefined
506
+ }
507
+ const text = readFileSync(file, 'utf8')
508
+ let raw: unknown
509
+ try {
510
+ raw = JSON.parse(text)
511
+ } catch {
512
+ problems.push({ file: PUBLIC_MANIFEST, message: 'not valid JSON' })
513
+ return undefined
514
+ }
515
+ const entries = isRecord(raw) && Array.isArray(raw.files) ? raw.files : undefined
516
+ if (!entries) {
517
+ problems.push({ file: PUBLIC_MANIFEST, message: 'has no list of files' })
518
+ return undefined
519
+ }
520
+ const files = entries.flatMap((entry): PublicFile[] => {
521
+ if (
522
+ isRecord(entry) &&
523
+ typeof entry.path === 'string' &&
524
+ isPublicPath(entry.path) &&
525
+ typeof entry.size === 'number' &&
526
+ typeof entry.sha256 === 'string'
527
+ )
528
+ return [{ path: entry.path, size: entry.size, sha256: entry.sha256 }]
529
+ problems.push({ file: PUBLIC_MANIFEST, message: 'lists a file with no usable path or hash' })
530
+ return []
531
+ })
532
+ return { files, text }
533
+ }
534
+
480
535
  /** A media key that stays inside `media/`. */
481
536
  const isSafeKey = (key: string): boolean =>
482
537
  key !== '' && !key.startsWith('/') && !key.split('/').some((part) => part === '' || part === '..')
@@ -567,6 +622,38 @@ export function loadBundle(dir: string): LoadedBundle {
567
622
  }
568
623
  }
569
624
 
625
+ // Public files are checked one at a time against `public.json`, which the
626
+ // bundle's hash covers, so none has to be held in memory.
627
+ const publicList = readPublicManifest(dir, manifest, problems)
628
+ if (publicList) {
629
+ read.push({ path: PUBLIC_MANIFEST, text: publicList.text })
630
+ manifest.publicFiles = publicList.files
631
+ manifest.carries = { ...manifest.carries, public: publicList.files.map((f) => f.path).sort() }
632
+ for (const { path, size, sha256 } of publicList.files) {
633
+ const file = join(dir, path)
634
+ if (!existsSync(file)) {
635
+ problems.push({
636
+ file: path,
637
+ message: 'the manifest says this file is carried, and it is not',
638
+ })
639
+ continue
640
+ }
641
+ if (statSync(file).size !== size || hashFile(file) !== sha256)
642
+ problems.push({ file: path, message: 'is not the file public.json lists' })
643
+ files.set(path, file)
644
+ }
645
+ }
646
+ const publicDir = join(dir, BUNDLE_SECTIONS.public)
647
+ if (existsSync(publicDir)) {
648
+ const listed = new Set(publicList?.files.map((f) => f.path))
649
+ for (const entry of readdirSync(publicDir, { recursive: true, withFileTypes: true })) {
650
+ if (!entry.isFile()) continue
651
+ const path = relative(dir, join(entry.parentPath, entry.name)).split(sep).join('/')
652
+ if (!listed.has(path))
653
+ problems.push({ file: path, message: 'this file is not listed in public.json' })
654
+ }
655
+ }
656
+
570
657
  // People are sealed bytes, opened only with the passphrase at install; here
571
658
  // they are only read, and counted in the hash like everything else.
572
659
  const people = new Map<string, Uint8Array>()
package/src/model.ts CHANGED
@@ -9,6 +9,7 @@
9
9
  */
10
10
 
11
11
  import type { BundlePeople, PeopleBlock } from './people.ts'
12
+ import type { PublicFile } from './public.ts'
12
13
  import type { SealParameters } from './seal.ts'
13
14
  import type { SiteDataBlock, SiteDataTable } from './site-data.ts'
14
15
 
@@ -20,11 +21,13 @@ import type { SiteDataBlock, SiteDataTable } from './site-data.ts'
20
21
  * it, and a site's code as a section (stories 192, 193).
21
22
  * 5: media listed in `media.json` at the root, each carried file at
22
23
  * `media/<key>` with a `bundle://` URI (story 195).
24
+ * 6: a Library folder may travel as a node (story 283).
25
+ * 7: the site's `public/` folder, listed in `public.json` (story 284).
23
26
  *
24
27
  * An older version is refused rather than read (no older layout is kept up),
25
28
  * with a message saying to rebuild it.
26
29
  */
27
- export const BUNDLE_FORMAT_VERSION = 5
30
+ export const BUNDLE_FORMAT_VERSION = 7
28
31
 
29
32
  /** The oldest version this reader reads: the first with media listed in `media.json`. */
30
33
  export const OLDEST_BUNDLE_FORMAT_VERSION = 5
@@ -55,6 +58,11 @@ export const BUNDLE_SECTIONS = {
55
58
  * `installBundleCode`'s, followed by a restart.
56
59
  */
57
60
  code: 'code',
61
+ /**
62
+ * The site's `public/` folder (story 284), which may be images: read as bytes,
63
+ * listed with their hashes in `public.json` rather than in `carries`.
64
+ */
65
+ public: 'public',
58
66
  } as const
59
67
 
60
68
  export type BundleSection = keyof typeof BUNDLE_SECTIONS
@@ -63,7 +71,10 @@ export type BundleSection = keyof typeof BUNDLE_SECTIONS
63
71
  * The kinds that travel. `member` is deliberately absent: members are personal
64
72
  * data, and a bundle is a file that gets copied around and committed.
65
73
  */
66
- export type BundleKind = 'document' | 'media' | 'element' | 'blueprint'
74
+ export type BundleKind = 'document' | 'media' | 'element' | 'blueprint' | 'element-folder'
75
+
76
+ /** What a Library folder names as its type: it has none, only a name and a place. */
77
+ export const ELEMENT_FOLDER_TYPE = { key: '', alias: 'elementFolder' } as const
67
78
 
68
79
  /** Which snapshot of a node's values the bundle carries. */
69
80
  export type BundleSnapshot = 'published' | 'drafts'
@@ -239,6 +250,10 @@ export interface BundleManifest {
239
250
  * the package root and inside the hash (story 195). `blobs` is read from it.
240
251
  */
241
252
  mediaManifest?: string
253
+ /** `public.json`, when the bundle carries the site's `public/` folder (story 284). */
254
+ publicManifest?: string
255
+ /** The public files, read from `public.json`; `carries.public` holds their paths. */
256
+ publicFiles?: PublicFile[]
242
257
  /**
243
258
  * The files this bundle carries beyond its content and media, by section, in
244
259
  * path order — the structure and the views that make a bundle installable
package/src/public.ts ADDED
@@ -0,0 +1,53 @@
1
+ /**
2
+ * The site's `public/` folder in a bundle (story 284). Its files can be large,
3
+ * so `public.json` lists each with its size and hash, the bundle's hash covers
4
+ * that list, and no file is held in memory. `docs/bundles.md`, "Public files".
5
+ */
6
+ import { closeSync, openSync, readSync } from 'node:fs'
7
+ import { BUNDLE_SECTIONS } from './model.ts'
8
+
9
+ /** The list of public files, at the package root beside `bundle.json`. */
10
+ export const PUBLIC_MANIFEST = 'public.json'
11
+
12
+ /** A public file a bundle carries, at its path inside the bundle. */
13
+ export interface PublicFile {
14
+ path: string
15
+ size: number
16
+ sha256: string
17
+ }
18
+
19
+ /** A public file on its way into a bundle, and where its bytes are. */
20
+ export interface PublicSource extends PublicFile {
21
+ source: string
22
+ }
23
+
24
+ /** A path that stays inside the bundle's `public/`. */
25
+ export const isPublicPath = (path: string): boolean => {
26
+ const [top, ...rest] = path.split('/')
27
+ return (
28
+ top === BUNDLE_SECTIONS.public &&
29
+ rest.length > 0 &&
30
+ !rest.some((part) => part === '' || part === '.' || part === '..' || part.includes('\\'))
31
+ )
32
+ }
33
+
34
+ /** A file's SHA-256, read a piece at a time. */
35
+ export function hashFile(file: string): string {
36
+ const hasher = new Bun.CryptoHasher('sha256')
37
+ const piece = new Uint8Array(1024 * 1024)
38
+ const fd = openSync(file, 'r')
39
+ try {
40
+ for (let read = readSync(fd, piece); read > 0; read = readSync(fd, piece))
41
+ hasher.update(piece.subarray(0, read))
42
+ } finally {
43
+ closeSync(fd)
44
+ }
45
+ return hasher.digest('hex')
46
+ }
47
+
48
+ export function writePublicManifest(files: readonly PublicFile[]): string {
49
+ const entries = [...files]
50
+ .sort((a, b) => a.path.localeCompare(b.path))
51
+ .map(({ path, size, sha256 }) => ({ path, size, sha256 }))
52
+ return `${JSON.stringify({ files: entries }, null, 2)}\n`
53
+ }
package/src/revert.ts CHANGED
@@ -14,6 +14,7 @@
14
14
  * edited a page after the import, and discarding that without asking would be
15
15
  * the worst thing this command could do.
16
16
  */
17
+ import { ObjectTypes, SystemNodes } from '@crombolo/core'
17
18
  import {
18
19
  appendCacheInstruction,
19
20
  ContentAppliedRepository,
@@ -21,6 +22,7 @@ import {
21
22
  type Db,
22
23
  DocumentRepository,
23
24
  type Finding,
25
+ FolderRepository,
24
26
  Locks,
25
27
  type NodeRow,
26
28
  PublishBlockedError,
@@ -76,7 +78,7 @@ export interface RevertPlan {
76
78
  * run created, and `unrecycle` brings back what a *reverted* run recycled —
77
79
  * which is what makes a revert revertible in its own right.
78
80
  */
79
- action: 'restore' | 'recycle' | 'unrecycle' | 'skip' | 'gone' | 'remove'
81
+ action: 'restore' | 'recycle' | 'unrecycle' | 'skip' | 'gone' | 'remove' | 'keep'
80
82
  resolution: RevertResolution | undefined
81
83
  }
82
84
 
@@ -97,6 +99,8 @@ export interface RevertResult {
97
99
  removed: string[]
98
100
  }
99
101
 
102
+ const FOLDER_KIND = 'element-folder'
103
+
100
104
  const link = (backOfficePath: string | undefined, key: string): string | null =>
101
105
  backOfficePath ? `${backOfficePath}/section/content/workspace/document/edit/${key}` : null
102
106
 
@@ -207,6 +211,41 @@ export async function checkRevert(
207
211
  continue
208
212
  }
209
213
 
214
+ // A Library folder has no versions and nothing of its own to lose. It goes
215
+ // when it is empty and stays when anything else has come to sit in it.
216
+ if (change.kind === FOLDER_KIND) {
217
+ if (change.action === 'recycle') {
218
+ plan.push({ key: change.nodeKey, name, action: 'unrecycle', resolution })
219
+ continue
220
+ }
221
+ if (node.trashed) {
222
+ plan.push({ key: change.nodeKey, name, action: 'gone', resolution })
223
+ continue
224
+ }
225
+ const others = (
226
+ await docs.nodes.descendants(node, [ObjectTypes.Element, ObjectTypes.ElementContainer])
227
+ ).filter((d) => !ownKeys.has(d.key))
228
+ if (others.length > 0)
229
+ findings.push({
230
+ kind: 'auto',
231
+ code: 'folder-kept',
232
+ subjectType: 'document',
233
+ subjectKey: change.nodeKey,
234
+ subjectName: name,
235
+ propertyAlias: null,
236
+ culture: null,
237
+ message: `the folder "${name}" stays: ${others.length} other item(s) sit in it now`,
238
+ link: null,
239
+ })
240
+ plan.push({
241
+ key: change.nodeKey,
242
+ name,
243
+ action: others.length > 0 ? 'keep' : 'recycle',
244
+ resolution,
245
+ })
246
+ continue
247
+ }
248
+
210
249
  const created = change.action === 'create'
211
250
  const repo = repoFor(db, nodeState, change.kind)
212
251
 
@@ -274,7 +313,12 @@ function repoFor(
274
313
  inTreeLock = false,
275
314
  ): DocumentRepository {
276
315
  return new DocumentRepository(db, {
277
- kind: kind === 'media' || kind === 'element' || kind === 'blueprint' ? kind : 'document',
316
+ kind:
317
+ kind === FOLDER_KIND
318
+ ? 'element'
319
+ : kind === 'media' || kind === 'element' || kind === 'blueprint'
320
+ ? kind
321
+ : 'document',
278
322
  nodeState,
279
323
  inTreeLock,
280
324
  })
@@ -314,11 +358,17 @@ export async function revertRun(
314
358
  })
315
359
 
316
360
  const skipped = new Set(
317
- check.plan.filter((p) => p.action === 'skip' || p.action === 'gone').map((p) => p.key),
361
+ check.plan
362
+ .filter((p) => p.action === 'skip' || p.action === 'gone' || p.action === 'keep')
363
+ .map((p) => p.key),
318
364
  )
319
- const todo = changes.filter(
365
+ const affected = changes.filter(
320
366
  (c) => c.action !== 'skip' && !skipped.has(c.nodeKey) && !isPersonKind(c.kind),
321
367
  )
368
+ const todo = affected.filter((c) => c.kind !== FOLDER_KIND)
369
+ const folderChanges = affected.filter((c) => c.kind === FOLDER_KIND)
370
+ const folders = new FolderRepository(tx, ObjectTypes.ElementContainer, ObjectTypes.Element)
371
+ const applied = new ContentAppliedRepository(tx)
322
372
  // Members before the groups they belong to.
323
373
  const removed: string[] = []
324
374
  const order = ['member', 'user', 'member-group', 'user-group']
@@ -330,7 +380,7 @@ export async function revertRun(
330
380
  )
331
381
  removed.push(change.nodeKey)
332
382
  const byKey = new Map<string, NodeRow>()
333
- for (const change of todo) {
383
+ for (const change of affected) {
334
384
  const node = await repo(change.kind).nodes.byKey(change.nodeKey)
335
385
  if (node) byKey.set(change.nodeKey, node)
336
386
  }
@@ -351,6 +401,22 @@ export async function revertRun(
351
401
  if (markers) before.set(change.nodeKey, markers)
352
402
  }
353
403
 
404
+ // Folders the previous revert recycled come back first, so what was in them
405
+ // has somewhere to return to. It recorded them deepest first, so this is
406
+ // parents first, and it kept where each one was.
407
+ const folderPlaces = await applied.byKeys(folderChanges.map((c) => c.nodeKey))
408
+ const backFolders: string[] = []
409
+ for (const change of [...folderChanges].reverse()) {
410
+ if (change.action !== 'recycle') continue
411
+ const parent = folderPlaces.get(change.nodeKey)?.parent ?? null
412
+ await folders.restore(
413
+ change.nodeKey,
414
+ parent && (await repo(FOLDER_KIND).nodes.byKey(parent)) ? parent : null,
415
+ )
416
+ restored.push(change.nodeKey)
417
+ backFolders.push(change.nodeKey)
418
+ }
419
+
354
420
  // What the previous run recycled comes back out of the bin first, so the
355
421
  // rollback and publish below have a node in the tree to work on.
356
422
  for (const change of inTreeOrder(todo)) {
@@ -397,6 +463,17 @@ export async function revertRun(
397
463
  recycled.push(change.nodeKey)
398
464
  }
399
465
 
466
+ // Folders last and deepest first, once what the run put in them has gone, and
467
+ // only if nothing else is left in them.
468
+ const goneFolders: Array<{ key: string; parent: string | null }> = []
469
+ for (const change of inTreeOrder(folderChanges).reverse()) {
470
+ if (change.action === 'recycle' || (await folders.holdsAnything(change.nodeKey))) continue
471
+ const parent = (await folders.byKey(change.nodeKey))?.parentKey ?? null
472
+ await folders.toRecycleBin(change.nodeKey, SystemNodes.ElementRecycleBin)
473
+ recycled.push(change.nodeKey)
474
+ goneFolders.push({ key: change.nodeKey, parent })
475
+ }
476
+
400
477
  for (const change of todo) {
401
478
  const markers = await repo(change.kind).eventMarkers(change.nodeKey)
402
479
  const was = before.get(change.nodeKey)
@@ -412,9 +489,34 @@ export async function revertRun(
412
489
  eventId: markers?.head ?? null,
413
490
  })
414
491
  }
492
+ for (const [nodeKey, action] of [
493
+ ...goneFolders.map(({ key }) => [key, 'recycle'] as const),
494
+ ...backFolders.map((key) => [key, 'update'] as const),
495
+ ])
496
+ await runs.record({
497
+ runId: newRunId,
498
+ nodeKey,
499
+ kind: FOLDER_KIND,
500
+ action,
501
+ beforeEventId: null,
502
+ beforePublishedEventId: null,
503
+ eventId: null,
504
+ })
415
505
  // What the next import compares against goes back with the content.
416
- await new ContentAppliedRepository(tx).revert(runId)
417
- await runs.finish(newRunId, todo.length + removed.length)
506
+ await applied.revert(runId)
507
+ // A folder keeps no parent of its own in the bin, so this run notes it.
508
+ for (const { key, parent } of goneFolders)
509
+ await applied.record(key, newRunId, {
510
+ values: {},
511
+ names: {},
512
+ parent,
513
+ bundleSortOrder: 0,
514
+ sortOrder: 0,
515
+ })
516
+ await runs.finish(
517
+ newRunId,
518
+ todo.length + goneFolders.length + backFolders.length + removed.length,
519
+ )
418
520
  await runs.setStatus(runId, 'reverted')
419
521
 
420
522
  await appendCacheInstruction(tx, {
package/src/write.ts CHANGED
@@ -24,6 +24,7 @@ import {
24
24
  type ContentSet,
25
25
  } from './model.ts'
26
26
  import { type BundlePeople, sealPeople } from './people.ts'
27
+ import { PUBLIC_MANIFEST, type PublicSource, writePublicManifest } from './public.ts'
27
28
  import { type Seal, sealBytes } from './seal.ts'
28
29
  import { type SiteDataTable, sealSiteData } from './site-data.ts'
29
30
 
@@ -34,6 +35,8 @@ export interface BundleFile {
34
35
  text?: string
35
36
  /** A media file's bytes; `text` and `bytes` are never both set. */
36
37
  bytes?: Uint8Array
38
+ /** A public file, copied from disk in place of `text` or `bytes`; never in the bundle's hash. */
39
+ source?: string
37
40
  }
38
41
 
39
42
  const json = (value: unknown): string => `${JSON.stringify(value, null, 2)}\n`
@@ -198,6 +201,7 @@ export function writeManifest(manifest: BundleManifest): string {
198
201
  // The files themselves are listed in `media.json`, which the hash covers;
199
202
  // this file, which holds the hash, only says that there is one (story 195).
200
203
  ...(manifest.blobs.length > 0 ? { mediaManifest: MEDIA_MANIFEST } : {}),
204
+ ...(manifest.publicManifest ? { publicManifest: manifest.publicManifest } : {}),
201
205
  // Absent on a content-only bundle, so version 1 output is unchanged.
202
206
  ...(manifest.carries && Object.keys(manifest.carries).length > 0
203
207
  ? { carries: sortedCarries(manifest.carries) }
@@ -307,6 +311,8 @@ export interface WriteOptions {
307
311
  people?: BundlePeople
308
312
  /** The site's and bundles' own tables: always sealed. */
309
313
  siteData?: readonly SiteDataTable[]
314
+ /** The site's `public/` files, which `public.json` lists and the caller's bytes follow. */
315
+ public?: readonly PublicSource[]
310
316
  }
311
317
 
312
318
  /** Without values, media files or usernames: what `omitted` writes. Pure. */
@@ -357,7 +363,13 @@ export function writeBundle(
357
363
  ...planned.manifest,
358
364
  integrity: bundleIntegrity([...planned.files, ...carried]),
359
365
  }
360
- return [{ path: MANIFEST_FILE, text: writeManifest(manifest) }, ...planned.files, ...carried]
366
+ const publicFiles = (options.public ?? []).map(({ path, source }) => ({ path, source }))
367
+ return [
368
+ { path: MANIFEST_FILE, text: writeManifest(manifest) },
369
+ ...planned.files,
370
+ ...carried,
371
+ ...publicFiles,
372
+ ]
361
373
  }
362
374
 
363
375
  /**
@@ -365,7 +377,8 @@ export function writeBundle(
365
377
  * every other file, from each carried file's size alone. The caller writes
366
378
  * `blobs` at `mediaPath` — sealed with `sealBytes` under the seal when `sealing`
367
379
  * — computes `bundleIntegrity` over those and `files` in path order, and writes
368
- * the manifest with it.
380
+ * the manifest with it. `options.public` is the caller's to write too, outside
381
+ * that hash.
369
382
  */
370
383
  export function planStreamedBundle(
371
384
  set: ContentSet,
@@ -410,16 +423,18 @@ function planBundle(
410
423
  const carries: Partial<Record<BundleSection, string[]>> = {}
411
424
  for (const file of extra) {
412
425
  const section = sectionOf(file.path)
413
- if (!section) throw new Error(`${file.path} is not in a bundle section`)
426
+ if (!section || section === 'public') throw new Error(`${file.path} is not in a bundle section`)
414
427
  carries[section] = [...(carries[section] ?? []), file.path]
415
428
  }
416
429
 
430
+ const publicFiles = options.public ?? []
417
431
  const people = options.people && seal ? sealPeople(options.people, seal) : undefined
418
432
  const siteData = options.siteData && seal ? sealSiteData(options.siteData, seal) : undefined
419
433
  const manifest: BundleManifest = {
420
434
  ...set.manifest,
421
435
  formatVersion: BUNDLE_FORMAT_VERSION,
422
436
  // Without values nothing names a media file, so none is expected either.
437
+ publicManifest: publicFiles.length > 0 ? PUBLIC_MANIFEST : undefined,
423
438
  blobs: (content === 'omitted' ? [] : set.manifest.blobs).map((blob) => {
424
439
  const size = carriedSizes.get(blob.key)
425
440
  return size !== undefined ? { ...blob, included: true, size } : { ...blob, included: false }
@@ -444,6 +459,9 @@ function planBundle(
444
459
  ...(people?.files ?? []),
445
460
  ...(siteData?.files ?? []),
446
461
  ...media,
462
+ ...(publicFiles.length > 0
463
+ ? [{ path: PUBLIC_MANIFEST, text: writePublicManifest(publicFiles) }]
464
+ : []),
447
465
  ],
448
466
  blobs: [...carriedSizes.keys()].sort((a, b) => a.localeCompare(b)),
449
467
  sealing,