@crombolo/transfer 0.50.0 → 0.51.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crombolo/transfer",
3
- "version": "0.50.0",
3
+ "version": "0.51.1",
4
4
  "description": "Internal to crombolo, with no compatibility promise: use @crombolo/crombolo. Content transfer for crombolo: bundles of content moved between environments.",
5
5
  "license": "MIT",
6
6
  "author": "Crombolo",
@@ -28,7 +28,7 @@
28
28
  "access": "public"
29
29
  },
30
30
  "dependencies": {
31
- "@crombolo/core": "0.50.0",
32
- "@crombolo/data": "0.50.0"
31
+ "@crombolo/core": "0.51.1",
32
+ "@crombolo/data": "0.51.1"
33
33
  }
34
34
  }
package/src/code.ts CHANGED
@@ -22,6 +22,7 @@ const SITE_FOLDERS = new Set([
22
22
  'components',
23
23
  'css',
24
24
  'scripts',
25
+ 'public',
25
26
  'content',
26
27
  'media',
27
28
  'logs',
package/src/index.ts CHANGED
@@ -7,6 +7,7 @@ export * from './model.ts'
7
7
  export * from './package.ts'
8
8
  export * from './people.ts'
9
9
  export * from './people-apply.ts'
10
+ export * from './public.ts'
10
11
  export * from './references.ts'
11
12
  export * from './resolutions.ts'
12
13
  export * from './revert.ts'
package/src/load.ts CHANGED
@@ -7,7 +7,7 @@
7
7
  * a crash rather than a finding.
8
8
  */
9
9
 
10
- import { existsSync, readdirSync, readFileSync } from 'node:fs'
10
+ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
11
11
  import { join, relative, sep } from 'node:path'
12
12
  import {
13
13
  BUNDLE_FORMAT_VERSION,
@@ -29,6 +29,7 @@ import {
29
29
  OLDEST_BUNDLE_FORMAT_VERSION,
30
30
  } from './model.ts'
31
31
  import { type BundlePeople, openPeople, type PeopleBlock, parsePeopleBlock } from './people.ts'
32
+ import { hashFile, isPublicPath, PUBLIC_MANIFEST, type PublicFile } from './public.ts'
32
33
  import { openBytes, parseSealParameters, unlock } from './seal.ts'
33
34
  import {
34
35
  openSiteData,
@@ -375,6 +376,7 @@ function parseManifest(text: string, problems: BundleProblem[]): BundleManifest
375
376
  // Filled from `media.json` by the loader, which the hash covers (story 195).
376
377
  blobs: [],
377
378
  ...(raw.mediaManifest === MEDIA_MANIFEST ? { mediaManifest: MEDIA_MANIFEST } : {}),
379
+ ...(raw.publicManifest === PUBLIC_MANIFEST ? { publicManifest: PUBLIC_MANIFEST } : {}),
378
380
  ...(isRecord(raw.carries) ? { carries: parseCarries(raw.carries) } : {}),
379
381
  ...(raw.people !== undefined && parsePeopleBlock(raw.people)
380
382
  ? { people: parsePeopleBlock(raw.people) as PeopleBlock }
@@ -406,6 +408,8 @@ function parseRequires(raw: Record<string, unknown>): Record<string, string> {
406
408
  function parseCarries(raw: Record<string, unknown>): Partial<Record<BundleSection, string[]>> {
407
409
  const out: Partial<Record<BundleSection, string[]>> = {}
408
410
  for (const section of Object.keys(BUNDLE_SECTIONS) as BundleSection[]) {
411
+ // Only ever from `public.json`, whose list is what the hash covers.
412
+ if (section === 'public') continue
409
413
  const paths = raw[section]
410
414
  if (!Array.isArray(paths)) continue
411
415
  const safe = paths.flatMap((path) =>
@@ -481,6 +485,53 @@ function readMediaManifest(
481
485
  return { blobs, text }
482
486
  }
483
487
 
488
+ /**
489
+ * `public.json`, read and checked: undefined when the bundle has none, and a
490
+ * problem when `bundle.json` says it has and it is missing or malformed.
491
+ */
492
+ function readPublicManifest(
493
+ dir: string,
494
+ manifest: BundleManifest,
495
+ problems: BundleProblem[],
496
+ ): { files: PublicFile[]; text: string } | undefined {
497
+ const file = join(dir, PUBLIC_MANIFEST)
498
+ if (!manifest.publicManifest) {
499
+ if (existsSync(file))
500
+ problems.push({ file: PUBLIC_MANIFEST, message: 'bundle.json does not declare this file' })
501
+ return undefined
502
+ }
503
+ if (!existsSync(file)) {
504
+ problems.push({ file: PUBLIC_MANIFEST, message: 'bundle.json declares it, and it is missing' })
505
+ return undefined
506
+ }
507
+ const text = readFileSync(file, 'utf8')
508
+ let raw: unknown
509
+ try {
510
+ raw = JSON.parse(text)
511
+ } catch {
512
+ problems.push({ file: PUBLIC_MANIFEST, message: 'not valid JSON' })
513
+ return undefined
514
+ }
515
+ const entries = isRecord(raw) && Array.isArray(raw.files) ? raw.files : undefined
516
+ if (!entries) {
517
+ problems.push({ file: PUBLIC_MANIFEST, message: 'has no list of files' })
518
+ return undefined
519
+ }
520
+ const files = entries.flatMap((entry): PublicFile[] => {
521
+ if (
522
+ isRecord(entry) &&
523
+ typeof entry.path === 'string' &&
524
+ isPublicPath(entry.path) &&
525
+ typeof entry.size === 'number' &&
526
+ typeof entry.sha256 === 'string'
527
+ )
528
+ return [{ path: entry.path, size: entry.size, sha256: entry.sha256 }]
529
+ problems.push({ file: PUBLIC_MANIFEST, message: 'lists a file with no usable path or hash' })
530
+ return []
531
+ })
532
+ return { files, text }
533
+ }
534
+
484
535
  /** A media key that stays inside `media/`. */
485
536
  const isSafeKey = (key: string): boolean =>
486
537
  key !== '' && !key.startsWith('/') && !key.split('/').some((part) => part === '' || part === '..')
@@ -571,6 +622,38 @@ export function loadBundle(dir: string): LoadedBundle {
571
622
  }
572
623
  }
573
624
 
625
+ // Public files are checked one at a time against `public.json`, which the
626
+ // bundle's hash covers, so none has to be held in memory.
627
+ const publicList = readPublicManifest(dir, manifest, problems)
628
+ if (publicList) {
629
+ read.push({ path: PUBLIC_MANIFEST, text: publicList.text })
630
+ manifest.publicFiles = publicList.files
631
+ manifest.carries = { ...manifest.carries, public: publicList.files.map((f) => f.path).sort() }
632
+ for (const { path, size, sha256 } of publicList.files) {
633
+ const file = join(dir, path)
634
+ if (!existsSync(file)) {
635
+ problems.push({
636
+ file: path,
637
+ message: 'the manifest says this file is carried, and it is not',
638
+ })
639
+ continue
640
+ }
641
+ if (statSync(file).size !== size || hashFile(file) !== sha256)
642
+ problems.push({ file: path, message: 'is not the file public.json lists' })
643
+ files.set(path, file)
644
+ }
645
+ }
646
+ const publicDir = join(dir, BUNDLE_SECTIONS.public)
647
+ if (existsSync(publicDir)) {
648
+ const listed = new Set(publicList?.files.map((f) => f.path))
649
+ for (const entry of readdirSync(publicDir, { recursive: true, withFileTypes: true })) {
650
+ if (!entry.isFile()) continue
651
+ const path = relative(dir, join(entry.parentPath, entry.name)).split(sep).join('/')
652
+ if (!listed.has(path))
653
+ problems.push({ file: path, message: 'this file is not listed in public.json' })
654
+ }
655
+ }
656
+
574
657
  // People are sealed bytes, opened only with the passphrase at install; here
575
658
  // they are only read, and counted in the hash like everything else.
576
659
  const people = new Map<string, Uint8Array>()
package/src/model.ts CHANGED
@@ -9,6 +9,7 @@
9
9
  */
10
10
 
11
11
  import type { BundlePeople, PeopleBlock } from './people.ts'
12
+ import type { PublicFile } from './public.ts'
12
13
  import type { SealParameters } from './seal.ts'
13
14
  import type { SiteDataBlock, SiteDataTable } from './site-data.ts'
14
15
 
@@ -21,11 +22,12 @@ import type { SiteDataBlock, SiteDataTable } from './site-data.ts'
21
22
  * 5: media listed in `media.json` at the root, each carried file at
22
23
  * `media/<key>` with a `bundle://` URI (story 195).
23
24
  * 6: a Library folder may travel as a node (story 283).
25
+ * 7: the site's `public/` folder, listed in `public.json` (story 284).
24
26
  *
25
27
  * An older version is refused rather than read (no older layout is kept up),
26
28
  * with a message saying to rebuild it.
27
29
  */
28
- export const BUNDLE_FORMAT_VERSION = 6
30
+ export const BUNDLE_FORMAT_VERSION = 7
29
31
 
30
32
  /** The oldest version this reader reads: the first with media listed in `media.json`. */
31
33
  export const OLDEST_BUNDLE_FORMAT_VERSION = 5
@@ -56,6 +58,11 @@ export const BUNDLE_SECTIONS = {
56
58
  * `installBundleCode`'s, followed by a restart.
57
59
  */
58
60
  code: 'code',
61
+ /**
62
+ * The site's `public/` folder (story 284), which may be images: read as bytes,
63
+ * listed with their hashes in `public.json` rather than in `carries`.
64
+ */
65
+ public: 'public',
59
66
  } as const
60
67
 
61
68
  export type BundleSection = keyof typeof BUNDLE_SECTIONS
@@ -243,6 +250,10 @@ export interface BundleManifest {
243
250
  * the package root and inside the hash (story 195). `blobs` is read from it.
244
251
  */
245
252
  mediaManifest?: string
253
+ /** `public.json`, when the bundle carries the site's `public/` folder (story 284). */
254
+ publicManifest?: string
255
+ /** The public files, read from `public.json`; `carries.public` holds their paths. */
256
+ publicFiles?: PublicFile[]
246
257
  /**
247
258
  * The files this bundle carries beyond its content and media, by section, in
248
259
  * path order — the structure and the views that make a bundle installable
package/src/public.ts ADDED
@@ -0,0 +1,53 @@
1
+ /**
2
+ * The site's `public/` folder in a bundle (story 284). Its files can be large,
3
+ * so `public.json` lists each with its size and hash, the bundle's hash covers
4
+ * that list, and no file is held in memory. `docs/bundles.md`, "Public files".
5
+ */
6
+ import { closeSync, openSync, readSync } from 'node:fs'
7
+ import { BUNDLE_SECTIONS } from './model.ts'
8
+
9
+ /** The list of public files, at the package root beside `bundle.json`. */
10
+ export const PUBLIC_MANIFEST = 'public.json'
11
+
12
+ /** A public file a bundle carries, at its path inside the bundle. */
13
+ export interface PublicFile {
14
+ path: string
15
+ size: number
16
+ sha256: string
17
+ }
18
+
19
+ /** A public file on its way into a bundle, and where its bytes are. */
20
+ export interface PublicSource extends PublicFile {
21
+ source: string
22
+ }
23
+
24
+ /** A path that stays inside the bundle's `public/`. */
25
+ export const isPublicPath = (path: string): boolean => {
26
+ const [top, ...rest] = path.split('/')
27
+ return (
28
+ top === BUNDLE_SECTIONS.public &&
29
+ rest.length > 0 &&
30
+ !rest.some((part) => part === '' || part === '.' || part === '..' || part.includes('\\'))
31
+ )
32
+ }
33
+
34
+ /** A file's SHA-256, read a piece at a time. */
35
+ export function hashFile(file: string): string {
36
+ const hasher = new Bun.CryptoHasher('sha256')
37
+ const piece = new Uint8Array(1024 * 1024)
38
+ const fd = openSync(file, 'r')
39
+ try {
40
+ for (let read = readSync(fd, piece); read > 0; read = readSync(fd, piece))
41
+ hasher.update(piece.subarray(0, read))
42
+ } finally {
43
+ closeSync(fd)
44
+ }
45
+ return hasher.digest('hex')
46
+ }
47
+
48
+ export function writePublicManifest(files: readonly PublicFile[]): string {
49
+ const entries = [...files]
50
+ .sort((a, b) => a.path.localeCompare(b.path))
51
+ .map(({ path, size, sha256 }) => ({ path, size, sha256 }))
52
+ return `${JSON.stringify({ files: entries }, null, 2)}\n`
53
+ }
package/src/write.ts CHANGED
@@ -24,6 +24,7 @@ import {
24
24
  type ContentSet,
25
25
  } from './model.ts'
26
26
  import { type BundlePeople, sealPeople } from './people.ts'
27
+ import { PUBLIC_MANIFEST, type PublicSource, writePublicManifest } from './public.ts'
27
28
  import { type Seal, sealBytes } from './seal.ts'
28
29
  import { type SiteDataTable, sealSiteData } from './site-data.ts'
29
30
 
@@ -34,6 +35,8 @@ export interface BundleFile {
34
35
  text?: string
35
36
  /** A media file's bytes; `text` and `bytes` are never both set. */
36
37
  bytes?: Uint8Array
38
+ /** A public file, copied from disk in place of `text` or `bytes`; never in the bundle's hash. */
39
+ source?: string
37
40
  }
38
41
 
39
42
  const json = (value: unknown): string => `${JSON.stringify(value, null, 2)}\n`
@@ -198,6 +201,7 @@ export function writeManifest(manifest: BundleManifest): string {
198
201
  // The files themselves are listed in `media.json`, which the hash covers;
199
202
  // this file, which holds the hash, only says that there is one (story 195).
200
203
  ...(manifest.blobs.length > 0 ? { mediaManifest: MEDIA_MANIFEST } : {}),
204
+ ...(manifest.publicManifest ? { publicManifest: manifest.publicManifest } : {}),
201
205
  // Absent on a content-only bundle, so version 1 output is unchanged.
202
206
  ...(manifest.carries && Object.keys(manifest.carries).length > 0
203
207
  ? { carries: sortedCarries(manifest.carries) }
@@ -307,6 +311,8 @@ export interface WriteOptions {
307
311
  people?: BundlePeople
308
312
  /** The site's and bundles' own tables: always sealed. */
309
313
  siteData?: readonly SiteDataTable[]
314
+ /** The site's `public/` files, which `public.json` lists and the caller's bytes follow. */
315
+ public?: readonly PublicSource[]
310
316
  }
311
317
 
312
318
  /** Without values, media files or usernames: what `omitted` writes. Pure. */
@@ -357,7 +363,13 @@ export function writeBundle(
357
363
  ...planned.manifest,
358
364
  integrity: bundleIntegrity([...planned.files, ...carried]),
359
365
  }
360
- return [{ path: MANIFEST_FILE, text: writeManifest(manifest) }, ...planned.files, ...carried]
366
+ const publicFiles = (options.public ?? []).map(({ path, source }) => ({ path, source }))
367
+ return [
368
+ { path: MANIFEST_FILE, text: writeManifest(manifest) },
369
+ ...planned.files,
370
+ ...carried,
371
+ ...publicFiles,
372
+ ]
361
373
  }
362
374
 
363
375
  /**
@@ -365,7 +377,8 @@ export function writeBundle(
365
377
  * every other file, from each carried file's size alone. The caller writes
366
378
  * `blobs` at `mediaPath` — sealed with `sealBytes` under the seal when `sealing`
367
379
  * — computes `bundleIntegrity` over those and `files` in path order, and writes
368
- * the manifest with it.
380
+ * the manifest with it. `options.public` is the caller's to write too, outside
381
+ * that hash.
369
382
  */
370
383
  export function planStreamedBundle(
371
384
  set: ContentSet,
@@ -410,16 +423,18 @@ function planBundle(
410
423
  const carries: Partial<Record<BundleSection, string[]>> = {}
411
424
  for (const file of extra) {
412
425
  const section = sectionOf(file.path)
413
- if (!section) throw new Error(`${file.path} is not in a bundle section`)
426
+ if (!section || section === 'public') throw new Error(`${file.path} is not in a bundle section`)
414
427
  carries[section] = [...(carries[section] ?? []), file.path]
415
428
  }
416
429
 
430
+ const publicFiles = options.public ?? []
417
431
  const people = options.people && seal ? sealPeople(options.people, seal) : undefined
418
432
  const siteData = options.siteData && seal ? sealSiteData(options.siteData, seal) : undefined
419
433
  const manifest: BundleManifest = {
420
434
  ...set.manifest,
421
435
  formatVersion: BUNDLE_FORMAT_VERSION,
422
436
  // Without values nothing names a media file, so none is expected either.
437
+ publicManifest: publicFiles.length > 0 ? PUBLIC_MANIFEST : undefined,
423
438
  blobs: (content === 'omitted' ? [] : set.manifest.blobs).map((blob) => {
424
439
  const size = carriedSizes.get(blob.key)
425
440
  return size !== undefined ? { ...blob, included: true, size } : { ...blob, included: false }
@@ -444,6 +459,9 @@ function planBundle(
444
459
  ...(people?.files ?? []),
445
460
  ...(siteData?.files ?? []),
446
461
  ...media,
462
+ ...(publicFiles.length > 0
463
+ ? [{ path: PUBLIC_MANIFEST, text: writePublicManifest(publicFiles) }]
464
+ : []),
447
465
  ],
448
466
  blobs: [...carriedSizes.keys()].sort((a, b) => a.localeCompare(b)),
449
467
  sealing,