@crombolo/transfer 0.50.0 → 0.51.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/code.ts +1 -0
- package/src/index.ts +1 -0
- package/src/load.ts +84 -1
- package/src/model.ts +12 -1
- package/src/public.ts +53 -0
- package/src/write.ts +21 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crombolo/transfer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.51.1",
|
|
4
4
|
"description": "Internal to crombolo, with no compatibility promise: use @crombolo/crombolo. Content transfer for crombolo: bundles of content moved between environments.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Crombolo",
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"access": "public"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@crombolo/core": "0.
|
|
32
|
-
"@crombolo/data": "0.
|
|
31
|
+
"@crombolo/core": "0.51.1",
|
|
32
|
+
"@crombolo/data": "0.51.1"
|
|
33
33
|
}
|
|
34
34
|
}
|
package/src/code.ts
CHANGED
package/src/index.ts
CHANGED
package/src/load.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* a crash rather than a finding.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
import { existsSync, readdirSync, readFileSync } from 'node:fs'
|
|
10
|
+
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
|
|
11
11
|
import { join, relative, sep } from 'node:path'
|
|
12
12
|
import {
|
|
13
13
|
BUNDLE_FORMAT_VERSION,
|
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
OLDEST_BUNDLE_FORMAT_VERSION,
|
|
30
30
|
} from './model.ts'
|
|
31
31
|
import { type BundlePeople, openPeople, type PeopleBlock, parsePeopleBlock } from './people.ts'
|
|
32
|
+
import { hashFile, isPublicPath, PUBLIC_MANIFEST, type PublicFile } from './public.ts'
|
|
32
33
|
import { openBytes, parseSealParameters, unlock } from './seal.ts'
|
|
33
34
|
import {
|
|
34
35
|
openSiteData,
|
|
@@ -375,6 +376,7 @@ function parseManifest(text: string, problems: BundleProblem[]): BundleManifest
|
|
|
375
376
|
// Filled from `media.json` by the loader, which the hash covers (story 195).
|
|
376
377
|
blobs: [],
|
|
377
378
|
...(raw.mediaManifest === MEDIA_MANIFEST ? { mediaManifest: MEDIA_MANIFEST } : {}),
|
|
379
|
+
...(raw.publicManifest === PUBLIC_MANIFEST ? { publicManifest: PUBLIC_MANIFEST } : {}),
|
|
378
380
|
...(isRecord(raw.carries) ? { carries: parseCarries(raw.carries) } : {}),
|
|
379
381
|
...(raw.people !== undefined && parsePeopleBlock(raw.people)
|
|
380
382
|
? { people: parsePeopleBlock(raw.people) as PeopleBlock }
|
|
@@ -406,6 +408,8 @@ function parseRequires(raw: Record<string, unknown>): Record<string, string> {
|
|
|
406
408
|
function parseCarries(raw: Record<string, unknown>): Partial<Record<BundleSection, string[]>> {
|
|
407
409
|
const out: Partial<Record<BundleSection, string[]>> = {}
|
|
408
410
|
for (const section of Object.keys(BUNDLE_SECTIONS) as BundleSection[]) {
|
|
411
|
+
// Only ever from `public.json`, whose list is what the hash covers.
|
|
412
|
+
if (section === 'public') continue
|
|
409
413
|
const paths = raw[section]
|
|
410
414
|
if (!Array.isArray(paths)) continue
|
|
411
415
|
const safe = paths.flatMap((path) =>
|
|
@@ -481,6 +485,53 @@ function readMediaManifest(
|
|
|
481
485
|
return { blobs, text }
|
|
482
486
|
}
|
|
483
487
|
|
|
488
|
+
/**
|
|
489
|
+
* `public.json`, read and checked: undefined when the bundle has none, and a
|
|
490
|
+
* problem when `bundle.json` says it has and it is missing or malformed.
|
|
491
|
+
*/
|
|
492
|
+
function readPublicManifest(
|
|
493
|
+
dir: string,
|
|
494
|
+
manifest: BundleManifest,
|
|
495
|
+
problems: BundleProblem[],
|
|
496
|
+
): { files: PublicFile[]; text: string } | undefined {
|
|
497
|
+
const file = join(dir, PUBLIC_MANIFEST)
|
|
498
|
+
if (!manifest.publicManifest) {
|
|
499
|
+
if (existsSync(file))
|
|
500
|
+
problems.push({ file: PUBLIC_MANIFEST, message: 'bundle.json does not declare this file' })
|
|
501
|
+
return undefined
|
|
502
|
+
}
|
|
503
|
+
if (!existsSync(file)) {
|
|
504
|
+
problems.push({ file: PUBLIC_MANIFEST, message: 'bundle.json declares it, and it is missing' })
|
|
505
|
+
return undefined
|
|
506
|
+
}
|
|
507
|
+
const text = readFileSync(file, 'utf8')
|
|
508
|
+
let raw: unknown
|
|
509
|
+
try {
|
|
510
|
+
raw = JSON.parse(text)
|
|
511
|
+
} catch {
|
|
512
|
+
problems.push({ file: PUBLIC_MANIFEST, message: 'not valid JSON' })
|
|
513
|
+
return undefined
|
|
514
|
+
}
|
|
515
|
+
const entries = isRecord(raw) && Array.isArray(raw.files) ? raw.files : undefined
|
|
516
|
+
if (!entries) {
|
|
517
|
+
problems.push({ file: PUBLIC_MANIFEST, message: 'has no list of files' })
|
|
518
|
+
return undefined
|
|
519
|
+
}
|
|
520
|
+
const files = entries.flatMap((entry): PublicFile[] => {
|
|
521
|
+
if (
|
|
522
|
+
isRecord(entry) &&
|
|
523
|
+
typeof entry.path === 'string' &&
|
|
524
|
+
isPublicPath(entry.path) &&
|
|
525
|
+
typeof entry.size === 'number' &&
|
|
526
|
+
typeof entry.sha256 === 'string'
|
|
527
|
+
)
|
|
528
|
+
return [{ path: entry.path, size: entry.size, sha256: entry.sha256 }]
|
|
529
|
+
problems.push({ file: PUBLIC_MANIFEST, message: 'lists a file with no usable path or hash' })
|
|
530
|
+
return []
|
|
531
|
+
})
|
|
532
|
+
return { files, text }
|
|
533
|
+
}
|
|
534
|
+
|
|
484
535
|
/** A media key that stays inside `media/`. */
|
|
485
536
|
const isSafeKey = (key: string): boolean =>
|
|
486
537
|
key !== '' && !key.startsWith('/') && !key.split('/').some((part) => part === '' || part === '..')
|
|
@@ -571,6 +622,38 @@ export function loadBundle(dir: string): LoadedBundle {
|
|
|
571
622
|
}
|
|
572
623
|
}
|
|
573
624
|
|
|
625
|
+
// Public files are checked one at a time against `public.json`, which the
|
|
626
|
+
// bundle's hash covers, so none has to be held in memory.
|
|
627
|
+
const publicList = readPublicManifest(dir, manifest, problems)
|
|
628
|
+
if (publicList) {
|
|
629
|
+
read.push({ path: PUBLIC_MANIFEST, text: publicList.text })
|
|
630
|
+
manifest.publicFiles = publicList.files
|
|
631
|
+
manifest.carries = { ...manifest.carries, public: publicList.files.map((f) => f.path).sort() }
|
|
632
|
+
for (const { path, size, sha256 } of publicList.files) {
|
|
633
|
+
const file = join(dir, path)
|
|
634
|
+
if (!existsSync(file)) {
|
|
635
|
+
problems.push({
|
|
636
|
+
file: path,
|
|
637
|
+
message: 'the manifest says this file is carried, and it is not',
|
|
638
|
+
})
|
|
639
|
+
continue
|
|
640
|
+
}
|
|
641
|
+
if (statSync(file).size !== size || hashFile(file) !== sha256)
|
|
642
|
+
problems.push({ file: path, message: 'is not the file public.json lists' })
|
|
643
|
+
files.set(path, file)
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
const publicDir = join(dir, BUNDLE_SECTIONS.public)
|
|
647
|
+
if (existsSync(publicDir)) {
|
|
648
|
+
const listed = new Set(publicList?.files.map((f) => f.path))
|
|
649
|
+
for (const entry of readdirSync(publicDir, { recursive: true, withFileTypes: true })) {
|
|
650
|
+
if (!entry.isFile()) continue
|
|
651
|
+
const path = relative(dir, join(entry.parentPath, entry.name)).split(sep).join('/')
|
|
652
|
+
if (!listed.has(path))
|
|
653
|
+
problems.push({ file: path, message: 'this file is not listed in public.json' })
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
|
|
574
657
|
// People are sealed bytes, opened only with the passphrase at install; here
|
|
575
658
|
// they are only read, and counted in the hash like everything else.
|
|
576
659
|
const people = new Map<string, Uint8Array>()
|
package/src/model.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import type { BundlePeople, PeopleBlock } from './people.ts'
|
|
12
|
+
import type { PublicFile } from './public.ts'
|
|
12
13
|
import type { SealParameters } from './seal.ts'
|
|
13
14
|
import type { SiteDataBlock, SiteDataTable } from './site-data.ts'
|
|
14
15
|
|
|
@@ -21,11 +22,12 @@ import type { SiteDataBlock, SiteDataTable } from './site-data.ts'
|
|
|
21
22
|
* 5: media listed in `media.json` at the root, each carried file at
|
|
22
23
|
* `media/<key>` with a `bundle://` URI (story 195).
|
|
23
24
|
* 6: a Library folder may travel as a node (story 283).
|
|
25
|
+
* 7: the site's `public/` folder, listed in `public.json` (story 284).
|
|
24
26
|
*
|
|
25
27
|
* An older version is refused rather than read (no older layout is kept up),
|
|
26
28
|
* with a message saying to rebuild it.
|
|
27
29
|
*/
|
|
28
|
-
export const BUNDLE_FORMAT_VERSION =
|
|
30
|
+
export const BUNDLE_FORMAT_VERSION = 7
|
|
29
31
|
|
|
30
32
|
/** The oldest version this reader reads: the first with media listed in `media.json`. */
|
|
31
33
|
export const OLDEST_BUNDLE_FORMAT_VERSION = 5
|
|
@@ -56,6 +58,11 @@ export const BUNDLE_SECTIONS = {
|
|
|
56
58
|
* `installBundleCode`'s, followed by a restart.
|
|
57
59
|
*/
|
|
58
60
|
code: 'code',
|
|
61
|
+
/**
|
|
62
|
+
* The site's `public/` folder (story 284), which may be images: read as bytes,
|
|
63
|
+
* listed with their hashes in `public.json` rather than in `carries`.
|
|
64
|
+
*/
|
|
65
|
+
public: 'public',
|
|
59
66
|
} as const
|
|
60
67
|
|
|
61
68
|
export type BundleSection = keyof typeof BUNDLE_SECTIONS
|
|
@@ -243,6 +250,10 @@ export interface BundleManifest {
|
|
|
243
250
|
* the package root and inside the hash (story 195). `blobs` is read from it.
|
|
244
251
|
*/
|
|
245
252
|
mediaManifest?: string
|
|
253
|
+
/** `public.json`, when the bundle carries the site's `public/` folder (story 284). */
|
|
254
|
+
publicManifest?: string
|
|
255
|
+
/** The public files, read from `public.json`; `carries.public` holds their paths. */
|
|
256
|
+
publicFiles?: PublicFile[]
|
|
246
257
|
/**
|
|
247
258
|
* The files this bundle carries beyond its content and media, by section, in
|
|
248
259
|
* path order — the structure and the views that make a bundle installable
|
package/src/public.ts
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The site's `public/` folder in a bundle (story 284). Its files can be large,
|
|
3
|
+
* so `public.json` lists each with its size and hash, the bundle's hash covers
|
|
4
|
+
* that list, and no file is held in memory. `docs/bundles.md`, "Public files".
|
|
5
|
+
*/
|
|
6
|
+
import { closeSync, openSync, readSync } from 'node:fs'
|
|
7
|
+
import { BUNDLE_SECTIONS } from './model.ts'
|
|
8
|
+
|
|
9
|
+
/** The list of public files, at the package root beside `bundle.json`. */
|
|
10
|
+
export const PUBLIC_MANIFEST = 'public.json'
|
|
11
|
+
|
|
12
|
+
/** A public file a bundle carries, at its path inside the bundle. */
|
|
13
|
+
export interface PublicFile {
|
|
14
|
+
path: string
|
|
15
|
+
size: number
|
|
16
|
+
sha256: string
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** A public file on its way into a bundle, and where its bytes are. */
|
|
20
|
+
export interface PublicSource extends PublicFile {
|
|
21
|
+
source: string
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** A path that stays inside the bundle's `public/`. */
|
|
25
|
+
export const isPublicPath = (path: string): boolean => {
|
|
26
|
+
const [top, ...rest] = path.split('/')
|
|
27
|
+
return (
|
|
28
|
+
top === BUNDLE_SECTIONS.public &&
|
|
29
|
+
rest.length > 0 &&
|
|
30
|
+
!rest.some((part) => part === '' || part === '.' || part === '..' || part.includes('\\'))
|
|
31
|
+
)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** A file's SHA-256, read a piece at a time. */
|
|
35
|
+
export function hashFile(file: string): string {
|
|
36
|
+
const hasher = new Bun.CryptoHasher('sha256')
|
|
37
|
+
const piece = new Uint8Array(1024 * 1024)
|
|
38
|
+
const fd = openSync(file, 'r')
|
|
39
|
+
try {
|
|
40
|
+
for (let read = readSync(fd, piece); read > 0; read = readSync(fd, piece))
|
|
41
|
+
hasher.update(piece.subarray(0, read))
|
|
42
|
+
} finally {
|
|
43
|
+
closeSync(fd)
|
|
44
|
+
}
|
|
45
|
+
return hasher.digest('hex')
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function writePublicManifest(files: readonly PublicFile[]): string {
|
|
49
|
+
const entries = [...files]
|
|
50
|
+
.sort((a, b) => a.path.localeCompare(b.path))
|
|
51
|
+
.map(({ path, size, sha256 }) => ({ path, size, sha256 }))
|
|
52
|
+
return `${JSON.stringify({ files: entries }, null, 2)}\n`
|
|
53
|
+
}
|
package/src/write.ts
CHANGED
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
type ContentSet,
|
|
25
25
|
} from './model.ts'
|
|
26
26
|
import { type BundlePeople, sealPeople } from './people.ts'
|
|
27
|
+
import { PUBLIC_MANIFEST, type PublicSource, writePublicManifest } from './public.ts'
|
|
27
28
|
import { type Seal, sealBytes } from './seal.ts'
|
|
28
29
|
import { type SiteDataTable, sealSiteData } from './site-data.ts'
|
|
29
30
|
|
|
@@ -34,6 +35,8 @@ export interface BundleFile {
|
|
|
34
35
|
text?: string
|
|
35
36
|
/** A media file's bytes; `text` and `bytes` are never both set. */
|
|
36
37
|
bytes?: Uint8Array
|
|
38
|
+
/** A public file, copied from disk in place of `text` or `bytes`; never in the bundle's hash. */
|
|
39
|
+
source?: string
|
|
37
40
|
}
|
|
38
41
|
|
|
39
42
|
const json = (value: unknown): string => `${JSON.stringify(value, null, 2)}\n`
|
|
@@ -198,6 +201,7 @@ export function writeManifest(manifest: BundleManifest): string {
|
|
|
198
201
|
// The files themselves are listed in `media.json`, which the hash covers;
|
|
199
202
|
// this file, which holds the hash, only says that there is one (story 195).
|
|
200
203
|
...(manifest.blobs.length > 0 ? { mediaManifest: MEDIA_MANIFEST } : {}),
|
|
204
|
+
...(manifest.publicManifest ? { publicManifest: manifest.publicManifest } : {}),
|
|
201
205
|
// Absent on a content-only bundle, so version 1 output is unchanged.
|
|
202
206
|
...(manifest.carries && Object.keys(manifest.carries).length > 0
|
|
203
207
|
? { carries: sortedCarries(manifest.carries) }
|
|
@@ -307,6 +311,8 @@ export interface WriteOptions {
|
|
|
307
311
|
people?: BundlePeople
|
|
308
312
|
/** The site's and bundles' own tables: always sealed. */
|
|
309
313
|
siteData?: readonly SiteDataTable[]
|
|
314
|
+
/** The site's `public/` files, which `public.json` lists and the caller's bytes follow. */
|
|
315
|
+
public?: readonly PublicSource[]
|
|
310
316
|
}
|
|
311
317
|
|
|
312
318
|
/** Without values, media files or usernames: what `omitted` writes. Pure. */
|
|
@@ -357,7 +363,13 @@ export function writeBundle(
|
|
|
357
363
|
...planned.manifest,
|
|
358
364
|
integrity: bundleIntegrity([...planned.files, ...carried]),
|
|
359
365
|
}
|
|
360
|
-
|
|
366
|
+
const publicFiles = (options.public ?? []).map(({ path, source }) => ({ path, source }))
|
|
367
|
+
return [
|
|
368
|
+
{ path: MANIFEST_FILE, text: writeManifest(manifest) },
|
|
369
|
+
...planned.files,
|
|
370
|
+
...carried,
|
|
371
|
+
...publicFiles,
|
|
372
|
+
]
|
|
361
373
|
}
|
|
362
374
|
|
|
363
375
|
/**
|
|
@@ -365,7 +377,8 @@ export function writeBundle(
|
|
|
365
377
|
* every other file, from each carried file's size alone. The caller writes
|
|
366
378
|
* `blobs` at `mediaPath` — sealed with `sealBytes` under the seal when `sealing`
|
|
367
379
|
* — computes `bundleIntegrity` over those and `files` in path order, and writes
|
|
368
|
-
* the manifest with it.
|
|
380
|
+
* the manifest with it. `options.public` is the caller's to write too, outside
|
|
381
|
+
* that hash.
|
|
369
382
|
*/
|
|
370
383
|
export function planStreamedBundle(
|
|
371
384
|
set: ContentSet,
|
|
@@ -410,16 +423,18 @@ function planBundle(
|
|
|
410
423
|
const carries: Partial<Record<BundleSection, string[]>> = {}
|
|
411
424
|
for (const file of extra) {
|
|
412
425
|
const section = sectionOf(file.path)
|
|
413
|
-
if (!section) throw new Error(`${file.path} is not in a bundle section`)
|
|
426
|
+
if (!section || section === 'public') throw new Error(`${file.path} is not in a bundle section`)
|
|
414
427
|
carries[section] = [...(carries[section] ?? []), file.path]
|
|
415
428
|
}
|
|
416
429
|
|
|
430
|
+
const publicFiles = options.public ?? []
|
|
417
431
|
const people = options.people && seal ? sealPeople(options.people, seal) : undefined
|
|
418
432
|
const siteData = options.siteData && seal ? sealSiteData(options.siteData, seal) : undefined
|
|
419
433
|
const manifest: BundleManifest = {
|
|
420
434
|
...set.manifest,
|
|
421
435
|
formatVersion: BUNDLE_FORMAT_VERSION,
|
|
422
436
|
// Without values nothing names a media file, so none is expected either.
|
|
437
|
+
publicManifest: publicFiles.length > 0 ? PUBLIC_MANIFEST : undefined,
|
|
423
438
|
blobs: (content === 'omitted' ? [] : set.manifest.blobs).map((blob) => {
|
|
424
439
|
const size = carriedSizes.get(blob.key)
|
|
425
440
|
return size !== undefined ? { ...blob, included: true, size } : { ...blob, included: false }
|
|
@@ -444,6 +459,9 @@ function planBundle(
|
|
|
444
459
|
...(people?.files ?? []),
|
|
445
460
|
...(siteData?.files ?? []),
|
|
446
461
|
...media,
|
|
462
|
+
...(publicFiles.length > 0
|
|
463
|
+
? [{ path: PUBLIC_MANIFEST, text: writePublicManifest(publicFiles) }]
|
|
464
|
+
: []),
|
|
447
465
|
],
|
|
448
466
|
blobs: [...carriedSizes.keys()].sort((a, b) => a.localeCompare(b)),
|
|
449
467
|
sealing,
|