@crombolo/transfer 0.31.0 → 0.33.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/code.ts +115 -0
- package/src/index.ts +1 -0
- package/src/load.ts +15 -1
- package/src/model.ts +16 -13
- package/src/package.ts +33 -17
- package/src/write.ts +1 -15
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crombolo/transfer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.33.1",
|
|
4
4
|
"description": "Content transfer for crombolo: bundles of content moved between environments.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Crombolo",
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"access": "public"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@crombolo/core": "0.
|
|
32
|
-
"@crombolo/data": "0.
|
|
31
|
+
"@crombolo/core": "0.33.1",
|
|
32
|
+
"@crombolo/data": "0.33.1"
|
|
33
33
|
}
|
|
34
34
|
}
|
package/src/code.ts
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A site's own code, carried in its bundle (stories 192, 193): the files its
|
|
3
|
+
* `package.json` names in `crombolo.code`, at their site paths under `code/`,
|
|
4
|
+
* listed in `bundle.json` and covered by its one hash like every other section.
|
|
5
|
+
* `docs/bundles.md`, "A site's bundle with code".
|
|
6
|
+
*/
|
|
7
|
+
import { existsSync, readFileSync } from 'node:fs'
|
|
8
|
+
import { isAbsolute, join, sep } from 'node:path'
|
|
9
|
+
import { loadBundle } from './load.ts'
|
|
10
|
+
import { BUNDLE_SECTIONS } from './model.ts'
|
|
11
|
+
import type { BundleFile } from './write.ts'
|
|
12
|
+
|
|
13
|
+
/** The section, and directory, that holds the site's code. */
|
|
14
|
+
export const CODE_DIR = BUNDLE_SECTIONS.code
|
|
15
|
+
|
|
16
|
+
/** Packed whenever there is code: what installing and booting it needs. */
|
|
17
|
+
export const CODE_ALWAYS = ['package.json', 'bun.lock', 'crombolo.config.ts'] as const
|
|
18
|
+
|
|
19
|
+
/** The folders a bundle already carries as sections, or that are a site's local state. */
|
|
20
|
+
const SITE_FOLDERS = new Set([
|
|
21
|
+
'schema',
|
|
22
|
+
'components',
|
|
23
|
+
'css',
|
|
24
|
+
'scripts',
|
|
25
|
+
'seed',
|
|
26
|
+
'media',
|
|
27
|
+
'logs',
|
|
28
|
+
'.crombolo',
|
|
29
|
+
])
|
|
30
|
+
|
|
31
|
+
/** Never packed, whatever a pattern matches: installs, history, secrets and databases. */
|
|
32
|
+
export function isNeverCode(path: string): boolean {
|
|
33
|
+
const parts = path.split('/')
|
|
34
|
+
const name = parts.at(-1) as string
|
|
35
|
+
return (
|
|
36
|
+
SITE_FOLDERS.has(parts[0] as string) ||
|
|
37
|
+
parts.some((part) => part === 'node_modules' || part === '.git') ||
|
|
38
|
+
name === '.env' ||
|
|
39
|
+
name.startsWith('.env.') ||
|
|
40
|
+
/\.sqlite(?:$|-|\.)/.test(name)
|
|
41
|
+
)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** `crombolo.code` in a site's `package.json`, or undefined when it names none. */
|
|
45
|
+
export function codePatternsOf(siteDir: string): string[] | undefined {
|
|
46
|
+
const file = join(siteDir, 'package.json')
|
|
47
|
+
if (!existsSync(file)) return undefined
|
|
48
|
+
const json = JSON.parse(readFileSync(file, 'utf8')) as { crombolo?: { code?: unknown } }
|
|
49
|
+
const code = json.crombolo?.code
|
|
50
|
+
return Array.isArray(code) ? code.filter((p): p is string => typeof p === 'string') : undefined
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The site's files the patterns name, plus `CODE_ALWAYS`, in path order. A
|
|
55
|
+
* pattern that matches nothing is a problem rather than a silent gap, and so is
|
|
56
|
+
* a site without the lockfile its code would be installed from.
|
|
57
|
+
*/
|
|
58
|
+
export function selectCodeFiles(
|
|
59
|
+
siteDir: string,
|
|
60
|
+
patterns: readonly string[],
|
|
61
|
+
): { paths: string[]; problems: string[] } {
|
|
62
|
+
const problems: string[] = []
|
|
63
|
+
const paths = new Set<string>()
|
|
64
|
+
for (const pattern of patterns) {
|
|
65
|
+
if (isAbsolute(pattern) || pattern.split('/').includes('..')) {
|
|
66
|
+
problems.push(`"${pattern}" reaches outside the site`)
|
|
67
|
+
continue
|
|
68
|
+
}
|
|
69
|
+
let matched = 0
|
|
70
|
+
for (const path of new Bun.Glob(pattern).scanSync({ cwd: siteDir, dot: true })) {
|
|
71
|
+
const normal = path.split(sep).join('/')
|
|
72
|
+
if (isNeverCode(normal)) continue
|
|
73
|
+
paths.add(normal)
|
|
74
|
+
matched++
|
|
75
|
+
}
|
|
76
|
+
if (matched === 0) problems.push(`"${pattern}" matches no file the bundle can carry`)
|
|
77
|
+
}
|
|
78
|
+
for (const always of CODE_ALWAYS) if (existsSync(join(siteDir, always))) paths.add(always)
|
|
79
|
+
if (!existsSync(join(siteDir, 'package.json'))) problems.push('the site has no package.json')
|
|
80
|
+
if (!existsSync(join(siteDir, 'bun.lock')))
|
|
81
|
+
problems.push('the site has no bun.lock, so its code could not be installed as it was tested')
|
|
82
|
+
return { paths: [...paths].sort(), problems }
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** The files at those paths, as bytes and as the bundle carries them, under `code/`. */
|
|
86
|
+
export function readCodeFiles(siteDir: string, paths: readonly string[]): BundleFile[] {
|
|
87
|
+
return paths.map((path) => ({
|
|
88
|
+
path: `${CODE_DIR}/${path}`,
|
|
89
|
+
bytes: new Uint8Array(readFileSync(join(siteDir, path))),
|
|
90
|
+
}))
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export interface LoadedBundleCode {
|
|
94
|
+
/** The absolute directory holding the code, when the bundle carries some. */
|
|
95
|
+
dir?: string
|
|
96
|
+
/** Every file under it, relative to it and `/`-separated. */
|
|
97
|
+
paths: string[]
|
|
98
|
+
/** The bundle's own problems, its hash among them, with any about the code. */
|
|
99
|
+
problems: string[]
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** A bundle package's code, read through the bundle's own loader and its one hash. */
|
|
103
|
+
export function loadBundleCode(packageDir: string): LoadedBundleCode {
|
|
104
|
+
const loaded = loadBundle(packageDir)
|
|
105
|
+
const problems = loaded.problems.map((p) => `${p.file}: ${p.message}`)
|
|
106
|
+
const carried = loaded.set?.manifest.carries?.code
|
|
107
|
+
if (!loaded.set) return { paths: [], problems }
|
|
108
|
+
if (!carried) return { paths: [], problems: [...problems, 'the bundle carries no code'] }
|
|
109
|
+
const prefix = `${CODE_DIR}/`
|
|
110
|
+
return {
|
|
111
|
+
dir: join(packageDir, CODE_DIR),
|
|
112
|
+
paths: carried.map((path) => path.slice(prefix.length)),
|
|
113
|
+
problems,
|
|
114
|
+
}
|
|
115
|
+
}
|
package/src/index.ts
CHANGED
package/src/load.ts
CHANGED
|
@@ -23,6 +23,7 @@ import {
|
|
|
23
23
|
type BundleValue,
|
|
24
24
|
type BundleVariant,
|
|
25
25
|
type ContentSet,
|
|
26
|
+
OLDEST_BUNDLE_FORMAT_VERSION,
|
|
26
27
|
} from './model.ts'
|
|
27
28
|
import { type BundlePeople, openPeople, type PeopleBlock, parsePeopleBlock } from './people.ts'
|
|
28
29
|
import { openBytes, parseSealParameters, unlock } from './seal.ts'
|
|
@@ -237,6 +238,10 @@ function parseManifest(text: string, problems: BundleProblem[]): BundleManifest
|
|
|
237
238
|
})
|
|
238
239
|
return undefined
|
|
239
240
|
}
|
|
241
|
+
if (formatVersion < OLDEST_BUNDLE_FORMAT_VERSION) {
|
|
242
|
+
problems.push({ file: MANIFEST_FILE, message: OLD_LAYOUT })
|
|
243
|
+
return undefined
|
|
244
|
+
}
|
|
240
245
|
const provenance = isRecord(raw.provenance) ? raw.provenance : {}
|
|
241
246
|
const selector = isRecord(raw.selector) ? raw.selector : {}
|
|
242
247
|
const dependencies = isRecord(raw.dependencies) ? raw.dependencies : {}
|
|
@@ -348,6 +353,10 @@ function parseCarries(raw: Record<string, unknown>): Partial<Record<BundleSectio
|
|
|
348
353
|
return out
|
|
349
354
|
}
|
|
350
355
|
|
|
356
|
+
/** Why a bundle in the layout before format 4 is refused, and what to do about it. */
|
|
357
|
+
export const OLD_LAYOUT =
|
|
358
|
+
'this bundle was written before crombolo 0.33.1, in a layout no longer read; rebuild it with the command that made it (bundle build, content export, export, or a created package download)'
|
|
359
|
+
|
|
351
360
|
/** Reads a bundle directory. `set` is undefined when it cannot be trusted at all. */
|
|
352
361
|
export function loadBundle(dir: string): LoadedBundle {
|
|
353
362
|
const problems: BundleProblem[] = []
|
|
@@ -404,7 +413,12 @@ export function loadBundle(dir: string): LoadedBundle {
|
|
|
404
413
|
continue
|
|
405
414
|
}
|
|
406
415
|
files.set(path, file)
|
|
407
|
-
|
|
416
|
+
// Code is a site's source, which is not only text.
|
|
417
|
+
read.push(
|
|
418
|
+
path.startsWith(`${BUNDLE_SECTIONS.code}/`)
|
|
419
|
+
? { path, bytes: new Uint8Array(readFileSync(file)) }
|
|
420
|
+
: { path, text: readFileSync(file, 'utf8') },
|
|
421
|
+
)
|
|
408
422
|
}
|
|
409
423
|
}
|
|
410
424
|
|
package/src/model.ts
CHANGED
|
@@ -11,22 +11,19 @@
|
|
|
11
11
|
import type { BundlePeople, PeopleBlock } from './people.ts'
|
|
12
12
|
import type { SealParameters } from './seal.ts'
|
|
13
13
|
|
|
14
|
-
/** Bumped when a change to the shapes below stops an older reader understanding a bundle. */
|
|
15
|
-
export const BUNDLE_FORMAT_VERSION = 3
|
|
16
|
-
|
|
17
|
-
/** What a bundle with sections and nothing sealed writes: version 2 readers still read it. */
|
|
18
|
-
export const SECTIONS_FORMAT_VERSION = 2
|
|
19
|
-
|
|
20
14
|
/**
|
|
21
|
-
*
|
|
15
|
+
* Bumped when a change to the shapes below stops an older reader understanding
|
|
16
|
+
* a bundle. Every bundle is written at this version.
|
|
22
17
|
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* rather than failing an integrity check it cannot explain.
|
|
18
|
+
* 4 (0.33.1): `bundle.json` at the package root, with what it carries beside
|
|
19
|
+
* it, and a site's code as a section (stories 192, 193). Versions 1–3 kept the
|
|
20
|
+
* payload in `content/`, and are refused rather than read (no older layout is
|
|
21
|
+
* kept up), with a message saying to rebuild them.
|
|
28
22
|
*/
|
|
29
|
-
export const
|
|
23
|
+
export const BUNDLE_FORMAT_VERSION = 4
|
|
24
|
+
|
|
25
|
+
/** The oldest version this reader reads: the first with the payload at the package root. */
|
|
26
|
+
export const OLDEST_BUNDLE_FORMAT_VERSION = 4
|
|
30
27
|
|
|
31
28
|
/**
|
|
32
29
|
* The sections a bundle can carry beyond its content, and the directory each
|
|
@@ -48,6 +45,12 @@ export const BUNDLE_SECTIONS = {
|
|
|
48
45
|
styles: 'styles',
|
|
49
46
|
scripts: 'scripts',
|
|
50
47
|
dictionary: 'dictionary',
|
|
48
|
+
/**
|
|
49
|
+
* A site's own code (story 192), at its site paths. Carried and hashed like
|
|
50
|
+
* the rest, read as bytes, and never applied by an import: installing it is
|
|
51
|
+
* `installBundleCode`'s, followed by a restart.
|
|
52
|
+
*/
|
|
53
|
+
code: 'code',
|
|
51
54
|
} as const
|
|
52
55
|
|
|
53
56
|
export type BundleSection = keyof typeof BUNDLE_SECTIONS
|
package/src/package.ts
CHANGED
|
@@ -1,17 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* A written bundle as the npm package every bundle is:
|
|
3
|
-
* `
|
|
4
|
-
* `docs/bundles.md`.
|
|
2
|
+
* A written bundle as the npm package every bundle is: `bundle.json` at the
|
|
3
|
+
* root beside `package.json`, and everything it carries beside them, under its
|
|
4
|
+
* one hash (story 193). `docs/bundles.md`.
|
|
5
5
|
*
|
|
6
|
-
* The payload is `writeBundle`'s output unchanged
|
|
7
|
-
*
|
|
8
|
-
* name, a version and a core it was written for.
|
|
6
|
+
* The payload is `writeBundle`'s output unchanged. The package is what gives
|
|
7
|
+
* it a name, a version and a core it was written for.
|
|
9
8
|
*/
|
|
10
9
|
import { PEOPLE_DIR } from './people.ts'
|
|
11
|
-
import type
|
|
12
|
-
|
|
13
|
-
/** The directory inside a package that holds the `bundle.json` payload. */
|
|
14
|
-
export const PAYLOAD_DIR = 'content'
|
|
10
|
+
import { type BundleFile, MANIFEST_FILE } from './write.ts'
|
|
15
11
|
|
|
16
12
|
/** The npm keyword a bundle publishes so the marketplace can find it. */
|
|
17
13
|
export const BUNDLE_KEYWORD = 'crombolo-bundle'
|
|
@@ -69,20 +65,30 @@ export function packageNameFor(text: string): string {
|
|
|
69
65
|
return bundleIdFor(text)
|
|
70
66
|
}
|
|
71
67
|
|
|
72
|
-
/**
|
|
73
|
-
export function
|
|
68
|
+
/** What `files` lists for a payload: the manifest and each top-level entry it carries. */
|
|
69
|
+
export function packageFilesFor(paths: Iterable<string>): string[] {
|
|
70
|
+
const top = new Set<string>([MANIFEST_FILE])
|
|
71
|
+
for (const path of paths) top.add(path.split('/')[0] as string)
|
|
72
|
+
return [...top].sort()
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** The `package.json` a bundle package is written with, for a payload at these paths. */
|
|
76
|
+
export function bundlePackageJson(
|
|
77
|
+
info: BundlePackageInfo,
|
|
78
|
+
paths: Iterable<string>,
|
|
79
|
+
): Record<string, unknown> {
|
|
74
80
|
return {
|
|
75
81
|
name: info.name,
|
|
76
82
|
version: info.version,
|
|
77
83
|
...(info.description ? { description: info.description } : {}),
|
|
78
84
|
...(info.private ? { private: true } : { keywords: [BUNDLE_KEYWORD] }),
|
|
79
|
-
files:
|
|
85
|
+
files: packageFilesFor(paths),
|
|
80
86
|
peerDependencies: { '@crombolo/crombolo': info.peerRange },
|
|
81
|
-
crombolo: { id: info.id, name: info.label, content:
|
|
87
|
+
crombolo: { id: info.id, name: info.label, content: true },
|
|
82
88
|
}
|
|
83
89
|
}
|
|
84
90
|
|
|
85
|
-
/** A payload's files,
|
|
91
|
+
/** A payload's files, with the `package.json` that makes them a package. */
|
|
86
92
|
export function writeBundlePackage(
|
|
87
93
|
payload: readonly BundleFile[],
|
|
88
94
|
info: BundlePackageInfo,
|
|
@@ -95,7 +101,17 @@ export function writeBundlePackage(
|
|
|
95
101
|
if (!info.private && payload.some((file) => file.path.startsWith('values/')))
|
|
96
102
|
throw new Error('A bundle whose content is encrypted can only be a private package.')
|
|
97
103
|
return [
|
|
98
|
-
{
|
|
99
|
-
|
|
104
|
+
{
|
|
105
|
+
path: 'package.json',
|
|
106
|
+
text: `${JSON.stringify(
|
|
107
|
+
bundlePackageJson(
|
|
108
|
+
info,
|
|
109
|
+
payload.map((file) => file.path),
|
|
110
|
+
),
|
|
111
|
+
null,
|
|
112
|
+
2,
|
|
113
|
+
)}\n`,
|
|
114
|
+
},
|
|
115
|
+
...payload,
|
|
100
116
|
]
|
|
101
117
|
}
|
package/src/write.ts
CHANGED
|
@@ -20,9 +20,7 @@ import {
|
|
|
20
20
|
type BundleSection,
|
|
21
21
|
type BundleValue,
|
|
22
22
|
type BundleVariant,
|
|
23
|
-
CONTENT_ONLY_FORMAT_VERSION,
|
|
24
23
|
type ContentSet,
|
|
25
|
-
SECTIONS_FORMAT_VERSION,
|
|
26
24
|
} from './model.ts'
|
|
27
25
|
import { type BundlePeople, sealPeople } from './people.ts'
|
|
28
26
|
import { type Seal, sealBytes } from './seal.ts'
|
|
@@ -367,21 +365,9 @@ function planBundle(
|
|
|
367
365
|
}
|
|
368
366
|
|
|
369
367
|
const people = options.people && seal ? sealPeople(options.people, seal) : undefined
|
|
370
|
-
// Rules a node carries beyond its content: an older reader would drop them,
|
|
371
|
-
// and a protected page it dropped would be public.
|
|
372
|
-
const nodeExtras = source.some(
|
|
373
|
-
(n) => n.access || (n.schedule?.length ?? 0) > 0 || (n.redirects?.length ?? 0) > 0,
|
|
374
|
-
)
|
|
375
368
|
const manifest: BundleManifest = {
|
|
376
369
|
...set.manifest,
|
|
377
|
-
|
|
378
|
-
// need 2, and anything sealed or a node's rules need 3.
|
|
379
|
-
formatVersion:
|
|
380
|
-
people || sealing || nodeExtras
|
|
381
|
-
? BUNDLE_FORMAT_VERSION
|
|
382
|
-
: extra.length > 0
|
|
383
|
-
? SECTIONS_FORMAT_VERSION
|
|
384
|
-
: CONTENT_ONLY_FORMAT_VERSION,
|
|
370
|
+
formatVersion: BUNDLE_FORMAT_VERSION,
|
|
385
371
|
// Without values nothing names a media file, so none is expected either.
|
|
386
372
|
blobs: (content === 'omitted' ? [] : set.manifest.blobs).map((blob) => {
|
|
387
373
|
const size = carriedSizes.get(blob.key)
|