@crombolo/transfer 0.31.0 → 0.33.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crombolo/transfer",
3
- "version": "0.31.0",
3
+ "version": "0.33.1",
4
4
  "description": "Content transfer for crombolo: bundles of content moved between environments.",
5
5
  "license": "MIT",
6
6
  "author": "Crombolo",
@@ -28,7 +28,7 @@
28
28
  "access": "public"
29
29
  },
30
30
  "dependencies": {
31
- "@crombolo/core": "0.31.0",
32
- "@crombolo/data": "0.31.0"
31
+ "@crombolo/core": "0.33.1",
32
+ "@crombolo/data": "0.33.1"
33
33
  }
34
34
  }
package/src/code.ts ADDED
@@ -0,0 +1,115 @@
1
+ /**
2
+ * A site's own code, carried in its bundle (stories 192, 193): the files its
3
+ * `package.json` names in `crombolo.code`, at their site paths under `code/`,
4
+ * listed in `bundle.json` and covered by its one hash like every other section.
5
+ * `docs/bundles.md`, "A site's bundle with code".
6
+ */
7
+ import { existsSync, readFileSync } from 'node:fs'
8
+ import { isAbsolute, join, sep } from 'node:path'
9
+ import { loadBundle } from './load.ts'
10
+ import { BUNDLE_SECTIONS } from './model.ts'
11
+ import type { BundleFile } from './write.ts'
12
+
13
+ /** The section, and directory, that holds the site's code. */
14
+ export const CODE_DIR = BUNDLE_SECTIONS.code
15
+
16
+ /** Packed whenever there is code: what installing and booting it needs. */
17
+ export const CODE_ALWAYS = ['package.json', 'bun.lock', 'crombolo.config.ts'] as const
18
+
19
+ /** The folders a bundle already carries as sections, or that are a site's local state. */
20
+ const SITE_FOLDERS = new Set([
21
+ 'schema',
22
+ 'components',
23
+ 'css',
24
+ 'scripts',
25
+ 'seed',
26
+ 'media',
27
+ 'logs',
28
+ '.crombolo',
29
+ ])
30
+
31
+ /** Never packed, whatever a pattern matches: installs, history, secrets and databases. */
32
+ export function isNeverCode(path: string): boolean {
33
+ const parts = path.split('/')
34
+ const name = parts.at(-1) as string
35
+ return (
36
+ SITE_FOLDERS.has(parts[0] as string) ||
37
+ parts.some((part) => part === 'node_modules' || part === '.git') ||
38
+ name === '.env' ||
39
+ name.startsWith('.env.') ||
40
+ /\.sqlite(?:$|-|\.)/.test(name)
41
+ )
42
+ }
43
+
44
+ /** `crombolo.code` in a site's `package.json`, or undefined when it names none. */
45
+ export function codePatternsOf(siteDir: string): string[] | undefined {
46
+ const file = join(siteDir, 'package.json')
47
+ if (!existsSync(file)) return undefined
48
+ const json = JSON.parse(readFileSync(file, 'utf8')) as { crombolo?: { code?: unknown } }
49
+ const code = json.crombolo?.code
50
+ return Array.isArray(code) ? code.filter((p): p is string => typeof p === 'string') : undefined
51
+ }
52
+
53
+ /**
54
+ * The site's files the patterns name, plus `CODE_ALWAYS`, in path order. A
55
+ * pattern that matches nothing is a problem rather than a silent gap, and so is
56
+ * a site without the lockfile its code would be installed from.
57
+ */
58
+ export function selectCodeFiles(
59
+ siteDir: string,
60
+ patterns: readonly string[],
61
+ ): { paths: string[]; problems: string[] } {
62
+ const problems: string[] = []
63
+ const paths = new Set<string>()
64
+ for (const pattern of patterns) {
65
+ if (isAbsolute(pattern) || pattern.split('/').includes('..')) {
66
+ problems.push(`"${pattern}" reaches outside the site`)
67
+ continue
68
+ }
69
+ let matched = 0
70
+ for (const path of new Bun.Glob(pattern).scanSync({ cwd: siteDir, dot: true })) {
71
+ const normal = path.split(sep).join('/')
72
+ if (isNeverCode(normal)) continue
73
+ paths.add(normal)
74
+ matched++
75
+ }
76
+ if (matched === 0) problems.push(`"${pattern}" matches no file the bundle can carry`)
77
+ }
78
+ for (const always of CODE_ALWAYS) if (existsSync(join(siteDir, always))) paths.add(always)
79
+ if (!existsSync(join(siteDir, 'package.json'))) problems.push('the site has no package.json')
80
+ if (!existsSync(join(siteDir, 'bun.lock')))
81
+ problems.push('the site has no bun.lock, so its code could not be installed as it was tested')
82
+ return { paths: [...paths].sort(), problems }
83
+ }
84
+
85
+ /** The files at those paths, as bytes and as the bundle carries them, under `code/`. */
86
+ export function readCodeFiles(siteDir: string, paths: readonly string[]): BundleFile[] {
87
+ return paths.map((path) => ({
88
+ path: `${CODE_DIR}/${path}`,
89
+ bytes: new Uint8Array(readFileSync(join(siteDir, path))),
90
+ }))
91
+ }
92
+
93
+ export interface LoadedBundleCode {
94
+ /** The absolute directory holding the code, when the bundle carries some. */
95
+ dir?: string
96
+ /** Every file under it, relative to it and `/`-separated. */
97
+ paths: string[]
98
+ /** The bundle's own problems, its hash among them, with any about the code. */
99
+ problems: string[]
100
+ }
101
+
102
+ /** A bundle package's code, read through the bundle's own loader and its one hash. */
103
+ export function loadBundleCode(packageDir: string): LoadedBundleCode {
104
+ const loaded = loadBundle(packageDir)
105
+ const problems = loaded.problems.map((p) => `${p.file}: ${p.message}`)
106
+ const carried = loaded.set?.manifest.carries?.code
107
+ if (!loaded.set) return { paths: [], problems }
108
+ if (!carried) return { paths: [], problems: [...problems, 'the bundle carries no code'] }
109
+ const prefix = `${CODE_DIR}/`
110
+ return {
111
+ dir: join(packageDir, CODE_DIR),
112
+ paths: carried.map((path) => path.slice(prefix.length)),
113
+ problems,
114
+ }
115
+ }
package/src/index.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  export * from './check.ts'
2
+ export * from './code.ts'
2
3
  export * from './export.ts'
3
4
  export * from './import.ts'
4
5
  export * from './load.ts'
package/src/load.ts CHANGED
@@ -23,6 +23,7 @@ import {
23
23
  type BundleValue,
24
24
  type BundleVariant,
25
25
  type ContentSet,
26
+ OLDEST_BUNDLE_FORMAT_VERSION,
26
27
  } from './model.ts'
27
28
  import { type BundlePeople, openPeople, type PeopleBlock, parsePeopleBlock } from './people.ts'
28
29
  import { openBytes, parseSealParameters, unlock } from './seal.ts'
@@ -237,6 +238,10 @@ function parseManifest(text: string, problems: BundleProblem[]): BundleManifest
237
238
  })
238
239
  return undefined
239
240
  }
241
+ if (formatVersion < OLDEST_BUNDLE_FORMAT_VERSION) {
242
+ problems.push({ file: MANIFEST_FILE, message: OLD_LAYOUT })
243
+ return undefined
244
+ }
240
245
  const provenance = isRecord(raw.provenance) ? raw.provenance : {}
241
246
  const selector = isRecord(raw.selector) ? raw.selector : {}
242
247
  const dependencies = isRecord(raw.dependencies) ? raw.dependencies : {}
@@ -348,6 +353,10 @@ function parseCarries(raw: Record<string, unknown>): Partial<Record<BundleSectio
348
353
  return out
349
354
  }
350
355
 
356
+ /** Why a bundle in the layout before format 4 is refused, and what to do about it. */
357
+ export const OLD_LAYOUT =
358
+ 'this bundle was written before crombolo 0.33.1, in a layout no longer read; rebuild it with the command that made it (bundle build, content export, export, or a created package download)'
359
+
351
360
  /** Reads a bundle directory. `set` is undefined when it cannot be trusted at all. */
352
361
  export function loadBundle(dir: string): LoadedBundle {
353
362
  const problems: BundleProblem[] = []
@@ -404,7 +413,12 @@ export function loadBundle(dir: string): LoadedBundle {
404
413
  continue
405
414
  }
406
415
  files.set(path, file)
407
- read.push({ path, text: readFileSync(file, 'utf8') })
416
+ // Code is a site's source, which is not only text.
417
+ read.push(
418
+ path.startsWith(`${BUNDLE_SECTIONS.code}/`)
419
+ ? { path, bytes: new Uint8Array(readFileSync(file)) }
420
+ : { path, text: readFileSync(file, 'utf8') },
421
+ )
408
422
  }
409
423
  }
410
424
 
package/src/model.ts CHANGED
@@ -11,22 +11,19 @@
11
11
  import type { BundlePeople, PeopleBlock } from './people.ts'
12
12
  import type { SealParameters } from './seal.ts'
13
13
 
14
- /** Bumped when a change to the shapes below stops an older reader understanding a bundle. */
15
- export const BUNDLE_FORMAT_VERSION = 3
16
-
17
- /** What a bundle with sections and nothing sealed writes: version 2 readers still read it. */
18
- export const SECTIONS_FORMAT_VERSION = 2
19
-
20
14
  /**
21
- * The version a bundle carrying only content is still written as.
15
+ * Bumped when a change to the shapes below stops an older reader understanding
16
+ * a bundle. Every bundle is written at this version.
22
17
  *
23
- * A content-only bundle is byte-for-byte what version 1 was, so it keeps saying
24
- * 1 and an older node can still read it. Only a bundle that carries a section
25
- * beyond its content needs a reader that knows about `carries`, and that one
26
- * says 2 — which an older node refuses with a message naming the upgrade,
27
- * rather than failing an integrity check it cannot explain.
18
+ * 4 (0.33.1): `bundle.json` at the package root, with what it carries beside
19
+ * it, and a site's code as a section (stories 192, 193). Versions 1–3 kept the
20
+ * payload in `content/`, and are refused rather than read (no older layout is
21
+ * kept up), with a message saying to rebuild them.
28
22
  */
29
- export const CONTENT_ONLY_FORMAT_VERSION = 1
23
+ export const BUNDLE_FORMAT_VERSION = 4
24
+
25
+ /** The oldest version this reader reads: the first with the payload at the package root. */
26
+ export const OLDEST_BUNDLE_FORMAT_VERSION = 4
30
27
 
31
28
  /**
32
29
  * The sections a bundle can carry beyond its content, and the directory each
@@ -48,6 +45,12 @@ export const BUNDLE_SECTIONS = {
48
45
  styles: 'styles',
49
46
  scripts: 'scripts',
50
47
  dictionary: 'dictionary',
48
+ /**
49
+ * A site's own code (story 192), at its site paths. Carried and hashed like
50
+ * the rest, read as bytes, and never applied by an import: installing it is
51
+ * `installBundleCode`'s, followed by a restart.
52
+ */
53
+ code: 'code',
51
54
  } as const
52
55
 
53
56
  export type BundleSection = keyof typeof BUNDLE_SECTIONS
package/src/package.ts CHANGED
@@ -1,17 +1,13 @@
1
1
  /**
2
- * A written bundle as the npm package every bundle is: a `package.json` whose
3
- * `crombolo` field declares the payload, and the payload under `content/`.
4
- * `docs/bundles.md`.
2
+ * A written bundle as the npm package every bundle is: `bundle.json` at the
3
+ * root beside `package.json`, and everything it carries beside them, under its
4
+ * one hash (story 193). `docs/bundles.md`.
5
5
  *
6
- * The payload is `writeBundle`'s output unchanged, so its manifest, integrity
7
- * hash and sections mean what they always did. The package is what gives it a
8
- * name, a version and a core it was written for.
6
+ * The payload is `writeBundle`'s output unchanged. The package is what gives
7
+ * it a name, a version and a core it was written for.
9
8
  */
10
9
  import { PEOPLE_DIR } from './people.ts'
11
- import type { BundleFile } from './write.ts'
12
-
13
- /** The directory inside a package that holds the `bundle.json` payload. */
14
- export const PAYLOAD_DIR = 'content'
10
+ import { type BundleFile, MANIFEST_FILE } from './write.ts'
15
11
 
16
12
  /** The npm keyword a bundle publishes so the marketplace can find it. */
17
13
  export const BUNDLE_KEYWORD = 'crombolo-bundle'
@@ -69,20 +65,30 @@ export function packageNameFor(text: string): string {
69
65
  return bundleIdFor(text)
70
66
  }
71
67
 
72
- /** The `package.json` a bundle package is written with. */
73
- export function bundlePackageJson(info: BundlePackageInfo): Record<string, unknown> {
68
+ /** What `files` lists for a payload: the manifest and each top-level entry it carries. */
69
+ export function packageFilesFor(paths: Iterable<string>): string[] {
70
+ const top = new Set<string>([MANIFEST_FILE])
71
+ for (const path of paths) top.add(path.split('/')[0] as string)
72
+ return [...top].sort()
73
+ }
74
+
75
+ /** The `package.json` a bundle package is written with, for a payload at these paths. */
76
+ export function bundlePackageJson(
77
+ info: BundlePackageInfo,
78
+ paths: Iterable<string>,
79
+ ): Record<string, unknown> {
74
80
  return {
75
81
  name: info.name,
76
82
  version: info.version,
77
83
  ...(info.description ? { description: info.description } : {}),
78
84
  ...(info.private ? { private: true } : { keywords: [BUNDLE_KEYWORD] }),
79
- files: [PAYLOAD_DIR],
85
+ files: packageFilesFor(paths),
80
86
  peerDependencies: { '@crombolo/crombolo': info.peerRange },
81
- crombolo: { id: info.id, name: info.label, content: PAYLOAD_DIR },
87
+ crombolo: { id: info.id, name: info.label, content: true },
82
88
  }
83
89
  }
84
90
 
85
- /** A payload's files, placed into a package under `content/` beside its `package.json`. */
91
+ /** A payload's files, with the `package.json` that makes them a package. */
86
92
  export function writeBundlePackage(
87
93
  payload: readonly BundleFile[],
88
94
  info: BundlePackageInfo,
@@ -95,7 +101,17 @@ export function writeBundlePackage(
95
101
  if (!info.private && payload.some((file) => file.path.startsWith('values/')))
96
102
  throw new Error('A bundle whose content is encrypted can only be a private package.')
97
103
  return [
98
- { path: 'package.json', text: `${JSON.stringify(bundlePackageJson(info), null, 2)}\n` },
99
- ...payload.map((file) => ({ ...file, path: `${PAYLOAD_DIR}/${file.path}` })),
104
+ {
105
+ path: 'package.json',
106
+ text: `${JSON.stringify(
107
+ bundlePackageJson(
108
+ info,
109
+ payload.map((file) => file.path),
110
+ ),
111
+ null,
112
+ 2,
113
+ )}\n`,
114
+ },
115
+ ...payload,
100
116
  ]
101
117
  }
package/src/write.ts CHANGED
@@ -20,9 +20,7 @@ import {
20
20
  type BundleSection,
21
21
  type BundleValue,
22
22
  type BundleVariant,
23
- CONTENT_ONLY_FORMAT_VERSION,
24
23
  type ContentSet,
25
- SECTIONS_FORMAT_VERSION,
26
24
  } from './model.ts'
27
25
  import { type BundlePeople, sealPeople } from './people.ts'
28
26
  import { type Seal, sealBytes } from './seal.ts'
@@ -367,21 +365,9 @@ function planBundle(
367
365
  }
368
366
 
369
367
  const people = options.people && seal ? sealPeople(options.people, seal) : undefined
370
- // Rules a node carries beyond its content: an older reader would drop them,
371
- // and a protected page it dropped would be public.
372
- const nodeExtras = source.some(
373
- (n) => n.access || (n.schedule?.length ?? 0) > 0 || (n.redirects?.length ?? 0) > 0,
374
- )
375
368
  const manifest: BundleManifest = {
376
369
  ...set.manifest,
377
- // The oldest version that can read it: content only stays 1, sections
378
- // need 2, and anything sealed or a node's rules need 3.
379
- formatVersion:
380
- people || sealing || nodeExtras
381
- ? BUNDLE_FORMAT_VERSION
382
- : extra.length > 0
383
- ? SECTIONS_FORMAT_VERSION
384
- : CONTENT_ONLY_FORMAT_VERSION,
370
+ formatVersion: BUNDLE_FORMAT_VERSION,
385
371
  // Without values nothing names a media file, so none is expected either.
386
372
  blobs: (content === 'omitted' ? [] : set.manifest.blobs).map((blob) => {
387
373
  const size = carriedSizes.get(blob.key)