@exodus/stasis-core 1.0.0-beta.15 → 1.0.0-beta.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -1
- package/src/add.js +109 -194
- package/src/artifact-util.js +234 -291
- package/src/brotli.js +3 -9
- package/src/bundle-util.js +66 -162
- package/src/bundle.js +55 -129
- package/src/cli.js +15 -45
- package/src/config.js +58 -102
- package/src/extract.js +40 -74
- package/src/fs.js +54 -89
- package/src/hooks.js +118 -235
- package/src/host.js +1 -4
- package/src/loader.js +1 -1
- package/src/lockfile.js +34 -78
- package/src/policy.js +222 -0
- package/src/prune.js +60 -114
- package/src/shard.js +10 -25
- package/src/state-util.js +11 -40
- package/src/state.js +498 -830
- package/src/util.js +97 -149
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@exodus/stasis-core",
|
|
3
|
-
"version": "1.0.0-beta.
|
|
3
|
+
"version": "1.0.0-beta.16",
|
|
4
4
|
"description": "Zero-dependency stasis runtime",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
"./util": "./src/util.js",
|
|
18
18
|
"./brotli": "./src/brotli.js",
|
|
19
19
|
"./state-util": "./src/state-util.js",
|
|
20
|
+
"./policy": "./src/policy.js",
|
|
20
21
|
"./prune": "./src/prune.js",
|
|
21
22
|
"./cli": "./src/cli.js"
|
|
22
23
|
},
|
|
@@ -36,6 +37,7 @@
|
|
|
36
37
|
"src/loader.js",
|
|
37
38
|
"src/lockfile.js",
|
|
38
39
|
"src/package.cjs",
|
|
40
|
+
"src/policy.js",
|
|
39
41
|
"src/prune.js",
|
|
40
42
|
"src/shard.js",
|
|
41
43
|
"src/state.js",
|
package/src/add.js
CHANGED
|
@@ -1,92 +1,63 @@
|
|
|
1
1
|
import { isUtf8 } from 'node:buffer'
|
|
2
2
|
import * as fs from 'node:fs'
|
|
3
|
-
import {
|
|
3
|
+
import { dirname, extname, join, relative, resolve, sep } from 'node:path'
|
|
4
4
|
|
|
5
5
|
import { Bundle } from './bundle.js'
|
|
6
6
|
import { Lockfile } from './lockfile.js'
|
|
7
7
|
import { compressBundleText, parseBundleBytes } from './brotli.js'
|
|
8
8
|
import { detectRepo, normalizeEntries, packageLookup, packageType, readJson, statStrict } from './bundle-util.js'
|
|
9
9
|
import { diskHost } from './host.js'
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
10
|
+
import { linksOn, parsePolicy, recordedIn } from './policy.js'
|
|
11
|
+
import { canonicalizePath, sha512integrity } from './state-util.js'
|
|
12
|
+
import { assertRealPathWithinBase, attestsAny, BUNDLE_FILE, classifyFormat, CONFIG_FILE, extraPattern, fileInBucket, hasNodeModulesSegment, heldToAccounts, inResources, installedAccounts, isAutoExcludedDir, isAutoExcludedFile, isBinaryPlist, isBrotliQuality, isExecutableMode, LOCK_FILE, memoized, moduleInfo, parseResourcesOption, sortedByPath, splitNodeModulesPath, toPosix, withAttestedOf, withMetadataOf } from './util.js'
|
|
12
13
|
|
|
13
|
-
// Snapshotted before `stasis run --fs` patches node:fs:
|
|
14
|
-
// these at write (outputMatcher), which must read the real disk, not be captured as the program's
|
|
15
|
-
// reads. It sweeps `extra` through its host, which the disk's snapshots too.
|
|
14
|
+
// Snapshotted before `stasis run --fs` patches node:fs: outputMatcher must read the real disk, not be captured.
|
|
16
15
|
const { existsSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } = fs
|
|
17
16
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
// A `.js`/`.ts` follows its package `type` (LIMITATION: none falls back to commonjs -- use `.mjs`/`.cjs`
|
|
22
|
-
// or set `type` when the module system matters). Any other name is classifyFormat's call: a format,
|
|
23
|
-
// null for a JS-family file (`.jsx`/`.tsx`: source attested with no loader format, as the runtime
|
|
24
|
-
// records one), undefined when unrecognized. Its package.json is read through `host`.
|
|
25
|
-
function sourceFormat(absFile, content, host) {
|
|
17
|
+
// LIMITATION: a `.js`/`.ts` in a package with no `type` is commonjs; use `.mjs`/`.cjs` or set it.
|
|
18
|
+
function sourceFormat(absFile, content, host, kept) {
|
|
26
19
|
const ext = extname(absFile).toLowerCase()
|
|
27
|
-
if (ext === '.js' || ext === '.ts') return `${packageType(absFile, host) ?? 'commonjs'}${ext === '.ts' ? '-typescript' : ''}`
|
|
20
|
+
if (ext === '.js' || ext === '.ts') return `${packageType(absFile, host, kept) ?? 'commonjs'}${ext === '.ts' ? '-typescript' : ''}`
|
|
28
21
|
return classifyFormat(absFile, { content })
|
|
29
22
|
}
|
|
30
23
|
|
|
31
|
-
|
|
32
|
-
const tally = (artifact) => {
|
|
33
|
-
let files = 0
|
|
34
|
-
let packages = 0
|
|
35
|
-
for (const m of artifact.modules.values()) {
|
|
36
|
-
const n = Object.keys(m.files).length
|
|
37
|
-
if (n > 0) {
|
|
38
|
-
files += n
|
|
39
|
-
packages += 1
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
return { files, packages }
|
|
43
|
-
}
|
|
24
|
+
const fileCount = (artifact) => artifact.modules.values().reduce((n, { files }) => n + Object.keys(files).length, 0)
|
|
44
25
|
|
|
45
26
|
function readAddConfig(baseDir) {
|
|
46
27
|
const cfg = readJson(join(baseDir, CONFIG_FILE))
|
|
47
|
-
if (cfg === null) {
|
|
48
|
-
throw new Error(`add requires a ${CONFIG_FILE} (its bundleFile, resourcesBundleFile, and resources are all optional)`)
|
|
49
|
-
}
|
|
28
|
+
if (cfg === null) throw new Error(`add requires a ${CONFIG_FILE} (its bundleFile, resourcesBundleFile, and resources are all optional)`)
|
|
50
29
|
const resources = parseResourcesOption(CONFIG_FILE, cfg.resources)
|
|
51
30
|
if (cfg.brotliQuality !== undefined && !isBrotliQuality(cfg.brotliQuality)) {
|
|
52
31
|
throw new Error(`${CONFIG_FILE}: brotliQuality must be an integer 0..11 (got ${JSON.stringify(cfg.brotliQuality)})`)
|
|
53
32
|
}
|
|
54
|
-
const bundleFile = cfg.bundleFile ??
|
|
33
|
+
const bundleFile = cfg.bundleFile ?? BUNDLE_FILE
|
|
55
34
|
const resourcesBundleFile = cfg.resourcesBundleFile
|
|
56
|
-
//
|
|
35
|
+
// Distinct, else they clobber on write; canonicalized so symlink and relative aliases are caught too.
|
|
57
36
|
if (resourcesBundleFile &&
|
|
58
37
|
canonicalizePath(resolve(baseDir, bundleFile)) === canonicalizePath(resolve(baseDir, resourcesBundleFile))) {
|
|
59
38
|
throw new Error(`${CONFIG_FILE}: bundleFile and resourcesBundleFile must name distinct paths (both resolve to ${resolve(baseDir, bundleFile)})`)
|
|
60
39
|
}
|
|
61
|
-
return { bundleFile, resourcesBundleFile, resources, brotliQuality: cfg.brotliQuality }
|
|
40
|
+
return { bundleFile, resourcesBundleFile, resources, brotliQuality: cfg.brotliQuality, policy: parsePolicy(CONFIG_FILE, cfg) }
|
|
62
41
|
}
|
|
63
42
|
|
|
64
|
-
//
|
|
65
|
-
|
|
66
|
-
// bundle the files will merge into, which may name its buckets otherwise (`solidity-bundle`, say).
|
|
67
|
-
// Their package.json files are read through `host`.
|
|
68
|
-
function assembleBundle(baseDir, files, workspaceName, workspaceVersion, repo, { reason = 'add', buckets = new Map(), host = diskHost } = {}) {
|
|
43
|
+
// With NO entries: `add` attests files, not entry points, so nothing in it runs via `--bundle=load`.
|
|
44
|
+
export function assembleBundle(baseDir, files, workspaceName, workspaceVersion, repo, { reason = 'add', buckets = new Map(), host = diskHost } = {}) {
|
|
69
45
|
const modules = new Map()
|
|
70
46
|
const formats = new Map()
|
|
71
47
|
const executable = new Set()
|
|
72
|
-
// `identity`'s name, version, ecosystem, repo and vendored (moduleInfo takes those alone); a later
|
|
73
|
-
// file's adds the vendored package it is in.
|
|
74
|
-
const bucketFiles = (dir, identity) => {
|
|
75
|
-
const bucket = modules.get(dir)
|
|
76
|
-
identity = buckets.get(dir) ?? identity
|
|
77
|
-
modules.set(dir, bucket === undefined ? moduleInfo({ ...identity, files: Object.create(null) }) : withMetadataOf(bucket, identity))
|
|
78
|
-
return modules.get(dir).files
|
|
79
|
-
}
|
|
80
48
|
const metaFor = packageLookup(baseDir, { host })
|
|
81
49
|
|
|
82
50
|
for (const [rel, { content, format, executable: isExec }] of files) {
|
|
83
51
|
const meta = metaFor(rel)
|
|
84
|
-
// A node_modules file whose nearest package.json is the workspace root (or none at all) is a misconfigured dep -- refuse.
|
|
85
52
|
if (splitNodeModulesPath(rel) !== null && !(meta && hasNodeModulesSegment(meta.pkgDir))) {
|
|
86
53
|
throw new Error(`add: no package.json with name+version found for ${rel}`)
|
|
87
54
|
}
|
|
88
|
-
const
|
|
89
|
-
|
|
55
|
+
const dir = meta ? meta.pkgDir : '.'
|
|
56
|
+
const identity = buckets.get(dir) ?? meta ?? { name: workspaceName, version: workspaceVersion }
|
|
57
|
+
const bucket = modules.get(dir)
|
|
58
|
+
const info = bucket === undefined ? moduleInfo({ ...identity, files: Object.create(null) }) : withMetadataOf(bucket, identity)
|
|
59
|
+
modules.set(dir, info)
|
|
60
|
+
info.files[fileInBucket(dir, rel)] = content
|
|
90
61
|
if (format != null) formats.set(rel, format)
|
|
91
62
|
if (isExec) executable.add(rel)
|
|
92
63
|
}
|
|
@@ -94,56 +65,50 @@ function assembleBundle(baseDir, files, workspaceName, workspaceVersion, repo, {
|
|
|
94
65
|
return new Bundle({ config: { scope: 'full' }, entries: new Set(), modules, formats, imports: new Map(), executable, repo }).withReason(reason)
|
|
95
66
|
}
|
|
96
67
|
|
|
97
|
-
//
|
|
98
|
-
//
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
let existing
|
|
106
|
-
try {
|
|
107
|
-
existing = parseBundleBytes(readFileSync(abs))
|
|
108
|
-
} catch (cause) {
|
|
109
|
-
throw new Error(`add: failed to read the existing bundle at ${targetPath}`, { cause })
|
|
110
|
-
}
|
|
111
|
-
mergedFrom = tally(existing).files
|
|
112
|
-
bundle = existing.merge(bundle)
|
|
68
|
+
// Each artifact's attested packages are held to the others' and to the install. The attest(existing, added, companion)
|
|
69
|
+
// returned has `added` attest as `existing`, else its split half `companion`, the others, the install.
|
|
70
|
+
function heldToInstall(baseDir, artifacts) {
|
|
71
|
+
const known = new Map()
|
|
72
|
+
for (const [path, artifact] of artifacts) {
|
|
73
|
+
const attested = new Map([...artifact.modules].filter(([, info]) => attestsAny(info)))
|
|
74
|
+
for (const [dir, info] of attested) known.set(dir, known.has(dir) ? withMetadataOf(known.get(dir), info, `${path}: module '${dir}' (against the other artifacts)`) : info)
|
|
75
|
+
heldToAccounts(attested, installedAccounts(attested, baseDir), `${path}: `)
|
|
113
76
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
},
|
|
120
|
-
counts: { path: targetPath, total: files, packages, added: mergedFrom === undefined ? files : files - mergedFrom },
|
|
77
|
+
return (existing, added, companion) => {
|
|
78
|
+
for (const [dir, account] of installedAccounts(added.modules, baseDir)) {
|
|
79
|
+
if (!account.locked) added.modules.set(dir, withAttestedOf(added.modules.get(dir), existing?.modules.get(dir) ?? companion?.modules.get(dir) ?? known.get(dir) ?? account))
|
|
80
|
+
}
|
|
81
|
+
return added
|
|
121
82
|
}
|
|
122
83
|
}
|
|
123
84
|
|
|
124
|
-
|
|
125
|
-
const bundleToLockfile = (bundle, integrities) => Lockfile.fromBundle(bundle, (file) => integrities.get(file))
|
|
126
|
-
|
|
127
|
-
// Merge into the project's stasis.lock.json (strict: divergent bytes throw), deferring the write like
|
|
128
|
-
// prepareBundleFile so a conflict here doesn't leave the bundles updated.
|
|
129
|
-
function prepareLockfile(lockPath, lockAdd) {
|
|
130
|
-
let existing
|
|
85
|
+
function existingOf(what, parse) {
|
|
131
86
|
try {
|
|
132
|
-
|
|
87
|
+
return parse()
|
|
133
88
|
} catch (cause) {
|
|
134
|
-
throw new Error(`add: failed to read the existing ${
|
|
89
|
+
throw new Error(`add: failed to read the existing ${what}`, { cause })
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// A thunk, so every target's conflicts surface before a byte lands; serializing in it keeps peak memory at one target's.
|
|
94
|
+
function prepareBundleFile(baseDir, targetPath, bundle, brotliQuality, check, existing) {
|
|
95
|
+
check?.(bundle)
|
|
96
|
+
const abs = resolve(baseDir, targetPath)
|
|
97
|
+
let before = 0
|
|
98
|
+
if (existing) {
|
|
99
|
+
check?.(existing, recordedIn(targetPath))
|
|
100
|
+
before = fileCount(existing)
|
|
101
|
+
bundle = existing.merge(bundle)
|
|
135
102
|
}
|
|
136
|
-
const
|
|
137
|
-
const
|
|
138
|
-
|
|
139
|
-
|
|
103
|
+
const total = fileCount(bundle)
|
|
104
|
+
const write = () => {
|
|
105
|
+
mkdirSync(dirname(abs), { recursive: true })
|
|
106
|
+
writeFileSync(abs, compressBundleText(bundle.serialize(), brotliQuality))
|
|
107
|
+
}
|
|
108
|
+
return { write, total, added: total - before }
|
|
140
109
|
}
|
|
141
110
|
|
|
142
|
-
//
|
|
143
|
-
// (project-relative): by name, or as the same file under another, a symlink or hard link to it -- its
|
|
144
|
-
// device and inode, which a write rewrites in place. Stats as numbers can round a large inode
|
|
145
|
-
// (Windows file ids), so a match on those is settled exactly (bigint), never taken on its own. The
|
|
146
|
-
// outputs are the disk's, where a run writes them: a sweep of another host's tree has none.
|
|
111
|
+
// By name, or dev+inode (a link to one, which a write rewrites in place), confirmed in bigint: a number rounds big inodes.
|
|
147
112
|
function outputMatcher(baseDir, outputs) {
|
|
148
113
|
const exact = new Set()
|
|
149
114
|
const rounded = new Set()
|
|
@@ -161,10 +126,7 @@ function outputMatcher(baseDir, outputs) {
|
|
|
161
126
|
}
|
|
162
127
|
}
|
|
163
128
|
|
|
164
|
-
//
|
|
165
|
-
// relative to it, walking into none `exclude` takes (given the path relative to `baseDir`,
|
|
166
|
-
// '/'-joined), which leaves one out where its `**` meets it alone. A visited name is checked as
|
|
167
|
-
// spelled: one holding `\` is for the caller to refuse where it is kept, not here.
|
|
129
|
+
// Node leaves out a path `exclude` takes only where its `**` meets it, so it is asked of each '/'-joined project path.
|
|
168
130
|
function globPaths(host, pattern, cwd, baseDir, exclude) {
|
|
169
131
|
const projectPath = (path) => {
|
|
170
132
|
const rel = relative(baseDir, path)
|
|
@@ -173,20 +135,10 @@ function globPaths(host, pattern, cwd, baseDir, exclude) {
|
|
|
173
135
|
return host.glob(pattern, { cwd, exclude: exclude && ((path) => exclude(projectPath(path))) })
|
|
174
136
|
}
|
|
175
137
|
|
|
176
|
-
//
|
|
177
|
-
//
|
|
178
|
-
// src/examples` sweeps the dir it was pointed at, `add src` skips one it merely found), file rules
|
|
179
|
-
// by name plus the caller's `outputs` (this run's own write targets, by name or alias: outputMatcher).
|
|
180
|
-
// A path the caller NAMED is always kept (naming is the override; a missing one keeps stats undefined
|
|
181
|
-
// so validation reports it) -- unless `keepNamedOutputs` is false, then one of `outputs` never is --
|
|
182
|
-
// and inclusion is monotone: a nearer root's sweep can only ever see FEWER excluded segments,
|
|
183
|
-
// so a path any entry keeps stays kept -- `add src src/examples` includes the subtree it names in
|
|
184
|
-
// either order. `stats` is the walk's own stat, reused by validation. Gotcha: `**/*` skips dotfiles
|
|
185
|
-
// (.env, .git/...) -- name them explicitly. A project-relative path `exclude` takes is dropped, named
|
|
186
|
-
// or swept, with everything under it: a named path below one too, and a directory it takes is never
|
|
187
|
-
// walked (globPaths). Paths are read through `host`, stat'ed as real stat answers (statStrict).
|
|
138
|
+
// A sweep drops what it FINDS auto-excluded: dir rules below the named root (`add src/examples` sweeps it, `add src`
|
|
139
|
+
// skips it), file rules by name and to `outputs`. A path `exclude` takes goes, named or swept, with all under it.
|
|
188
140
|
function expandDirectories(baseDir, rels, outputs, { keepNamedOutputs = true, exclude, host = diskHost } = {}) {
|
|
189
|
-
const files = new Map() // rel -> stats
|
|
141
|
+
const files = new Map() // rel -> stats, the walk's own, reused by validation
|
|
190
142
|
const excluded = new Set()
|
|
191
143
|
const isOutput = outputMatcher(baseDir, outputs)
|
|
192
144
|
for (const rel of rels) {
|
|
@@ -194,11 +146,13 @@ function expandDirectories(baseDir, rels, outputs, { keepNamedOutputs = true, ex
|
|
|
194
146
|
if (exclude !== undefined && rel.split('/').some((_, i, segments) => exclude(segments.slice(0, i + 1).join('/')))) continue
|
|
195
147
|
const dirAbs = join(baseDir, rel)
|
|
196
148
|
const stats = statStrict(host, dirAbs, rel) ?? undefined
|
|
149
|
+
// A NAMED path is kept (naming is the override; a missing one keeps undefined stats for validation to report).
|
|
197
150
|
if (stats === undefined || !stats.isDirectory()) {
|
|
198
151
|
if (keepNamedOutputs || !isOutput(rel, stats)) files.set(rel, stats)
|
|
199
152
|
continue
|
|
200
153
|
}
|
|
201
154
|
const found = []
|
|
155
|
+
// Gotcha: `**/*` skips dotfiles (.env, .git/...); they must be named explicitly.
|
|
202
156
|
for (const match of globPaths(host, '**/*', dirAbs, baseDir, exclude)) {
|
|
203
157
|
const fileAbs = join(dirAbs, match)
|
|
204
158
|
const fileRel = relative(baseDir, fileAbs)
|
|
@@ -206,26 +160,23 @@ function expandDirectories(baseDir, rels, outputs, { keepNamedOutputs = true, ex
|
|
|
206
160
|
const fileStats = statStrict(host, fileAbs, fileRel) ?? undefined
|
|
207
161
|
if (fileStats?.isFile()) found.push([toPosix(fileRel), match, fileStats])
|
|
208
162
|
}
|
|
209
|
-
// Sorted
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
// Node's glob holds `exclude` to its `**` alone: a file below the directory swept still
|
|
213
|
-
// matches the `*` after it (a directory it takes is never walked).
|
|
163
|
+
// Sorted, whatever the readdir order, so multi-file diagnostics keep a stable order.
|
|
164
|
+
for (const [file, match, fileStats] of sortedByPath(found, ([path]) => path)) {
|
|
165
|
+
// Node's glob holds `exclude` to its `**` alone: a file below the directory swept still matches the `*` after it.
|
|
214
166
|
if (exclude?.(file)) continue
|
|
215
167
|
if (toPosix(match).split('/').slice(0, -1).some(isAutoExcludedDir)
|
|
216
168
|
|| isOutput(file, fileStats) || isAutoExcludedFile(file)) excluded.add(file)
|
|
217
169
|
else files.set(file, fileStats)
|
|
218
170
|
}
|
|
219
171
|
}
|
|
220
|
-
//
|
|
172
|
+
// Inclusion is monotone: a path any entry kept stays kept, so `add src src/examples` keeps the subtree in either order.
|
|
221
173
|
for (const file of files.keys()) excluded.delete(file)
|
|
222
174
|
return { files, excluded }
|
|
223
175
|
}
|
|
224
176
|
|
|
225
|
-
const listPaths = (rels) =>
|
|
177
|
+
const listPaths = (rels) => sortedByPath(rels).join(', ')
|
|
226
178
|
|
|
227
|
-
// One error for every file validation rejected, so a
|
|
228
|
-
// instead of one per re-run. A lone offender keeps its exact standalone message. `label` prefixes it.
|
|
179
|
+
// One error for every file validation rejected, so a sweep names all of them at once; a lone offender keeps its own message.
|
|
229
180
|
export function validationError({ missing, undeclared, nonUtf8 }, label) {
|
|
230
181
|
const phrase = (items, one, many) => (items.length === 1 ? [one(items[0])] : items.length > 1 ? [many(items)] : [])
|
|
231
182
|
const withFormat = (format) => (format == null ? '' : ` (format '${format}')`)
|
|
@@ -244,47 +195,30 @@ export function validationError({ missing, undeclared, nonUtf8 }, label) {
|
|
|
244
195
|
return new Error(`${label}: ${parts.join('; ')}`)
|
|
245
196
|
}
|
|
246
197
|
|
|
247
|
-
//
|
|
248
|
-
//
|
|
249
|
-
// for itself as it does without bash's nullglob (so a missing path is reported missing), then each
|
|
250
|
-
// path as `add` takes it (expandDirectories): a file named or matched is kept, a directory swept.
|
|
251
|
-
// One of `outputs`, the files this very run writes, is never kept, named, matched (`dist/*`) or
|
|
252
|
-
// under another name (outputMatcher). A `!` entry (`!ios/Pods`, `!**/xcuserdata`) is no path: it
|
|
253
|
-
// takes out every path its pattern matches, in any order of the entries, along with everything under
|
|
254
|
-
// one that is a directory, which is then never walked, whether by a pattern's expansion or a sweep:
|
|
255
|
-
// a pattern whose every match is excluded matches nothing. One matching nothing is no error.
|
|
256
|
-
// `baseDir` is read through `host`, globbed and matched as Node globs and matches the disk
|
|
257
|
-
// (fs.globSync, path.matchesGlob), whichever host's it is.
|
|
198
|
+
// As a shell expands `stasis add`'s arguments: Node's glob (its `*` skips dotfiles), a pattern matching nothing kept as
|
|
199
|
+
// itself (no nullglob: a missing path is reported), then each path as `add` takes it, but one of `outputs`.
|
|
258
200
|
export function expandExtra(baseDir, patterns, outputs, host = diskHost) {
|
|
259
|
-
//
|
|
260
|
-
const exclusions = patterns.filter((pattern) => pattern.startsWith('!')).map(
|
|
261
|
-
//
|
|
262
|
-
|
|
263
|
-
const excluded = new Map()
|
|
264
|
-
const exclude = exclusions.length === 0 ? undefined : (rel) => {
|
|
265
|
-
if (!excluded.has(rel)) excluded.set(rel, exclusions.some((pattern) => host.matchesGlob(rel, pattern)))
|
|
266
|
-
return excluded.get(rel)
|
|
267
|
-
}
|
|
201
|
+
// A `!` entry takes out every match and all under it (never walked), in any entry order.
|
|
202
|
+
const exclusions = patterns.filter((pattern) => pattern.startsWith('!')).map(extraPattern)
|
|
203
|
+
// Memoized: a walk asks of each path, a sweep again, and a named path of every directory above it.
|
|
204
|
+
const exclude = exclusions.length === 0 ? undefined : memoized((rel) => exclusions.some((pattern) => host.matchesGlob(rel, pattern)))
|
|
268
205
|
const entries = patterns.filter((pattern) => !pattern.startsWith('!')).flatMap((pattern) => {
|
|
269
206
|
const matches = globPaths(host, pattern, baseDir, baseDir, exclude)
|
|
270
|
-
return matches.length === 0 ? [pattern] : matches
|
|
207
|
+
return matches.length === 0 ? [pattern] : sortedByPath(matches)
|
|
271
208
|
})
|
|
272
209
|
return expandDirectories(baseDir, normalizeEntries(entries, baseDir), outputs, { keepNamedOutputs: false, exclude, host })
|
|
273
210
|
}
|
|
274
211
|
|
|
275
|
-
//
|
|
276
|
-
|
|
277
|
-
// step's rel -> stats map (stats undefined = not on disk). Containment is the one rule NOT collected:
|
|
278
|
-
// a symlink escaping the root is a security invariant, so it fails closed on the spot. Bytes are
|
|
279
|
-
// hashed only when there's a lockfile to receive the integrities (`withIntegrity`). `label` prefixes the error.
|
|
280
|
-
// Files are read through `host`.
|
|
281
|
-
function validateFiles({ baseDir, realBase, files, resources, withIntegrity, label = 'add', host = diskHost }) {
|
|
212
|
+
// The WHOLE set is classified before any target is touched; only a symlink escaping the root fails on the spot.
|
|
213
|
+
export function validateFiles({ baseDir, realBase, files, resources, withIntegrity, label = 'add', host = diskHost }) {
|
|
282
214
|
const codeFiles = new Map()
|
|
283
215
|
const resourceFiles = new Map()
|
|
284
216
|
const integrities = new Map()
|
|
285
217
|
const missing = []
|
|
286
218
|
const undeclared = []
|
|
287
219
|
const nonUtf8 = []
|
|
220
|
+
const types = new Map()
|
|
221
|
+
const kept = (path, read) => (types.has(path) ? types.get(path) : types.set(path, read()).get(path))
|
|
288
222
|
for (const [rel, stats] of files) {
|
|
289
223
|
if (stats === undefined) {
|
|
290
224
|
missing.push(rel)
|
|
@@ -295,13 +229,13 @@ function validateFiles({ baseDir, realBase, files, resources, withIntegrity, lab
|
|
|
295
229
|
const abs = join(baseDir, rel)
|
|
296
230
|
const buf = host.readFile(abs)
|
|
297
231
|
if (withIntegrity) integrities.set(rel, sha512integrity(buf))
|
|
298
|
-
const format = sourceFormat(abs, buf, host)
|
|
299
|
-
// A binary plist can't be stored as the UTF-8 string its 'xml' format implies
|
|
232
|
+
const format = sourceFormat(abs, buf, host, kept)
|
|
233
|
+
// A binary plist can't be stored as the UTF-8 string its 'xml' format implies: a resource (opaque base64).
|
|
300
234
|
if (format !== undefined && !isBinaryPlist(rel, buf)) {
|
|
301
235
|
// Source is stored as a UTF-8 string, so non-UTF-8 bytes would lossily diverge from the file on disk.
|
|
302
236
|
if (isUtf8(buf)) codeFiles.set(rel, { content: buf.toString('utf8'), format, executable })
|
|
303
237
|
else nonUtf8.push({ rel, format })
|
|
304
|
-
} else if (
|
|
238
|
+
} else if (inResources(rel, resources)) {
|
|
305
239
|
const utf8 = isUtf8(buf)
|
|
306
240
|
resourceFiles.set(rel, { content: utf8 ? buf.toString('utf8') : buf.toString('base64'), format: utf8 ? 'resource' : 'resource:base64', executable })
|
|
307
241
|
} else {
|
|
@@ -312,57 +246,27 @@ function validateFiles({ baseDir, realBase, files, resources, withIntegrity, lab
|
|
|
312
246
|
return { codeFiles, resourceFiles, integrities }
|
|
313
247
|
}
|
|
314
248
|
|
|
315
|
-
// The files stasis.config.json's `extra` names under `baseDir` (expandExtra), packed as `add` packs
|
|
316
|
-
// them, for `stasis bundle` to merge into `bundle`, one built there without a State (always full
|
|
317
|
-
// scope): those it doesn't carry already (its own record of a file stands), in its buckets where it
|
|
318
|
-
// has them. `outputs` are the project-relative paths the build writes, never swept in. The files
|
|
319
|
-
// are read through `host`, the disk by default.
|
|
320
|
-
// -> { bundle, lockfile } of the `extra` consumer (null when there is nothing to add), the lockfile
|
|
321
|
-
// built when first read, and `carried`: the files named that `bundle` carries already, the `extra`
|
|
322
|
-
// consumer's too
|
|
323
|
-
export function extraArtifacts({ baseDir, patterns, resources, bundle, outputs = new Set(), host = diskHost }) {
|
|
324
|
-
const named = expandExtra(baseDir, patterns, outputs, host).files
|
|
325
|
-
// Bundle#sources builds its Map afresh on each read: once.
|
|
326
|
-
const bundled = bundle.sources
|
|
327
|
-
const carried = [...named.keys()].filter((rel) => bundled.has(rel))
|
|
328
|
-
const files = new Map([...named].filter(([rel]) => !bundled.has(rel)))
|
|
329
|
-
if (files.size === 0) return { bundle: null, lockfile: null, carried }
|
|
330
|
-
const { codeFiles, resourceFiles } = validateFiles({ baseDir, realBase: host.realpath(baseDir), files, resources, label: 'extra', host })
|
|
331
|
-
const rootPkg = readJson(join(baseDir, 'package.json'), host) ?? {}
|
|
332
|
-
const extra = assembleBundle(baseDir, new Map([...codeFiles, ...resourceFiles]), rootPkg.name ?? 'workspace', rootPkg.version ?? '0.0.0', undefined, { reason: 'extra', buckets: bundle.modules, host })
|
|
333
|
-
// Hashed from the bundle, which holds the bytes read, once a lockfile is wanted.
|
|
334
|
-
const lockfile = deferredLockfile(extra)
|
|
335
|
-
return { bundle: extra, get lockfile() { return lockfile() }, carried }
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
// GLOB the listed paths, FILTERING the auto-excluded set out of what a sweep finds (see
|
|
339
|
-
// expandDirectories) -> VALIDATE the whole surviving set -> only then ADD it to the target
|
|
340
|
-
// bundle(s), add-if-missing. A stasis.lock.json is updated only when one already exists.
|
|
341
249
|
export function addCommand({ cwd = process.cwd(), entries, logLabel = 'stasis-core' } = {}) {
|
|
342
|
-
if (!Array.isArray(entries) || entries.length === 0)
|
|
343
|
-
throw new Error('add: at least one file is required')
|
|
344
|
-
}
|
|
250
|
+
if (!Array.isArray(entries) || entries.length === 0) throw new Error('add: at least one file is required')
|
|
345
251
|
const baseDir = resolve(cwd)
|
|
346
252
|
const realBase = realpathSync(baseDir)
|
|
347
|
-
const { bundleFile, resourcesBundleFile, resources, brotliQuality } = readAddConfig(baseDir)
|
|
253
|
+
const { bundleFile, resourcesBundleFile, resources, brotliQuality, policy } = readAddConfig(baseDir)
|
|
254
|
+
const links = linksOn(baseDir)
|
|
255
|
+
const check = policy && ((artifact, hint) => policy.assertArtifact(artifact, { hint, links }))
|
|
348
256
|
|
|
349
257
|
const rootPkg = readJson(join(baseDir, 'package.json')) ?? {}
|
|
350
258
|
const workspaceName = rootPkg.name ?? 'workspace'
|
|
351
259
|
const workspaceVersion = rootPkg.version ?? '0.0.0'
|
|
352
260
|
|
|
353
|
-
// Lockfile updated only if one already exists (never created); hash packed bytes to match a frozen run.
|
|
354
261
|
const lockPath = join(baseDir, LOCK_FILE)
|
|
355
262
|
const hasLock = existsSync(lockPath)
|
|
356
263
|
|
|
357
|
-
//
|
|
358
|
-
//
|
|
359
|
-
// by-name rule can't recognize (it does cover the lockfile). One configured outside the project
|
|
360
|
-
// yields a '..' path, which no swept path can equal.
|
|
264
|
+
// A sweep must never attest what this run writes, and isAutoExcludedFile's by-name rule can't know a target named
|
|
265
|
+
// `dist/code.br`. One outside the project yields a '..' path, which no swept path can equal.
|
|
361
266
|
const outputs = new Set([bundleFile, resourcesBundleFile]
|
|
362
267
|
.filter((file) => file !== undefined)
|
|
363
268
|
.map((file) => toPosix(relative(baseDir, resolve(baseDir, file)))))
|
|
364
269
|
|
|
365
|
-
// `excluded` is counted in the summary below rather than dropped silently.
|
|
366
270
|
const { files, excluded } = expandDirectories(baseDir, normalizeEntries(entries, cwd), outputs)
|
|
367
271
|
if (files.size === 0) {
|
|
368
272
|
throw new Error(`add: no files to add (directory entries matched ${excluded.size === 0
|
|
@@ -371,31 +275,42 @@ export function addCommand({ cwd = process.cwd(), entries, logLabel = 'stasis-co
|
|
|
371
275
|
|
|
372
276
|
const { codeFiles, resourceFiles, integrities } = validateFiles({ baseDir, realBase, files, resources, withIntegrity: hasLock })
|
|
373
277
|
|
|
374
|
-
//
|
|
278
|
+
// Every target's merge is computed first, then the writes run -- see prepareBundleFile.
|
|
375
279
|
const summary = []
|
|
376
280
|
const writes = []
|
|
377
281
|
const repo = detectRepo(baseDir)
|
|
282
|
+
// Each artifact is read before a file is added to any: what one attests of a package is held of the others too.
|
|
283
|
+
const bundleAt = (file, abs = resolve(baseDir, file)) => (existsSync(abs) ? existingOf(`bundle at ${file}`, () => parseBundleBytes(readFileSync(abs))) : undefined)
|
|
284
|
+
const bundles = new Map([bundleFile, resourcesBundleFile].filter(Boolean).map((file) => [file, bundleAt(file)]))
|
|
285
|
+
const lock = hasLock ? existingOf(LOCK_FILE, () => Lockfile.parse(readFileSync(lockPath, 'utf8'))) : undefined
|
|
286
|
+
const attest = heldToInstall(baseDir, [...bundles, [LOCK_FILE, lock]].filter(([, artifact]) => artifact))
|
|
378
287
|
const planTarget = (target, entriesForTarget, kind) => {
|
|
379
288
|
if (entriesForTarget.size === 0) return
|
|
380
|
-
const
|
|
289
|
+
const existing = bundles.get(target)
|
|
290
|
+
const companion = bundles.get(target === bundleFile ? resourcesBundleFile : bundleFile)
|
|
291
|
+
const added = attest(existing, assembleBundle(baseDir, entriesForTarget, workspaceName, workspaceVersion, repo), companion)
|
|
292
|
+
const { write, total, added: count } = prepareBundleFile(baseDir, target, added, brotliQuality, check, existing)
|
|
381
293
|
writes.push(write)
|
|
382
|
-
summary.push(`+${
|
|
294
|
+
summary.push(`+${count} ${kind} (${total} total) -> ${target}`)
|
|
383
295
|
}
|
|
296
|
+
const allFiles = new Map([...codeFiles, ...resourceFiles])
|
|
384
297
|
if (resourcesBundleFile) {
|
|
385
298
|
planTarget(bundleFile, codeFiles, 'source')
|
|
386
299
|
planTarget(resourcesBundleFile, resourceFiles, 'resource')
|
|
387
300
|
} else {
|
|
388
301
|
const kind = codeFiles.size > 0 && resourceFiles.size > 0 ? 'file' : resourceFiles.size > 0 ? 'resource' : 'source'
|
|
389
|
-
planTarget(bundleFile,
|
|
302
|
+
planTarget(bundleFile, allFiles, kind)
|
|
390
303
|
}
|
|
391
304
|
|
|
392
|
-
// One lockfile attests both split targets, so
|
|
305
|
+
// One lockfile attests both split targets, so it takes every packed file's integrity (code + resources).
|
|
393
306
|
if (hasLock) {
|
|
394
|
-
const
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
307
|
+
const lockAdd = attest(lock, Lockfile.fromBundle(assembleBundle(baseDir, allFiles, workspaceName, workspaceVersion),
|
|
308
|
+
(file) => integrities.get(file)))
|
|
309
|
+
// `check` sees only what it already records: the files added were checked as the bundles'.
|
|
310
|
+
check?.(lock, recordedIn(LOCK_FILE))
|
|
311
|
+
const merged = lock.merge(lockAdd)
|
|
312
|
+
writes.push(() => writeFileSync(lockPath, merged.serialize()))
|
|
313
|
+
summary.push(`+${fileCount(merged) - fileCount(lock)} (${fileCount(merged)} total) -> ${LOCK_FILE}`)
|
|
399
314
|
}
|
|
400
315
|
if (excluded.size > 0) summary.push(`skipped ${excluded.size} auto-excluded`)
|
|
401
316
|
|