@lakutata/builder 3.0.0-beta.3 → 3.0.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/esm/lib/Bundle.js +95 -36
- package/dist/esm/lib/Executable.js +3 -1
- package/dist/esm/lib/Files.js +557 -0
- package/dist/esm/lib/Native.js +28 -9
- package/dist/esm/lib/NativeRuntime.js +29 -2
- package/dist/esm/lib/electron/Electron.js +3 -0
- package/dist/types/lib/Bundle.d.ts +14 -0
- package/dist/types/lib/Executable.d.ts +9 -0
- package/dist/types/lib/Files.d.ts +120 -0
- package/dist/types/lib/Native.d.ts +6 -2
- package/dist/types/lib/NativeRuntime.d.ts +5 -0
- package/package.json +8 -2
package/README.md
CHANGED
|
@@ -16,7 +16,11 @@ await buildInstaller({project: process.cwd()})
|
|
|
16
16
|
```
|
|
17
17
|
|
|
18
18
|
The application is compiled first (`tsc`, which emits the metadata of the decorators). The bundle is one CommonJS file
|
|
19
|
-
(esbuild); the native addons, which cannot be bundled, are gathered with what they load at run time into a native tree
|
|
19
|
+
(esbuild); the native addons, which cannot be bundled, are gathered with what they load at run time into a native tree,
|
|
20
|
+
with the files the application reads by their paths, which the bundler does not follow (the scripts it runs on their
|
|
21
|
+
own: the workers, the pools of workers, `fork()`, `require(path.join(__dirname, 'x.js'))`, bundled; its data files and
|
|
22
|
+
directories, copied: the paths given from their modules are found and rewritten; what cannot be shipped is in the
|
|
23
|
+
`warnings` of the result);
|
|
20
24
|
the resources of the application (the `lakutata.resources` of its `package.json`) are found at run time by
|
|
21
25
|
`resourcesPath()` of lakutata. The installers are configured by the `lakutata.installer` of the `package.json`. See the
|
|
22
26
|
documentation of `lakutata package bundle`, `lakutata package exe` and `lakutata package installer` in `@lakutata/cli`.
|
package/dist/esm/lib/Bundle.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { existsSync, readdirSync, statSync } from 'node:fs';
|
|
1
|
+
import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs';
|
|
2
2
|
import { chmod, cp, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { hashNativeTree, NativeCollector, writeNativeTree } from './Native.js';
|
|
5
5
|
import { EMBEDDED_RESOURCES, nativeBanner, packagedBanner } from './NativeRuntime.js';
|
|
6
|
+
import { FILES_DIRECTORY, FileCollector } from './Files.js';
|
|
6
7
|
/**
|
|
7
8
|
* The name of the program of a package: its first bin, or the name of the package without its scope
|
|
8
9
|
* @param manifest
|
|
@@ -57,6 +58,18 @@ export function resourceFiles(project, manifest) {
|
|
|
57
58
|
}
|
|
58
59
|
return { declared: declarations.map(({ to }) => to), files: files };
|
|
59
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* A package.json copied into the tree, its type CommonJS; undefined when it is not JSON (copied as it is)
|
|
63
|
+
* @param file
|
|
64
|
+
*/
|
|
65
|
+
function commonJSManifest(file) {
|
|
66
|
+
try {
|
|
67
|
+
return Buffer.from(`${JSON.stringify({ ...JSON.parse(readFileSync(file, 'utf8')), type: 'commonjs' }, null, 2)}\n`);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
60
73
|
/**
|
|
61
74
|
* The code with a code before it, after its hashbang
|
|
62
75
|
* @param code
|
|
@@ -85,49 +98,92 @@ export async function bundleProject(options) {
|
|
|
85
98
|
throw new Error(`The entry ${path.relative(process.cwd(), entry) || entry} does not exist: build the project first`);
|
|
86
99
|
const outfile = path.resolve(project, options.outfile ?? path.join('dist', `${name}.cjs`));
|
|
87
100
|
const collector = new NativeCollector(project, options.runtime);
|
|
101
|
+
const files = new FileCollector(project);
|
|
88
102
|
//esbuild is loaded by the commands which need it
|
|
89
103
|
const { build } = await import('esbuild');
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
104
|
+
//The application, and each script of its workers (the bundler does not follow new Worker()): their native packages
|
|
105
|
+
//gathered into one tree, the workers they start found too
|
|
106
|
+
const bundle = async (input, output) => {
|
|
107
|
+
const result = await build({
|
|
108
|
+
entryPoints: [input],
|
|
109
|
+
outfile: output,
|
|
110
|
+
write: false,
|
|
111
|
+
metafile: true,
|
|
112
|
+
bundle: true,
|
|
113
|
+
platform: 'node',
|
|
114
|
+
target: `node${process.versions.node.split('.')[0]}`,
|
|
115
|
+
format: 'cjs',
|
|
116
|
+
minify: options.minify ?? true,
|
|
117
|
+
//The names of the classes are the ones of the objects of the container and of the errors
|
|
118
|
+
keepNames: true,
|
|
119
|
+
external: options.external ?? [],
|
|
120
|
+
//The dynamic imports of the modules left out are requires (the require of the bundle loads the native ones)
|
|
121
|
+
supported: { 'dynamic-import': false },
|
|
122
|
+
banner: { js: `var ${IMPORT_META_URL}=require('node:url').pathToFileURL(__filename).href;` },
|
|
123
|
+
define: {
|
|
124
|
+
'import.meta.url': IMPORT_META_URL,
|
|
125
|
+
//Bun replaces process.env.NODE_ENV with the mode of its build: the one of the run is read
|
|
126
|
+
...options.runtime === 'bun' ? { 'process.env.NODE_ENV': 'globalThis.process.env.NODE_ENV' } : {}
|
|
127
|
+
},
|
|
128
|
+
plugins: [collector.plugin([(source, file) => files.transform(source, file)])],
|
|
129
|
+
logLevel: 'silent'
|
|
130
|
+
}).catch((error) => {
|
|
131
|
+
const messages = (error.errors ?? []).map(({ text, location }) => location ? `${location.file}:${location.line}: ${text}` : text);
|
|
132
|
+
//The bundles are CommonJS, which the executables of Node.js run
|
|
133
|
+
const hint = messages.some((message) => message.includes('Top-level await')) ? '\nThe bundles are CommonJS modules, without top-level await: await in an async function' : '';
|
|
134
|
+
throw new Error(`The bundle failed:\n ${messages.length ? messages.join('\n ') : error.message}${hint}`);
|
|
135
|
+
});
|
|
136
|
+
//The files bundled (their real paths; the modules of the plugins are in their namespaces)
|
|
137
|
+
const inputs = new Set(Object.keys(result.metafile.inputs).filter((input) => !/^[a-z-]+:/.test(input)).map((input) => {
|
|
138
|
+
const absolute = path.resolve(input);
|
|
139
|
+
return existsSync(absolute) ? realpathSync(absolute) : absolute;
|
|
140
|
+
}));
|
|
141
|
+
return { code: result.outputFiles[0].text, inputs: inputs };
|
|
142
|
+
};
|
|
143
|
+
const application = await bundle(entry, outfile);
|
|
144
|
+
let code = application.code;
|
|
122
145
|
const base = path.basename(outfile, path.extname(outfile));
|
|
123
146
|
const nativeDirectory = path.resolve(path.dirname(outfile), options.directories?.native ?? `${base}.native`);
|
|
124
147
|
const resourcesDirectory = path.resolve(path.dirname(outfile), options.directories?.resources ?? `${base}.resources`);
|
|
148
|
+
//The scripts of the workers, until the ones they start are bundled too
|
|
149
|
+
const scripts = new Map();
|
|
150
|
+
for (let pending = [...files.scripts]; pending.length; pending = [...files.scripts].filter(([at]) => !scripts.has(at))) {
|
|
151
|
+
for (const [at, script] of pending)
|
|
152
|
+
scripts.set(at, (await bundle(script, path.join(nativeDirectory, ...at.split('/')))).code);
|
|
153
|
+
}
|
|
154
|
+
files.warnBundled(application.inputs);
|
|
155
|
+
const embedded = options.native === 'embedded';
|
|
156
|
+
let native = await collector.collect();
|
|
157
|
+
const resources = resourceFiles(project, manifest);
|
|
158
|
+
if (scripts.size || files.files.size) {
|
|
159
|
+
//The scripts run from the tree: their require loads the native packages from it, their resources are the ones of
|
|
160
|
+
//the application (an executable gives them from the tree it extracted)
|
|
161
|
+
const modules = native?.modules ?? [];
|
|
162
|
+
const tree = [{ path: `${FILES_DIRECTORY}/package.json`, source: manifestFile, mode: 0o644, content: Buffer.from('{"type": "commonjs"}\n') }];
|
|
163
|
+
for (const [at, script] of [...scripts].sort(([a], [b]) => a.localeCompare(b))) {
|
|
164
|
+
const directory = path.dirname(path.join(nativeDirectory, ...at.split('/')));
|
|
165
|
+
const banner = packagedBanner(path.relative(directory, resourcesDirectory).split(path.sep).join('/')) +
|
|
166
|
+
nativeBanner({ name: name, hash: '', directory: '', modules: modules, embedded: embedded, root: path.relative(directory, nativeDirectory).split(path.sep).join('/') });
|
|
167
|
+
tree.push({ path: at, source: files.scripts.get(at), mode: 0o644, content: Buffer.from(prepend(script, banner)) });
|
|
168
|
+
}
|
|
169
|
+
//The other files the application reads by their paths, as they are; a package.json read by its package (its
|
|
170
|
+
//version) is of the CommonJS scripts of the tree next to it (its type would make them ES modules)
|
|
171
|
+
for (const [at, data] of files.files)
|
|
172
|
+
tree.push({ path: at, source: data, mode: statSync(data).mode & 0o777, content: path.basename(data) === 'package.json' ? commonJSManifest(data) : undefined });
|
|
173
|
+
const all = [...native?.files ?? [], ...tree];
|
|
174
|
+
native = { modules: modules, files: all, links: native?.links ?? [], hash: hashNativeTree(all, native?.links ?? []) };
|
|
175
|
+
}
|
|
125
176
|
await rm(nativeDirectory, { recursive: true, force: true });
|
|
126
177
|
await rm(resourcesDirectory, { recursive: true, force: true });
|
|
127
178
|
//The resources of an executable are extracted with its native tree
|
|
128
179
|
if (embedded && resources.files.length) {
|
|
129
|
-
const
|
|
130
|
-
native = { modules: native?.modules ?? [], files:
|
|
180
|
+
const embeddedFiles = [...native?.files ?? [], ...resources.files.map((file) => ({ ...file, path: `${EMBEDDED_RESOURCES}/${file.path}` }))];
|
|
181
|
+
native = { modules: native?.modules ?? [], files: embeddedFiles, links: native?.links ?? [], hash: hashNativeTree(embeddedFiles, native?.links ?? []) };
|
|
182
|
+
}
|
|
183
|
+
if (native && embedded) {
|
|
184
|
+
//An executable runs the scripts of its tree it is started for (fork()) instead of the application
|
|
185
|
+
const hashbang = /^#![^\n]*\n/.exec(code)?.[0] ?? '';
|
|
186
|
+
code = `${hashbang}if (!require.lakutataScript()) {\n${code.slice(hashbang.length)}\n}\n`;
|
|
131
187
|
}
|
|
132
188
|
if (native) {
|
|
133
189
|
//The require loading the native packages is the first code, after the hashbang
|
|
@@ -157,6 +213,9 @@ export async function bundleProject(options) {
|
|
|
157
213
|
native: native,
|
|
158
214
|
nativeDirectory: native && !embedded ? nativeDirectory : undefined,
|
|
159
215
|
resources: resources.declared,
|
|
160
|
-
resourcesDirectory: resources.files.length && !embedded ? resourcesDirectory : undefined
|
|
216
|
+
resourcesDirectory: resources.files.length && !embedded ? resourcesDirectory : undefined,
|
|
217
|
+
workers: [...new Set([...scripts.keys()].map((at) => files.nameOf(files.scripts.get(at))))].sort(),
|
|
218
|
+
files: [...files.roots].filter((root) => !files.scriptPaths().has(root)).map((root) => files.nameOf(root)).sort(),
|
|
219
|
+
warnings: files.warnings
|
|
161
220
|
};
|
|
162
221
|
}
|
|
@@ -196,7 +196,7 @@ export async function buildExecutable(options) {
|
|
|
196
196
|
//ignores the case), whose signature it writes beside the executable (_CodeSignature)
|
|
197
197
|
const made = path.join(work, 'executable', path.basename(file));
|
|
198
198
|
await mkdir(path.dirname(made), { recursive: true });
|
|
199
|
-
const warnings = [];
|
|
199
|
+
const warnings = [...bundle.warnings];
|
|
200
200
|
//The icon of the application: the one of the executables of Windows (the ones of macOS and Linux have none)
|
|
201
201
|
const application = applicationIcon(project, manifest);
|
|
202
202
|
const icon = application && process.platform === 'win32' ? icoFile(application.image) : undefined;
|
|
@@ -209,6 +209,8 @@ export async function buildExecutable(options) {
|
|
|
209
209
|
platform: `${process.platform === 'win32' ? 'win' : process.platform}-${process.arch}`,
|
|
210
210
|
size: (await stat(file)).size,
|
|
211
211
|
native: bundle.native?.modules ?? [],
|
|
212
|
+
workers: bundle.workers,
|
|
213
|
+
files: bundle.files,
|
|
212
214
|
resources: bundle.resources,
|
|
213
215
|
directory: directory ? target : undefined,
|
|
214
216
|
runtime: runtime,
|
|
@@ -0,0 +1,557 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { existsSync, readdirSync, realpathSync, statSync } from 'node:fs';
|
|
3
|
+
import { createRequire } from 'node:module';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { parse } from 'acorn';
|
|
6
|
+
/**
|
|
7
|
+
* The directory of the files the application reads by their paths in the native tree: the scripts it runs on their own
|
|
8
|
+
* (bundled), the other files and the directories of data (copied), their directories as in the project
|
|
9
|
+
*/
|
|
10
|
+
export const FILES_DIRECTORY = '.lakutata/files';
|
|
11
|
+
/**
|
|
12
|
+
* The scripts run on their own: the CommonJS scripts, which they are bundled as
|
|
13
|
+
*/
|
|
14
|
+
const SCRIPT = /\.c?js$/;
|
|
15
|
+
/**
|
|
16
|
+
* The code a directory of data must not hold: the scripts, which need their packages, and the packages
|
|
17
|
+
*/
|
|
18
|
+
const CODE = /\.(c|m)?js$|^package\.json$|^node_modules$/;
|
|
19
|
+
/**
|
|
20
|
+
* The files of a directory, deep
|
|
21
|
+
* @param directory
|
|
22
|
+
*/
|
|
23
|
+
function filesOf(directory) {
|
|
24
|
+
const files = [];
|
|
25
|
+
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
26
|
+
const at = path.join(directory, entry.name);
|
|
27
|
+
if (entry.isDirectory())
|
|
28
|
+
files.push(...filesOf(at));
|
|
29
|
+
else if (entry.isFile())
|
|
30
|
+
files.push(at);
|
|
31
|
+
}
|
|
32
|
+
return files;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Whether a directory holds code: scripts, packages
|
|
36
|
+
* @param directory
|
|
37
|
+
*/
|
|
38
|
+
function holdsCode(directory) {
|
|
39
|
+
return readdirSync(directory, { withFileTypes: true }).some((entry) => CODE.test(entry.name) || (entry.isDirectory() && holdsCode(path.join(directory, entry.name))));
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The modules of the worker threads
|
|
43
|
+
*/
|
|
44
|
+
const WORKER_THREADS = new Set(['worker_threads', 'node:worker_threads']);
|
|
45
|
+
/**
|
|
46
|
+
* The name of a function called: the identifier, the property (path.join, node_path_1.join), the last expression of a
|
|
47
|
+
* sequence (the calls of tsc: (0, node_path_1.join)(...))
|
|
48
|
+
* @param node
|
|
49
|
+
*/
|
|
50
|
+
function calleeName(node) {
|
|
51
|
+
if (node?.type === 'Identifier')
|
|
52
|
+
return node.name;
|
|
53
|
+
if (node?.type === 'MemberExpression')
|
|
54
|
+
return node.computed ? (typeof node.property.value === 'string' ? node.property.value : undefined) : node.property.name;
|
|
55
|
+
if (node?.type === 'SequenceExpression')
|
|
56
|
+
return calleeName(node.expressions[node.expressions.length - 1]);
|
|
57
|
+
return undefined;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Whether a node is import.meta.<name>
|
|
61
|
+
* @param node
|
|
62
|
+
* @param name
|
|
63
|
+
*/
|
|
64
|
+
function isImportMeta(node, name) {
|
|
65
|
+
return node?.type === 'MemberExpression' && !node.computed && node.object.type === 'MetaProperty' && node.object.meta.name === 'import' && node.property.name === name;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Whether a node is the URL of the file of the module: import.meta.url, pathToFileURL(__filename) (and its href)
|
|
69
|
+
* @param node
|
|
70
|
+
*/
|
|
71
|
+
function isFileURL(node) {
|
|
72
|
+
if (isImportMeta(node, 'url'))
|
|
73
|
+
return true;
|
|
74
|
+
if (node?.type === 'CallExpression' && calleeName(node.callee) === 'pathToFileURL')
|
|
75
|
+
return isFilename(node.arguments[0]);
|
|
76
|
+
if (node?.type === 'MemberExpression' && !node.computed && node.property.name === 'href')
|
|
77
|
+
return isFileURL(node.object);
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Whether a node is the file of the module: __filename, import.meta.filename, fileURLToPath(import.meta.url)
|
|
82
|
+
* @param node
|
|
83
|
+
*/
|
|
84
|
+
function isFilename(node) {
|
|
85
|
+
if (node?.type === 'Identifier')
|
|
86
|
+
return node.name === '__filename';
|
|
87
|
+
if (isImportMeta(node, 'filename'))
|
|
88
|
+
return true;
|
|
89
|
+
return node?.type === 'CallExpression' && calleeName(node.callee) === 'fileURLToPath' && isImportMeta(node.arguments[0], 'url');
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Whether a node is the directory of the module: __dirname, import.meta.dirname, dirname(<its file>),
|
|
93
|
+
* fileURLToPath(new URL('.', <its URL>))
|
|
94
|
+
* @param node
|
|
95
|
+
*/
|
|
96
|
+
function isDirname(node) {
|
|
97
|
+
if (node?.type === 'Identifier')
|
|
98
|
+
return node.name === '__dirname';
|
|
99
|
+
if (isImportMeta(node, 'dirname'))
|
|
100
|
+
return true;
|
|
101
|
+
if (node?.type !== 'CallExpression')
|
|
102
|
+
return false;
|
|
103
|
+
const name = calleeName(node.callee);
|
|
104
|
+
if (name === 'dirname')
|
|
105
|
+
return isFilename(node.arguments[0]);
|
|
106
|
+
if (name !== 'fileURLToPath')
|
|
107
|
+
return false;
|
|
108
|
+
const url = node.arguments[0];
|
|
109
|
+
return url?.type === 'NewExpression' && calleeName(url.callee) === 'URL' && url.arguments[0]?.type === 'Literal' && ['.', './'].includes(url.arguments[0].value) && isFileURL(url.arguments[1]);
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Whether a node is require('<module>') of one of the modules
|
|
113
|
+
* @param node
|
|
114
|
+
* @param modules
|
|
115
|
+
*/
|
|
116
|
+
function isRequireOf(node, modules) {
|
|
117
|
+
return node?.type === 'CallExpression' && node.callee.type === 'Identifier' && node.callee.name === 'require' && node.arguments[0]?.type === 'Literal' && modules.has(node.arguments[0].value);
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* The pieces of a string expression: its literal texts, the expressions known at run time only
|
|
121
|
+
* @param node
|
|
122
|
+
* @param source
|
|
123
|
+
*/
|
|
124
|
+
function partsOf(node, source) {
|
|
125
|
+
if (node?.type === 'Literal')
|
|
126
|
+
return typeof node.value === 'string' ? [{ text: node.value }] : undefined;
|
|
127
|
+
if (node?.type === 'TemplateLiteral') {
|
|
128
|
+
const parts = [];
|
|
129
|
+
node.quasis.forEach((quasi, index) => {
|
|
130
|
+
if (quasi.value.cooked)
|
|
131
|
+
parts.push({ text: quasi.value.cooked });
|
|
132
|
+
if (index < node.expressions.length)
|
|
133
|
+
parts.push({ expression: source.slice(node.expressions[index].start, node.expressions[index].end) });
|
|
134
|
+
});
|
|
135
|
+
return parts;
|
|
136
|
+
}
|
|
137
|
+
if (node?.type === 'BinaryExpression' && node.operator === '+') {
|
|
138
|
+
const left = partsOf(node.left, source);
|
|
139
|
+
const right = partsOf(node.right, source);
|
|
140
|
+
return left && right ? [...left, ...right] : undefined;
|
|
141
|
+
}
|
|
142
|
+
if (node?.type === 'SpreadElement')
|
|
143
|
+
return undefined;
|
|
144
|
+
return [{ expression: source.slice(node.start, node.end) }];
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* The operands of a chain of concatenations, from the first one
|
|
148
|
+
* @param node
|
|
149
|
+
*/
|
|
150
|
+
function operandsOf(node) {
|
|
151
|
+
return node?.type === 'BinaryExpression' && node.operator === '+' ? [...operandsOf(node.left), node.right] : [node];
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* The parts of the operands following a base
|
|
155
|
+
* @param operands
|
|
156
|
+
* @param source
|
|
157
|
+
*/
|
|
158
|
+
function partsOfOperands(operands, source) {
|
|
159
|
+
const parts = [];
|
|
160
|
+
for (const operand of operands) {
|
|
161
|
+
const pieces = partsOf(operand, source);
|
|
162
|
+
if (!pieces)
|
|
163
|
+
return undefined;
|
|
164
|
+
parts.push(...pieces);
|
|
165
|
+
}
|
|
166
|
+
return parts;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* The text of parts known when the application is bundled, undefined when one is known at run time only
|
|
170
|
+
* @param parts
|
|
171
|
+
*/
|
|
172
|
+
function textOf(parts) {
|
|
173
|
+
return parts.every((part) => 'text' in part) ? parts.map((part) => part.text).join('') : undefined;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* The path to a script an expression gives from its module, undefined when it gives none
|
|
177
|
+
* @param node
|
|
178
|
+
* @param source
|
|
179
|
+
* @param file the module
|
|
180
|
+
*/
|
|
181
|
+
function scriptPathOf(node, source, file) {
|
|
182
|
+
//new URL('./worker.js', import.meta.url), new URL('./worker.js', pathToFileURL(__filename))
|
|
183
|
+
if (node?.type === 'NewExpression' && calleeName(node.callee) === 'URL' && node.arguments.length === 2 && isFileURL(node.arguments[1])) {
|
|
184
|
+
const parts = partsOf(node.arguments[0], source);
|
|
185
|
+
const first = parts?.[0];
|
|
186
|
+
//A path relative to the module
|
|
187
|
+
if (!parts || !first || !('text' in first) || /^([a-z][a-z0-9+.-]*:|\/)/i.test(first.text))
|
|
188
|
+
return undefined;
|
|
189
|
+
return { kind: 'url', parts: parts };
|
|
190
|
+
}
|
|
191
|
+
if (node?.type === 'CallExpression') {
|
|
192
|
+
const name = calleeName(node.callee);
|
|
193
|
+
//path.join(__dirname, 'worker.js'), path.resolve(__dirname, 'workers', `${name}.js`)
|
|
194
|
+
if ((name === 'join' || name === 'resolve') && isDirname(node.arguments[0])) {
|
|
195
|
+
const parts = [];
|
|
196
|
+
for (const argument of node.arguments.slice(1)) {
|
|
197
|
+
const pieces = partsOf(argument, source);
|
|
198
|
+
if (!pieces)
|
|
199
|
+
return undefined;
|
|
200
|
+
parts.push({ text: '/' }, ...pieces);
|
|
201
|
+
}
|
|
202
|
+
return { kind: 'path', parts: parts };
|
|
203
|
+
}
|
|
204
|
+
//require.resolve('./worker'), require.resolve(path.resolve(__dirname, '../lib/worker')): the extension of node found
|
|
205
|
+
if (name === 'resolve' && node.callee.type === 'MemberExpression' && node.callee.object.type === 'Identifier' && node.callee.object.name === 'require' && node.arguments.length === 1) {
|
|
206
|
+
const argument = node.arguments[0];
|
|
207
|
+
let request;
|
|
208
|
+
if (argument.type === 'Literal' && typeof argument.value === 'string' && /^\.\.?\//.test(argument.value))
|
|
209
|
+
request = argument.value;
|
|
210
|
+
const inner = request === undefined ? scriptPathOf(argument, source, file) : undefined;
|
|
211
|
+
const text = inner?.kind === 'path' ? textOf(inner.parts) : undefined;
|
|
212
|
+
if (text !== undefined)
|
|
213
|
+
request = path.join(inner.base ?? path.dirname(file), text);
|
|
214
|
+
if (request === undefined)
|
|
215
|
+
return undefined;
|
|
216
|
+
try {
|
|
217
|
+
const resolved = createRequire(file).resolve(request);
|
|
218
|
+
return { kind: 'path', parts: [{ text: path.basename(resolved) }], base: path.dirname(resolved) };
|
|
219
|
+
}
|
|
220
|
+
catch {
|
|
221
|
+
return undefined;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if (node?.type === 'BinaryExpression' && node.operator === '+') {
|
|
226
|
+
const [base, ...rest] = operandsOf(node);
|
|
227
|
+
//__dirname + '/worker.js'
|
|
228
|
+
if (isDirname(base)) {
|
|
229
|
+
const parts = partsOfOperands(rest, source);
|
|
230
|
+
return parts ? { kind: 'path', parts: parts } : undefined;
|
|
231
|
+
}
|
|
232
|
+
//import.meta.url + '/../entry/worker.js': the URL of the module, as a directory
|
|
233
|
+
if (isFileURL(base)) {
|
|
234
|
+
const parts = partsOfOperands(rest, source);
|
|
235
|
+
const first = parts?.[0];
|
|
236
|
+
return parts && first && 'text' in first && first.text.startsWith('/') ? { kind: 'url-text', parts: parts, base: file } : undefined;
|
|
237
|
+
}
|
|
238
|
+
return undefined;
|
|
239
|
+
}
|
|
240
|
+
//`${__dirname}/worker.js`
|
|
241
|
+
if (node?.type === 'TemplateLiteral' && node.expressions.length && !node.quasis[0].value.cooked && isDirname(node.expressions[0])) {
|
|
242
|
+
const parts = [];
|
|
243
|
+
node.quasis.forEach((quasi, index) => {
|
|
244
|
+
if (quasi.value.cooked)
|
|
245
|
+
parts.push({ text: quasi.value.cooked });
|
|
246
|
+
if (index > 0 && index < node.expressions.length)
|
|
247
|
+
parts.push({ expression: source.slice(node.expressions[index].start, node.expressions[index].end) });
|
|
248
|
+
});
|
|
249
|
+
return { kind: 'path', parts: parts };
|
|
250
|
+
}
|
|
251
|
+
return undefined;
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Visit the nodes of a syntax tree, depth first: the children of a node are not visited when the visitor returns false
|
|
255
|
+
* @param node
|
|
256
|
+
* @param visit
|
|
257
|
+
*/
|
|
258
|
+
function walk(node, visit) {
|
|
259
|
+
if (!node || typeof node.type !== 'string')
|
|
260
|
+
return;
|
|
261
|
+
if (visit(node) === false)
|
|
262
|
+
return;
|
|
263
|
+
for (const key of Object.keys(node)) {
|
|
264
|
+
const value = node[key];
|
|
265
|
+
if (Array.isArray(value))
|
|
266
|
+
for (const child of value)
|
|
267
|
+
walk(child, visit);
|
|
268
|
+
else if (value && typeof value === 'object' && typeof value.type === 'string')
|
|
269
|
+
walk(value, visit);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* A text in a template literal
|
|
274
|
+
* @param text
|
|
275
|
+
*/
|
|
276
|
+
function templateText(text) {
|
|
277
|
+
return text.replace(/[`\\]|\$\{/g, (matched) => `\\${matched}`);
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* The scripts an application runs on its own, by their paths, which the bundler does not follow: the scripts of its
|
|
281
|
+
* workers (new Worker(), the pools of workers given a file), of its child processes (fork()), the modules it loads by
|
|
282
|
+
* their paths (require(path.join(__dirname, 'x.js'))), in its modules and in the ones of its packages. A path to a script
|
|
283
|
+
* given from its module, wherever it is written, is found when the module is bundled: the script is bundled on its own
|
|
284
|
+
* into the native tree (FILES_DIRECTORY, the directories mirrored), and the expression rewritten to it. A path known at
|
|
285
|
+
* run time only in part (a template, a concatenation, an argument of path.join: `${name}.js`) gives every script of its
|
|
286
|
+
* directory its pattern matches, the expression of the part kept. The scripts of the workers of node (worker_threads)
|
|
287
|
+
* whose paths cannot be followed are warned of, in the modules of the application (the packages start workers from the
|
|
288
|
+
* paths their users give)
|
|
289
|
+
*/
|
|
290
|
+
export class FileCollector {
|
|
291
|
+
/**
|
|
292
|
+
* @param project The directory of the project: the scripts are named from it (as the modules the bundler gives, its
|
|
293
|
+
* real path)
|
|
294
|
+
*/
|
|
295
|
+
constructor(project) {
|
|
296
|
+
/**
|
|
297
|
+
* The scripts: their paths in the native tree, and their files
|
|
298
|
+
*/
|
|
299
|
+
this.scripts = new Map();
|
|
300
|
+
/**
|
|
301
|
+
* The other files: their paths in the native tree, and their files
|
|
302
|
+
*/
|
|
303
|
+
this.files = new Map();
|
|
304
|
+
/**
|
|
305
|
+
* The files and the directories of data the paths name
|
|
306
|
+
*/
|
|
307
|
+
this.roots = new Set();
|
|
308
|
+
this.warnings = [];
|
|
309
|
+
this.project = realpathSync(project);
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* The key of a directory in the tree ("/" separated): its path from the project; outside of the project (the
|
|
313
|
+
* packages of a workspace, linked), its path from the node_modules holding it, else a hash of its path (the paths of
|
|
314
|
+
* the machine bundling it are not shipped)
|
|
315
|
+
* @param directory
|
|
316
|
+
*/
|
|
317
|
+
keyOf(directory) {
|
|
318
|
+
const relative = path.relative(this.project, directory);
|
|
319
|
+
if (!relative.startsWith('..'))
|
|
320
|
+
return relative.split(path.sep).join('/');
|
|
321
|
+
const segments = directory.split(path.sep);
|
|
322
|
+
const modules = segments.indexOf('node_modules');
|
|
323
|
+
return modules >= 0 ? segments.slice(modules).join('/') : `_outside/${createHash('sha256').update(directory).digest('hex').slice(0, 16)}`;
|
|
324
|
+
}
|
|
325
|
+
/**
|
|
326
|
+
* The files of the scripts
|
|
327
|
+
*/
|
|
328
|
+
scriptPaths() {
|
|
329
|
+
return new Set(this.scripts.values());
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* The name of a script: its path from the project ("/" separated), or from the node_modules holding it outside of the
|
|
333
|
+
* project (the packages of a workspace)
|
|
334
|
+
* @param script
|
|
335
|
+
*/
|
|
336
|
+
nameOf(script) {
|
|
337
|
+
const relative = path.relative(this.project, script);
|
|
338
|
+
const segments = script.split(path.sep);
|
|
339
|
+
const modules = segments.indexOf('node_modules');
|
|
340
|
+
return relative.startsWith('..') && modules >= 0 ? segments.slice(modules).join('/') : relative.split(path.sep).join('/');
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* What a path gives: the file or the directory, or the ones its pattern matches (an expression matches a part of a
|
|
344
|
+
* name); the directory they are named from, the parts of their paths from it
|
|
345
|
+
* @param found
|
|
346
|
+
* @param file the module
|
|
347
|
+
*/
|
|
348
|
+
targetsOf(found, file) {
|
|
349
|
+
//The directory: the texts before the first expression, up to their last separator
|
|
350
|
+
const firstExpression = found.parts.findIndex((part) => 'expression' in part);
|
|
351
|
+
const leading = firstExpression < 0 ? found.parts : found.parts.slice(0, firstExpression);
|
|
352
|
+
const prefix = leading.map((part) => part.text).join('');
|
|
353
|
+
const cut = prefix.lastIndexOf('/');
|
|
354
|
+
const directory = path.resolve(path.join(found.base ?? path.dirname(file), prefix.slice(0, cut + 1)));
|
|
355
|
+
const rest = [...prefix.slice(cut + 1) ? [{ text: prefix.slice(cut + 1) }] : [], ...found.parts.slice(leading.length)];
|
|
356
|
+
const targets = { directory: directory, rest: rest, scripts: [], files: [], roots: [] };
|
|
357
|
+
//A file: bundled when it is a script, copied otherwise; a directory of data: its files copied
|
|
358
|
+
const take = (at, strict) => {
|
|
359
|
+
const stats = statSync(at);
|
|
360
|
+
if (stats.isFile()) {
|
|
361
|
+
(SCRIPT.test(at) ? targets.scripts : targets.files).push(at);
|
|
362
|
+
targets.roots.push(at);
|
|
363
|
+
}
|
|
364
|
+
else if (stats.isDirectory()) {
|
|
365
|
+
//A directory holding the module (the application, its packages), or holding code: not shipped
|
|
366
|
+
if ((file + path.sep).startsWith(at + path.sep)) {
|
|
367
|
+
if (strict)
|
|
368
|
+
targets.refused = 'ancestor';
|
|
369
|
+
}
|
|
370
|
+
else if (holdsCode(at)) {
|
|
371
|
+
if (strict)
|
|
372
|
+
targets.refused = 'code';
|
|
373
|
+
}
|
|
374
|
+
else {
|
|
375
|
+
targets.files.push(...filesOf(at));
|
|
376
|
+
targets.roots.push(at);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
};
|
|
380
|
+
if (firstExpression < 0) {
|
|
381
|
+
const target = path.join(directory, prefix.slice(cut + 1));
|
|
382
|
+
if (existsSync(target))
|
|
383
|
+
take(target, true);
|
|
384
|
+
return targets;
|
|
385
|
+
}
|
|
386
|
+
const pattern = new RegExp(`^${rest.map((part) => 'text' in part ? part.text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') : '[^/]*').join('')}$`);
|
|
387
|
+
const depth = rest.map((part) => 'text' in part ? part.text : '').join('').split('/').length;
|
|
388
|
+
const search = (at, relative, level) => {
|
|
389
|
+
if (level > depth || !existsSync(at))
|
|
390
|
+
return;
|
|
391
|
+
for (const entry of readdirSync(at, { withFileTypes: true })) {
|
|
392
|
+
const name = relative ? `${relative}/${entry.name}` : entry.name;
|
|
393
|
+
if (entry.name === 'node_modules')
|
|
394
|
+
continue;
|
|
395
|
+
if (pattern.test(name) && (entry.isFile() || entry.isDirectory()))
|
|
396
|
+
take(path.join(at, entry.name), false);
|
|
397
|
+
else if (entry.isDirectory())
|
|
398
|
+
search(path.join(at, entry.name), name, level + 1);
|
|
399
|
+
}
|
|
400
|
+
};
|
|
401
|
+
search(directory, '', 1);
|
|
402
|
+
targets.scripts.sort();
|
|
403
|
+
targets.files.sort();
|
|
404
|
+
return targets;
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Warn of a worker in a module of the application
|
|
408
|
+
* @param file
|
|
409
|
+
* @param line
|
|
410
|
+
* @param message
|
|
411
|
+
*/
|
|
412
|
+
warn(file, line, message) {
|
|
413
|
+
if (file.split(path.sep).includes('node_modules'))
|
|
414
|
+
return;
|
|
415
|
+
this.warnings.push(`${path.relative(this.project, file)}:${line}: ${message}`);
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Warn of the scripts bundled with the application and run on their own too: two copies of their modules
|
|
419
|
+
* @param bundled the files of the bundle of the application (their real paths)
|
|
420
|
+
*/
|
|
421
|
+
warnBundled(bundled) {
|
|
422
|
+
for (const script of new Set(this.scripts.values())) {
|
|
423
|
+
if (!bundled.has(script) || script.split(path.sep).includes('node_modules'))
|
|
424
|
+
continue;
|
|
425
|
+
this.warnings.push(`${path.relative(this.project, script)}: is bundled with the application and run on its own by its path (a worker, a child process, a module loaded by its path): two copies of its module, their state and their classes apart — load it by its path only, or import it only`);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* The source of a module with its paths to scripts rewritten to the scripts in the native tree; undefined when it
|
|
430
|
+
* gives none
|
|
431
|
+
* @param source
|
|
432
|
+
* @param file
|
|
433
|
+
*/
|
|
434
|
+
transform(source, file) {
|
|
435
|
+
if (!/__dirname|__filename|import\.meta|require\.resolve|\bWorker\b/.test(source))
|
|
436
|
+
return undefined;
|
|
437
|
+
let program;
|
|
438
|
+
for (const sourceType of ['module', 'script']) {
|
|
439
|
+
try {
|
|
440
|
+
program = parse(source, { ecmaVersion: 'latest', sourceType: sourceType, locations: true, allowHashBang: true, allowReturnOutsideFunction: true, allowAwaitOutsideFunction: true, allowImportExportEverywhere: true });
|
|
441
|
+
break;
|
|
442
|
+
}
|
|
443
|
+
catch {
|
|
444
|
+
program = undefined;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
if (!program)
|
|
448
|
+
return undefined;
|
|
449
|
+
//The values of the variables (const script = ...) and of the properties of the classes (workerPath = ..., this.workerPath
|
|
450
|
+
//= ...) declared once; the constructors of the workers of node and their modules (Worker, worker_threads_1)
|
|
451
|
+
const constants = new Map();
|
|
452
|
+
const members = new Map();
|
|
453
|
+
const constructors = new Set();
|
|
454
|
+
const namespaces = new Set();
|
|
455
|
+
const remember = (values, name, value) => values.set(name, values.has(name) ? null : value);
|
|
456
|
+
walk(program, (node) => {
|
|
457
|
+
if (node.type === 'ImportDeclaration' && WORKER_THREADS.has(node.source.value)) {
|
|
458
|
+
for (const specifier of node.specifiers) {
|
|
459
|
+
if (specifier.type === 'ImportSpecifier' && (specifier.imported.name ?? specifier.imported.value) === 'Worker')
|
|
460
|
+
constructors.add(specifier.local.name);
|
|
461
|
+
else if (specifier.type !== 'ImportSpecifier')
|
|
462
|
+
namespaces.add(specifier.local.name);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
if (node.type === 'VariableDeclaration') {
|
|
466
|
+
for (const declarator of node.declarations) {
|
|
467
|
+
const init = declarator.init?.type === 'CallExpression' && isRequireOf(declarator.init.arguments[0], WORKER_THREADS) ? declarator.init.arguments[0] : declarator.init;
|
|
468
|
+
if (declarator.id.type === 'Identifier') {
|
|
469
|
+
remember(constants, declarator.id.name, declarator.init);
|
|
470
|
+
if (isRequireOf(init, WORKER_THREADS))
|
|
471
|
+
namespaces.add(declarator.id.name);
|
|
472
|
+
if (init?.type === 'MemberExpression' && isRequireOf(init.object, WORKER_THREADS) && calleeName(init) === 'Worker')
|
|
473
|
+
constructors.add(declarator.id.name);
|
|
474
|
+
}
|
|
475
|
+
else if (declarator.id.type === 'ObjectPattern' && isRequireOf(init, WORKER_THREADS)) {
|
|
476
|
+
for (const property of declarator.id.properties)
|
|
477
|
+
if (property.key?.name === 'Worker' && property.value?.type === 'Identifier')
|
|
478
|
+
constructors.add(property.value.name);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
if (node.type === 'PropertyDefinition' && !node.computed && node.key && node.value)
|
|
483
|
+
remember(members, node.key.name, node.value);
|
|
484
|
+
if (node.type === 'AssignmentExpression' && node.operator === '=' && node.left.type === 'MemberExpression' && node.left.object.type === 'ThisExpression' && !node.left.computed)
|
|
485
|
+
remember(members, node.left.property.name, node.right);
|
|
486
|
+
});
|
|
487
|
+
//The paths to files, wherever they are written
|
|
488
|
+
const edits = [];
|
|
489
|
+
const followed = new Set();
|
|
490
|
+
walk(program, (node) => {
|
|
491
|
+
const found = scriptPathOf(node, source, file);
|
|
492
|
+
if (!found)
|
|
493
|
+
return true;
|
|
494
|
+
const targets = this.targetsOf(found, file);
|
|
495
|
+
const written = source.slice(node.start, node.end);
|
|
496
|
+
if (targets.refused === 'ancestor') {
|
|
497
|
+
this.warn(file, node.loc.start.line, `the path (${written}) is a directory holding the module, which is not where it is once bundled (the bundle is elsewhere, the code and the packages of the application are not shipped): declare the files it reads as resources (lakutata.resources) and read them from resourcesPath()`);
|
|
498
|
+
return false;
|
|
499
|
+
}
|
|
500
|
+
if (targets.refused === 'code') {
|
|
501
|
+
this.warn(file, node.loc.start.line, `the path (${written}) is a directory of scripts or of packages, which need their packages at run time: it is not shipped with the bundle — import its scripts, or declare it as a resource (lakutata.resources) with what it loads`);
|
|
502
|
+
return false;
|
|
503
|
+
}
|
|
504
|
+
//A path to nothing yet (a directory the application creates): left as it is
|
|
505
|
+
if (!targets.scripts.length && !targets.files.length)
|
|
506
|
+
return true;
|
|
507
|
+
const key = this.keyOf(targets.directory);
|
|
508
|
+
const at = (target) => [FILES_DIRECTORY, key, path.relative(targets.directory, target).split(path.sep).join('/')].filter(Boolean).join('/');
|
|
509
|
+
for (const script of targets.scripts)
|
|
510
|
+
this.scripts.set(at(script), script);
|
|
511
|
+
for (const data of targets.files)
|
|
512
|
+
this.files.set(at(data), data);
|
|
513
|
+
for (const root of targets.roots)
|
|
514
|
+
this.roots.add(root);
|
|
515
|
+
const relative = `\`${targets.rest.map((part) => 'text' in part ? templateText(part.text) : `\${${part.expression}}`).join('')}\``;
|
|
516
|
+
const text = found.kind === 'url' ? `new URL(${relative}, require.lakutataFilesURL(${JSON.stringify(key)}))` :
|
|
517
|
+
found.kind === 'url-text' ? `(require.lakutataFilesURL(${JSON.stringify(key)}) + ${relative})` :
|
|
518
|
+
`require("node:path").join(require.lakutataFiles(${JSON.stringify(key)}), ${relative})`;
|
|
519
|
+
followed.add(node);
|
|
520
|
+
edits.push({ start: node.start, end: node.end, text: text });
|
|
521
|
+
return false;
|
|
522
|
+
});
|
|
523
|
+
//The workers of node whose scripts are not followed
|
|
524
|
+
walk(program, (node) => {
|
|
525
|
+
if (node.type !== 'NewExpression' || !node.arguments.length)
|
|
526
|
+
return;
|
|
527
|
+
const callee = node.callee;
|
|
528
|
+
const isWorker = callee.type === 'Identifier' ? constructors.has(callee.name) :
|
|
529
|
+
calleeName(callee) === 'Worker' && callee.type === 'MemberExpression' && ((callee.object.type === 'Identifier' && namespaces.has(callee.object.name)) || isRequireOf(callee.object, WORKER_THREADS));
|
|
530
|
+
if (!isWorker)
|
|
531
|
+
return;
|
|
532
|
+
//A worker running a source (eval: true)
|
|
533
|
+
const options = node.arguments[1];
|
|
534
|
+
if (options?.type === 'ObjectExpression' && options.properties.some((property) => property.key?.name === 'eval' && property.value?.value === true))
|
|
535
|
+
return;
|
|
536
|
+
const argument = node.arguments[0];
|
|
537
|
+
const value = argument.type === 'Identifier' && constants.get(argument.name) ? constants.get(argument.name) :
|
|
538
|
+
argument.type === 'MemberExpression' && argument.object.type === 'ThisExpression' && !argument.computed && members.get(argument.property.name) ? members.get(argument.property.name) : argument;
|
|
539
|
+
if (followed.has(value))
|
|
540
|
+
return;
|
|
541
|
+
//As written: the warnings name it
|
|
542
|
+
const written = source.slice(argument.start, argument.end);
|
|
543
|
+
const line = node.loc.start.line;
|
|
544
|
+
if (scriptPathOf(value, source, file))
|
|
545
|
+
return this.warn(file, line, `the script of the worker (${written}) is not found: a .js or .cjs file from the module, built before the bundle`);
|
|
546
|
+
if (value.type === 'Literal' && typeof value.value === 'string')
|
|
547
|
+
return this.warn(file, line, `the script of the worker (${value.value}) is a path from the current directory of the process, not the bundle: give it from the directory of the module, path.join(__dirname, ${JSON.stringify(value.value)}) or new URL(${JSON.stringify(value.value)}, import.meta.url)`);
|
|
548
|
+
this.warn(file, line, `the script of the worker cannot be followed (${written}): give its path from the directory of the module (path.join(__dirname, 'worker.js'), new URL('./worker.js', import.meta.url)), or ship it as a resource (lakutata.resources)`);
|
|
549
|
+
});
|
|
550
|
+
if (!edits.length)
|
|
551
|
+
return undefined;
|
|
552
|
+
let result = source;
|
|
553
|
+
for (const edit of edits.sort((a, b) => b.start - a.start))
|
|
554
|
+
result = result.slice(0, edit.start) + edit.text + result.slice(edit.end);
|
|
555
|
+
return result;
|
|
556
|
+
}
|
|
557
|
+
}
|
package/dist/esm/lib/Native.js
CHANGED
|
@@ -4,6 +4,13 @@ import { mkdir, readFile, rm, symlink, writeFile } from 'node:fs/promises';
|
|
|
4
4
|
import { createRequire, isBuiltin } from 'node:module';
|
|
5
5
|
import path from 'node:path';
|
|
6
6
|
import { gzipSync } from 'node:zlib';
|
|
7
|
+
/**
|
|
8
|
+
* The content of a file of the native tree
|
|
9
|
+
* @param file
|
|
10
|
+
*/
|
|
11
|
+
function contentOf(file) {
|
|
12
|
+
return file.content ?? readFileSync(file.source);
|
|
13
|
+
}
|
|
7
14
|
/**
|
|
8
15
|
* The shared libraries: the addons, and the libraries they are linked with (sharp and libvips)
|
|
9
16
|
*/
|
|
@@ -227,8 +234,10 @@ export class NativeCollector {
|
|
|
227
234
|
}
|
|
228
235
|
/**
|
|
229
236
|
* The plugin of the bundle: the native packages are external, the addons of the application are loaded from the tree
|
|
237
|
+
* @param transforms the transformations of the sources of the modules (the paths of the workers), giving their new
|
|
238
|
+
* source or undefined
|
|
230
239
|
*/
|
|
231
|
-
plugin() {
|
|
240
|
+
plugin(transforms = []) {
|
|
232
241
|
return {
|
|
233
242
|
name: 'lakutata-native',
|
|
234
243
|
setup: (build) => {
|
|
@@ -263,11 +272,21 @@ export class NativeCollector {
|
|
|
263
272
|
});
|
|
264
273
|
//The requires of the ES modules are the one of the bundle: esbuild follows what they load (the addons)
|
|
265
274
|
build.onLoad({ filter: /\.(c|m)?js$/ }, async (args) => {
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
275
|
+
let source = await readFile(args.path, 'utf8');
|
|
276
|
+
let changed = false;
|
|
277
|
+
if (MODULE_REQUIRE.test(source)) {
|
|
278
|
+
MODULE_REQUIRE.lastIndex = 0;
|
|
279
|
+
source = source.replace(MODULE_REQUIRE, 'require');
|
|
280
|
+
changed = true;
|
|
281
|
+
}
|
|
282
|
+
for (const transform of transforms) {
|
|
283
|
+
const transformed = transform(source, args.path);
|
|
284
|
+
if (transformed === undefined)
|
|
285
|
+
continue;
|
|
286
|
+
source = transformed;
|
|
287
|
+
changed = true;
|
|
288
|
+
}
|
|
289
|
+
return changed ? { contents: source, loader: 'js', resolveDir: path.dirname(args.path) } : undefined;
|
|
271
290
|
});
|
|
272
291
|
//The addons of the application are loaded by the require of the bundle, from the extracted tree
|
|
273
292
|
build.onLoad({ filter: /.*/, namespace: 'lakutata-native' }, (args) => ({
|
|
@@ -487,7 +506,7 @@ export class NativeCollector {
|
|
|
487
506
|
export function hashNativeTree(files, links) {
|
|
488
507
|
const hash = createHash('sha256');
|
|
489
508
|
for (const file of files)
|
|
490
|
-
hash.update(`${file.path}\0${file.mode}\0`).update(
|
|
509
|
+
hash.update(`${file.path}\0${file.mode}\0`).update(contentOf(file));
|
|
491
510
|
for (const link of links)
|
|
492
511
|
hash.update(`${link.path}\0${link.target}\0`);
|
|
493
512
|
return hash.digest('hex').slice(0, 16);
|
|
@@ -502,7 +521,7 @@ export async function writeNativeTree(tree, directory) {
|
|
|
502
521
|
for (const file of tree.files) {
|
|
503
522
|
const target = path.join(directory, ...file.path.split('/'));
|
|
504
523
|
await mkdir(path.dirname(target), { recursive: true });
|
|
505
|
-
await writeFile(target,
|
|
524
|
+
await writeFile(target, contentOf(file), { mode: file.mode });
|
|
506
525
|
}
|
|
507
526
|
for (const link of tree.links) {
|
|
508
527
|
const at = path.join(directory, ...link.path.split('/'));
|
|
@@ -524,7 +543,7 @@ export function packNativeTree(tree) {
|
|
|
524
543
|
const files = [];
|
|
525
544
|
let offset = 0;
|
|
526
545
|
for (const file of tree.files) {
|
|
527
|
-
const content =
|
|
546
|
+
const content = contentOf(file);
|
|
528
547
|
files.push({ path: file.path, offset: offset, size: content.length, mode: file.mode });
|
|
529
548
|
chunks.push(content);
|
|
530
549
|
offset += content.length;
|
|
@@ -97,7 +97,10 @@ export function lakutataNativeRequire(nodeRequire, config) {
|
|
|
97
97
|
const locate = () => {
|
|
98
98
|
if (root)
|
|
99
99
|
return root;
|
|
100
|
-
if (config.
|
|
100
|
+
if (config.root !== undefined) {
|
|
101
|
+
root = path.resolve(__dirname, config.root);
|
|
102
|
+
}
|
|
103
|
+
else if (config.embedded) {
|
|
101
104
|
try {
|
|
102
105
|
root = extract(cacheDirectory());
|
|
103
106
|
}
|
|
@@ -121,7 +124,9 @@ export function lakutataNativeRequire(nodeRequire, config) {
|
|
|
121
124
|
return modules.has(id.startsWith('@') ? parts.slice(0, 2).join('/') : parts[0]);
|
|
122
125
|
};
|
|
123
126
|
const tree = () => treeRequire ??= createRequire(path.join(locate(), 'index.js'));
|
|
124
|
-
|
|
127
|
+
//The files loaded by their paths (the scripts of the tree): the require of an executable of Node.js loads the modules of
|
|
128
|
+
//node only
|
|
129
|
+
const require = ((id) => fromTree(id) ? tree()(id) : path.isAbsolute(id) ? createRequire(id)(id) : nodeRequire(id));
|
|
125
130
|
require.resolve = ((id, options) => fromTree(id) ? tree().resolve(id, options) : nodeRequire.resolve(id, options));
|
|
126
131
|
require.cache = nodeRequire.cache;
|
|
127
132
|
require.main = nodeRequire.main;
|
|
@@ -132,6 +137,28 @@ export function lakutataNativeRequire(nodeRequire, config) {
|
|
|
132
137
|
process.dlopen(addon, path.join(locate(), ...key.split('/')));
|
|
133
138
|
return addon.exports;
|
|
134
139
|
};
|
|
140
|
+
//The files the application reads by their paths, in the tree (FILES_DIRECTORY): the directory of a key, its URL
|
|
141
|
+
const files = (key) => path.join(locate(), '.lakutata', 'files', ...key.split('/'));
|
|
142
|
+
require.lakutataFiles = files;
|
|
143
|
+
require.lakutataFilesURL = (key) => nodeRequire('node:url').pathToFileURL(files(key) + path.sep).href;
|
|
144
|
+
//An executable started to run a script of its tree (fork(), spawn(process.execPath, [script]): process.execPath is the
|
|
145
|
+
//executable): it runs the script as node runs it, the application not started
|
|
146
|
+
require.lakutataScript = () => {
|
|
147
|
+
if (!config.embedded || config.root !== undefined)
|
|
148
|
+
return false;
|
|
149
|
+
const scripts = path.join(locate(), '.lakutata', 'files') + path.sep;
|
|
150
|
+
const at = [1, 2].find((index) => !!process.argv[index] && path.resolve(process.argv[index]).startsWith(scripts)) ?? -1;
|
|
151
|
+
if (at < 0)
|
|
152
|
+
return false;
|
|
153
|
+
const script = path.resolve(process.argv[at]);
|
|
154
|
+
process.argv.splice(1, at - 1);
|
|
155
|
+
//The main module of node (require.main); Bun loads it by a require (its Module._load does nothing)
|
|
156
|
+
if (process.versions.bun)
|
|
157
|
+
createRequire(script)(script);
|
|
158
|
+
else
|
|
159
|
+
nodeRequire('node:module')._load(script, null, true);
|
|
160
|
+
return true;
|
|
161
|
+
};
|
|
135
162
|
//The resources of an executable are extracted with the tree (PACKAGED, EMBEDDED_RESOURCES)
|
|
136
163
|
if (config.embedded)
|
|
137
164
|
globalThis[Symbol.for('lakutata.packaged')] = { resources: () => path.join(locate(), '.lakutata', 'resources') };
|
|
@@ -236,6 +236,7 @@ export async function buildElectronInstaller(project, manifest, config, options)
|
|
|
236
236
|
outfile: path.join(application, 'main.cjs'),
|
|
237
237
|
directories: { native: path.join(work, 'main.native'), resources: path.join(work, 'main.resources') }
|
|
238
238
|
});
|
|
239
|
+
warnings.push(...bundle.warnings);
|
|
239
240
|
const maintainer = platform === 'linux' ? packageMaintainer(author, name, warnings) : author ?? name;
|
|
240
241
|
//The packages of fpm need a homepage
|
|
241
242
|
let homepage = config.homepage ?? (typeof manifest.homepage === 'string' ? manifest.homepage : undefined);
|
|
@@ -354,6 +355,8 @@ export async function buildElectronInstaller(project, manifest, config, options)
|
|
|
354
355
|
platform: `${process.platform}-${process.arch}`,
|
|
355
356
|
size: bundle.size,
|
|
356
357
|
native: bundle.native?.modules ?? [],
|
|
358
|
+
workers: bundle.workers,
|
|
359
|
+
files: bundle.files,
|
|
357
360
|
resources: bundle.resources,
|
|
358
361
|
runtime: `Electron v${electronVersion}`,
|
|
359
362
|
warnings: []
|
|
@@ -70,6 +70,20 @@ export type BundleResult = {
|
|
|
70
70
|
*/
|
|
71
71
|
resources: string[];
|
|
72
72
|
resourcesDirectory?: string;
|
|
73
|
+
/**
|
|
74
|
+
* The scripts of the workers of the application (their files, from the project), bundled into the native tree
|
|
75
|
+
*/
|
|
76
|
+
workers: string[];
|
|
77
|
+
/**
|
|
78
|
+
* The other files and the directories of data the application reads by their paths (from the project), copied into
|
|
79
|
+
* the native tree
|
|
80
|
+
*/
|
|
81
|
+
files: string[];
|
|
82
|
+
/**
|
|
83
|
+
* What the bundle cannot follow: the scripts of the workers given by paths known at run time only, the directories
|
|
84
|
+
* which cannot be shipped (holding the modules, holding code), the scripts bundled twice
|
|
85
|
+
*/
|
|
86
|
+
warnings: string[];
|
|
73
87
|
};
|
|
74
88
|
/**
|
|
75
89
|
* The name of the program of a package: its first bin, or the name of the package without its scope
|
|
@@ -23,6 +23,15 @@ export type ExecutableResult = {
|
|
|
23
23
|
* The packages of native addons embedded, extracted at the first run, or next to the executable
|
|
24
24
|
*/
|
|
25
25
|
native: string[];
|
|
26
|
+
/**
|
|
27
|
+
* The scripts of the workers of the application (their files, from the project), with the native addons
|
|
28
|
+
*/
|
|
29
|
+
workers: string[];
|
|
30
|
+
/**
|
|
31
|
+
* The other files and the directories of data the application reads by their paths (from the project), with the
|
|
32
|
+
* native addons
|
|
33
|
+
*/
|
|
34
|
+
files: string[];
|
|
26
35
|
/**
|
|
27
36
|
* The resources of the application (their paths in the resources)
|
|
28
37
|
*/
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The directory of the files the application reads by their paths in the native tree: the scripts it runs on their own
|
|
3
|
+
* (bundled), the other files and the directories of data (copied), their directories as in the project
|
|
4
|
+
*/
|
|
5
|
+
export declare const FILES_DIRECTORY: string;
|
|
6
|
+
/**
|
|
7
|
+
* A piece of a path to a script: a text, or an expression known at run time only (the source of the expression)
|
|
8
|
+
*/
|
|
9
|
+
type Part = {
|
|
10
|
+
text: string;
|
|
11
|
+
} | {
|
|
12
|
+
expression: string;
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* A path to a script given from a module: from its directory (path.join, path.resolve, a concatenation or a template of
|
|
16
|
+
* __dirname), a URL relative to its file (new URL('./x.js', import.meta.url)), a URL text from its file
|
|
17
|
+
* (import.meta.url + '/../x.js'), or a file resolved when the application is bundled (require.resolve)
|
|
18
|
+
*/
|
|
19
|
+
type ScriptPath = {
|
|
20
|
+
/**
|
|
21
|
+
* What the expression gives: a path, a URL, the text of a URL
|
|
22
|
+
*/
|
|
23
|
+
kind: 'path' | 'url' | 'url-text';
|
|
24
|
+
parts: Part[];
|
|
25
|
+
/**
|
|
26
|
+
* The path the parts are joined to, the directory of the module by default
|
|
27
|
+
*/
|
|
28
|
+
base?: string;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* What a path from a module gives: the directory of its pattern, the parts from it, the scripts to bundle and the files
|
|
32
|
+
* to copy (the files of its directories of data), or why it cannot be shipped
|
|
33
|
+
*/
|
|
34
|
+
type Targets = {
|
|
35
|
+
directory: string;
|
|
36
|
+
rest: Part[];
|
|
37
|
+
scripts: string[];
|
|
38
|
+
files: string[];
|
|
39
|
+
/**
|
|
40
|
+
* The files and the directories it names (the roots of the files)
|
|
41
|
+
*/
|
|
42
|
+
roots: string[];
|
|
43
|
+
refused?: 'ancestor' | 'code';
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* The scripts an application runs on its own, by their paths, which the bundler does not follow: the scripts of its
|
|
47
|
+
* workers (new Worker(), the pools of workers given a file), of its child processes (fork()), the modules it loads by
|
|
48
|
+
* their paths (require(path.join(__dirname, 'x.js'))), in its modules and in the ones of its packages. A path to a script
|
|
49
|
+
* given from its module, wherever it is written, is found when the module is bundled: the script is bundled on its own
|
|
50
|
+
* into the native tree (FILES_DIRECTORY, the directories mirrored), and the expression rewritten to it. A path known at
|
|
51
|
+
* run time only in part (a template, a concatenation, an argument of path.join: `${name}.js`) gives every script of its
|
|
52
|
+
* directory its pattern matches, the expression of the part kept. The scripts of the workers of node (worker_threads)
|
|
53
|
+
* whose paths cannot be followed are warned of, in the modules of the application (the packages start workers from the
|
|
54
|
+
* paths their users give)
|
|
55
|
+
*/
|
|
56
|
+
export declare class FileCollector {
|
|
57
|
+
/**
|
|
58
|
+
* The scripts: their paths in the native tree, and their files
|
|
59
|
+
*/
|
|
60
|
+
readonly scripts: Map<string, string>;
|
|
61
|
+
/**
|
|
62
|
+
* The other files: their paths in the native tree, and their files
|
|
63
|
+
*/
|
|
64
|
+
readonly files: Map<string, string>;
|
|
65
|
+
/**
|
|
66
|
+
* The files and the directories of data the paths name
|
|
67
|
+
*/
|
|
68
|
+
readonly roots: Set<string>;
|
|
69
|
+
readonly warnings: string[];
|
|
70
|
+
protected readonly project: string;
|
|
71
|
+
/**
|
|
72
|
+
* @param project The directory of the project: the scripts are named from it (as the modules the bundler gives, its
|
|
73
|
+
* real path)
|
|
74
|
+
*/
|
|
75
|
+
constructor(project: string);
|
|
76
|
+
/**
|
|
77
|
+
* The key of a directory in the tree ("/" separated): its path from the project; outside of the project (the
|
|
78
|
+
* packages of a workspace, linked), its path from the node_modules holding it, else a hash of its path (the paths of
|
|
79
|
+
* the machine bundling it are not shipped)
|
|
80
|
+
* @param directory
|
|
81
|
+
*/
|
|
82
|
+
protected keyOf(directory: string): string;
|
|
83
|
+
/**
|
|
84
|
+
* The files of the scripts
|
|
85
|
+
*/
|
|
86
|
+
scriptPaths(): Set<string>;
|
|
87
|
+
/**
|
|
88
|
+
* The name of a script: its path from the project ("/" separated), or from the node_modules holding it outside of the
|
|
89
|
+
* project (the packages of a workspace)
|
|
90
|
+
* @param script
|
|
91
|
+
*/
|
|
92
|
+
nameOf(script: string): string;
|
|
93
|
+
/**
|
|
94
|
+
* What a path gives: the file or the directory, or the ones its pattern matches (an expression matches a part of a
|
|
95
|
+
* name); the directory they are named from, the parts of their paths from it
|
|
96
|
+
* @param found
|
|
97
|
+
* @param file the module
|
|
98
|
+
*/
|
|
99
|
+
protected targetsOf(found: ScriptPath, file: string): Targets;
|
|
100
|
+
/**
|
|
101
|
+
* Warn of a worker in a module of the application
|
|
102
|
+
* @param file
|
|
103
|
+
* @param line
|
|
104
|
+
* @param message
|
|
105
|
+
*/
|
|
106
|
+
protected warn(file: string, line: number, message: string): void;
|
|
107
|
+
/**
|
|
108
|
+
* Warn of the scripts bundled with the application and run on their own too: two copies of their modules
|
|
109
|
+
* @param bundled the files of the bundle of the application (their real paths)
|
|
110
|
+
*/
|
|
111
|
+
warnBundled(bundled: Set<string>): void;
|
|
112
|
+
/**
|
|
113
|
+
* The source of a module with its paths to scripts rewritten to the scripts in the native tree; undefined when it
|
|
114
|
+
* gives none
|
|
115
|
+
* @param source
|
|
116
|
+
* @param file
|
|
117
|
+
*/
|
|
118
|
+
transform(source: string, file: string): string | undefined;
|
|
119
|
+
}
|
|
120
|
+
export {};
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import type { Plugin, PluginBuild } from 'esbuild';
|
|
2
2
|
/**
|
|
3
|
-
* A file of the native tree: its path in the tree ("/" separated) and where it is read
|
|
3
|
+
* A file of the native tree: its path in the tree ("/" separated) and where it is read, or its content (a script of a
|
|
4
|
+
* worker, bundled)
|
|
4
5
|
*/
|
|
5
6
|
export type NativeFile = {
|
|
6
7
|
path: string;
|
|
7
8
|
source: string;
|
|
8
9
|
mode: number;
|
|
10
|
+
content?: Buffer;
|
|
9
11
|
};
|
|
10
12
|
/**
|
|
11
13
|
* A link of the native tree to a directory of the tree, as the links of the node_modules (pnpm, the workspaces)
|
|
@@ -136,8 +138,10 @@ export declare class NativeCollector {
|
|
|
136
138
|
}): Promise<string | undefined>;
|
|
137
139
|
/**
|
|
138
140
|
* The plugin of the bundle: the native packages are external, the addons of the application are loaded from the tree
|
|
141
|
+
* @param transforms the transformations of the sources of the modules (the paths of the workers), giving their new
|
|
142
|
+
* source or undefined
|
|
139
143
|
*/
|
|
140
|
-
plugin(): Plugin;
|
|
144
|
+
plugin(transforms?: ((source: string, file: string) => string | undefined)[]): Plugin;
|
|
141
145
|
/**
|
|
142
146
|
* The packages a native package loads at run time: the JavaScript it requires (followed by esbuild), the packages of
|
|
143
147
|
* shared libraries it depends on (the libraries linked with its addons), and the native packages it loads
|
|
@@ -24,6 +24,11 @@ export type NativeRuntimeConfig = {
|
|
|
24
24
|
* in its .lakutata/resources), or a directory next to the bundle
|
|
25
25
|
*/
|
|
26
26
|
embedded: boolean;
|
|
27
|
+
/**
|
|
28
|
+
* The tree from the directory of the bundle, for the scripts of the workers, which run from the tree (its
|
|
29
|
+
* .lakutata/workers): the tree is not looked for (nor extracted)
|
|
30
|
+
*/
|
|
31
|
+
root?: string;
|
|
27
32
|
};
|
|
28
33
|
/**
|
|
29
34
|
* The global symbol of the packed applications giving the directory of their resources (resourcesPath of lakutata)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lakutata/builder",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.5",
|
|
4
4
|
"description": "The builder of the applications of the lakutata framework: their bundles, their executables (Node.js, Bun) with their native addons and their resources.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/esm/index.js",
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
"test:installer:windows": "npm run compile && node --test dist/esm/tests/InstallerWindows.spec.js"
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
|
+
"acorn": "^8.19.0",
|
|
26
27
|
"esbuild": "^0.28.2",
|
|
27
28
|
"postject": "^1.0.0-alpha.6",
|
|
28
29
|
"resedit": "^1.7.2"
|
|
@@ -47,5 +48,10 @@
|
|
|
47
48
|
"*.md",
|
|
48
49
|
"LICENSE",
|
|
49
50
|
"package.json"
|
|
50
|
-
]
|
|
51
|
+
],
|
|
52
|
+
"devDependencies": {
|
|
53
|
+
"piscina": "^5.3.2",
|
|
54
|
+
"tinypool": "^2.2.0",
|
|
55
|
+
"workerpool": "^10.0.3"
|
|
56
|
+
}
|
|
51
57
|
}
|