@lark-apaas/fullstack-cli 1.1.64 → 1.1.65-alpha.20260902125800
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +2625 -49
- package/package.json +4 -1
- package/templates/scripts/build.sh +0 -12
- package/templates/scripts/dev.js +700 -54
- package/templates/scripts/server-handoff-runtime.mjs +1904 -0
package/templates/scripts/dev.js
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
'use strict';
|
|
3
3
|
|
|
4
4
|
const fs = require('fs');
|
|
5
|
+
const http = require('http');
|
|
6
|
+
const net = require('net');
|
|
5
7
|
const path = require('path');
|
|
6
8
|
const { spawn, execSync } = require('child_process');
|
|
7
9
|
const readline = require('readline');
|
|
@@ -31,13 +33,39 @@ loadEnv();
|
|
|
31
33
|
|
|
32
34
|
// ── Configuration ─────────────────────────────────────────────────────────────
|
|
33
35
|
const LOG_DIR = process.env.LOG_DIR || 'logs';
|
|
34
|
-
const MAX_RESTART_COUNT =
|
|
35
|
-
|
|
36
|
-
|
|
36
|
+
const MAX_RESTART_COUNT =
|
|
37
|
+
process.env.MAX_RESTART_COUNT != null && process.env.MAX_RESTART_COUNT !== ''
|
|
38
|
+
? parseInt(process.env.MAX_RESTART_COUNT, 10)
|
|
39
|
+
: Infinity;
|
|
37
40
|
const RESTART_DELAY = parseInt(process.env.RESTART_DELAY, 10) || 2;
|
|
38
41
|
const MAX_DELAY = 8;
|
|
39
|
-
const
|
|
42
|
+
const LEGACY_SERVER_PORT = process.env.SERVER_PORT || '3000';
|
|
43
|
+
const PLATFORM_HANDOFF_PUBLIC_PORT = 3000;
|
|
44
|
+
const MAX_HANDOFF_READINESS_RESPONSE_BYTES = 64 * 1024;
|
|
40
45
|
const CLIENT_DEV_PORT = process.env.CLIENT_DEV_PORT || '8080';
|
|
46
|
+
// This is an ownership receipt, not the raw TCC flag. The sandbox entrypoint
|
|
47
|
+
// sets it only after the handoff runtime has bound the public port and
|
|
48
|
+
// published its owner marker. If that handshake fails, the legacy path below
|
|
49
|
+
// remains the sole backend owner.
|
|
50
|
+
const BACKEND_OWNED_BY_HANDOFF =
|
|
51
|
+
process.env.MIAODA_NEST_BACKEND_OWNER === 'handoff';
|
|
52
|
+
const HANDOFF_CONTROL_ROOT = '/tmp/miaoda-cold-start';
|
|
53
|
+
const HANDOFF_DECISION_FILE = path.join(HANDOFF_CONTROL_ROOT, 'decision.json');
|
|
54
|
+
const HANDOFF_OWNER_FILE = path.join(
|
|
55
|
+
HANDOFF_CONTROL_ROOT,
|
|
56
|
+
'runtime-owner.json'
|
|
57
|
+
);
|
|
58
|
+
const HANDOFF_PROCESS_REGISTRY_FILE = path.join(
|
|
59
|
+
HANDOFF_CONTROL_ROOT,
|
|
60
|
+
'runtime-processes.json'
|
|
61
|
+
);
|
|
62
|
+
const HANDOFF_DECISION = BACKEND_OWNED_BY_HANDOFF
|
|
63
|
+
? readStrictColdStartDecision()
|
|
64
|
+
: null;
|
|
65
|
+
const HANDOFF_BOOT_ID = HANDOFF_DECISION?.bootId || '';
|
|
66
|
+
const HANDOFF_RESTORE_EPOCH_MS = HANDOFF_DECISION?.restoreEpochMs || 0;
|
|
67
|
+
const PUBLIC_SERVER_PORT =
|
|
68
|
+
HANDOFF_DECISION?.publicPort || Number(LEGACY_SERVER_PORT);
|
|
41
69
|
|
|
42
70
|
fs.mkdirSync(LOG_DIR, { recursive: true });
|
|
43
71
|
|
|
@@ -50,11 +78,16 @@ const devLogFd = fs.openSync(devLogPath, 'a');
|
|
|
50
78
|
function timestamp() {
|
|
51
79
|
const now = new Date();
|
|
52
80
|
return (
|
|
53
|
-
now.getFullYear() +
|
|
54
|
-
|
|
55
|
-
String(now.
|
|
56
|
-
|
|
57
|
-
String(now.
|
|
81
|
+
now.getFullYear() +
|
|
82
|
+
'-' +
|
|
83
|
+
String(now.getMonth() + 1).padStart(2, '0') +
|
|
84
|
+
'-' +
|
|
85
|
+
String(now.getDate()).padStart(2, '0') +
|
|
86
|
+
' ' +
|
|
87
|
+
String(now.getHours()).padStart(2, '0') +
|
|
88
|
+
':' +
|
|
89
|
+
String(now.getMinutes()).padStart(2, '0') +
|
|
90
|
+
':' +
|
|
58
91
|
String(now.getSeconds()).padStart(2, '0')
|
|
59
92
|
);
|
|
60
93
|
}
|
|
@@ -63,26 +96,55 @@ function timestamp() {
|
|
|
63
96
|
// Each pending fs.write retains ~1.5 KB (msg + libuv/V8 wrappers); 1000 ≈ ~1.5 MB ceiling.
|
|
64
97
|
let _stdoutInFlight = 0;
|
|
65
98
|
const STDOUT_MAX_INFLIGHT = 1000;
|
|
99
|
+
const reportedBestEffortFailures = new Set();
|
|
100
|
+
|
|
101
|
+
function reportBestEffortFailure(operation, error) {
|
|
102
|
+
if (reportedBestEffortFailures.has(operation)) return;
|
|
103
|
+
reportedBestEffortFailures.add(operation);
|
|
104
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
105
|
+
try {
|
|
106
|
+
fs.write(2, `[dev.js] ${operation}: ${detail}\n`, () => {});
|
|
107
|
+
} catch {
|
|
108
|
+
// fd 2 is the last-resort diagnostic sink; there is no safe recursive
|
|
109
|
+
// fallback if submitting this best-effort write itself fails.
|
|
110
|
+
}
|
|
111
|
+
}
|
|
66
112
|
|
|
67
113
|
/** Write to dev.std.log (sync, guaranteed) + mirror to terminal (async, non-blocking) */
|
|
68
114
|
function writeOutput(msg) {
|
|
69
115
|
// File first and synchronously — read-logs reads this file; it must never be gated
|
|
70
116
|
// by the terminal consumer.
|
|
71
|
-
try {
|
|
117
|
+
try {
|
|
118
|
+
fs.writeSync(devStdLogFd, msg);
|
|
119
|
+
} catch (error) {
|
|
120
|
+
reportBestEffortFailure('write dev.std.log failed', error);
|
|
121
|
+
}
|
|
72
122
|
// stdout mirror via async fs.write: if process.stdout is a pty/pipe whose consumer
|
|
73
123
|
// stalls, the block happens on the libuv threadpool, NOT the event loop — so the
|
|
74
124
|
// synchronous log-FILE writes above (and subsequent readline callbacks) keep running.
|
|
75
125
|
// Drop overflow when too many writes are already pending (best-effort mirror).
|
|
76
|
-
if (_stdoutInFlight >= STDOUT_MAX_INFLIGHT) {
|
|
126
|
+
if (_stdoutInFlight >= STDOUT_MAX_INFLIGHT) {
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
77
129
|
_stdoutInFlight++;
|
|
78
|
-
try {
|
|
130
|
+
try {
|
|
131
|
+
fs.write(1, msg, () => {
|
|
132
|
+
_stdoutInFlight--;
|
|
133
|
+
});
|
|
134
|
+
} catch {
|
|
135
|
+
_stdoutInFlight--;
|
|
136
|
+
}
|
|
79
137
|
}
|
|
80
138
|
|
|
81
139
|
/** Structured event log → terminal + dev.std.log + dev.log */
|
|
82
140
|
function logEvent(level, name, message) {
|
|
83
141
|
const msg = `[${timestamp()}] [${level}] [${name}] ${message}\n`;
|
|
84
142
|
writeOutput(msg);
|
|
85
|
-
try {
|
|
143
|
+
try {
|
|
144
|
+
fs.writeSync(devLogFd, msg);
|
|
145
|
+
} catch (error) {
|
|
146
|
+
reportBestEffortFailure('write dev.log failed', error);
|
|
147
|
+
}
|
|
86
148
|
}
|
|
87
149
|
|
|
88
150
|
// ── Process group management ──────────────────────────────────────────────────
|
|
@@ -94,11 +156,20 @@ function killProcessGroup(pid, signal) {
|
|
|
94
156
|
|
|
95
157
|
function killOrphansByPort(port) {
|
|
96
158
|
try {
|
|
97
|
-
const pids = execSync(`lsof -ti :${port}`, {
|
|
159
|
+
const pids = execSync(`lsof -ti :${port}`, {
|
|
160
|
+
encoding: 'utf8',
|
|
161
|
+
timeout: 5000,
|
|
162
|
+
}).trim();
|
|
98
163
|
if (pids) {
|
|
99
164
|
const pidList = pids.split('\n').filter(Boolean);
|
|
100
165
|
for (const p of pidList) {
|
|
101
|
-
try {
|
|
166
|
+
try {
|
|
167
|
+
process.kill(parseInt(p, 10), 'SIGKILL');
|
|
168
|
+
} catch (error) {
|
|
169
|
+
if (error?.code !== 'ESRCH') {
|
|
170
|
+
reportBestEffortFailure('kill legacy port owner failed', error);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
102
173
|
}
|
|
103
174
|
return pidList;
|
|
104
175
|
}
|
|
@@ -111,18 +182,440 @@ let stopping = false;
|
|
|
111
182
|
const managedProcesses = []; // { name, pid, child }
|
|
112
183
|
|
|
113
184
|
function sleep(ms) {
|
|
114
|
-
return new Promise(
|
|
185
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function hasExactKeys(value, expected) {
|
|
189
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
|
190
|
+
const actual = Object.keys(value).sort();
|
|
191
|
+
const sortedExpected = [...expected].sort();
|
|
192
|
+
return (
|
|
193
|
+
actual.length === sortedExpected.length &&
|
|
194
|
+
actual.every((key, index) => key === sortedExpected[index])
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function readLinuxProcessIdentity(pid) {
|
|
199
|
+
if (process.platform !== 'linux' || !Number.isSafeInteger(pid) || pid <= 0)
|
|
200
|
+
return null;
|
|
201
|
+
try {
|
|
202
|
+
const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8');
|
|
203
|
+
const commandEnd = stat.lastIndexOf(')');
|
|
204
|
+
if (commandEnd < 0) return null;
|
|
205
|
+
const fields = stat
|
|
206
|
+
.slice(commandEnd + 1)
|
|
207
|
+
.trim()
|
|
208
|
+
.split(/\s+/);
|
|
209
|
+
if (!/^\d+$/.test(fields[19] || '') || !/^\d+$/.test(fields[2] || ''))
|
|
210
|
+
return null;
|
|
211
|
+
return {
|
|
212
|
+
state: fields[0],
|
|
213
|
+
processStartTicks: fields[19],
|
|
214
|
+
processGroupId: Number(fields[2]),
|
|
215
|
+
};
|
|
216
|
+
} catch {
|
|
217
|
+
return null;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function processIdentityIsLive(pid, expectedStartTicks) {
|
|
222
|
+
if (process.platform === 'linux') {
|
|
223
|
+
const identity = readLinuxProcessIdentity(pid);
|
|
224
|
+
return Boolean(
|
|
225
|
+
identity &&
|
|
226
|
+
identity.state !== 'Z' &&
|
|
227
|
+
identity.processStartTicks === expectedStartTicks
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
try {
|
|
231
|
+
process.kill(pid, 0);
|
|
232
|
+
return true;
|
|
233
|
+
} catch {
|
|
234
|
+
return false;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function readJsonNoFollow(file) {
|
|
239
|
+
try {
|
|
240
|
+
const parent = fs.lstatSync(path.dirname(file));
|
|
241
|
+
if (
|
|
242
|
+
parent.isSymbolicLink() ||
|
|
243
|
+
!parent.isDirectory() ||
|
|
244
|
+
(parent.mode & 0o777) !== 0o700 ||
|
|
245
|
+
(typeof process.getuid === 'function' && parent.uid !== process.getuid())
|
|
246
|
+
)
|
|
247
|
+
return null;
|
|
248
|
+
const before = fs.lstatSync(file);
|
|
249
|
+
if (
|
|
250
|
+
before.isSymbolicLink() ||
|
|
251
|
+
!before.isFile() ||
|
|
252
|
+
(before.mode & 0o777) !== 0o600 ||
|
|
253
|
+
(typeof process.getuid === 'function' && before.uid !== process.getuid())
|
|
254
|
+
)
|
|
255
|
+
return null;
|
|
256
|
+
const fd = fs.openSync(
|
|
257
|
+
file,
|
|
258
|
+
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)
|
|
259
|
+
);
|
|
260
|
+
try {
|
|
261
|
+
const opened = fs.fstatSync(fd);
|
|
262
|
+
if (
|
|
263
|
+
!opened.isFile() ||
|
|
264
|
+
before.dev !== opened.dev ||
|
|
265
|
+
before.ino !== opened.ino
|
|
266
|
+
)
|
|
267
|
+
return null;
|
|
268
|
+
return { payload: JSON.parse(fs.readFileSync(fd, 'utf8')), stat: opened };
|
|
269
|
+
} finally {
|
|
270
|
+
fs.closeSync(fd);
|
|
271
|
+
}
|
|
272
|
+
} catch {
|
|
273
|
+
return null;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function readStrictColdStartDecision() {
|
|
278
|
+
const decision = readJsonNoFollow(HANDOFF_DECISION_FILE)?.payload;
|
|
279
|
+
if (
|
|
280
|
+
!hasExactKeys(decision, [
|
|
281
|
+
'schemaVersion',
|
|
282
|
+
'appId',
|
|
283
|
+
'bootId',
|
|
284
|
+
'restoreEpochMs',
|
|
285
|
+
'publicPort',
|
|
286
|
+
'viteDepsCacheEnabled',
|
|
287
|
+
'nestBundleHandoffEnabled',
|
|
288
|
+
]) ||
|
|
289
|
+
decision.schemaVersion !== 1 ||
|
|
290
|
+
typeof decision.appId !== 'string' ||
|
|
291
|
+
decision.appId.length === 0 ||
|
|
292
|
+
typeof decision.bootId !== 'string' ||
|
|
293
|
+
!/^[a-zA-Z0-9._:-]{1,128}$/.test(decision.bootId) ||
|
|
294
|
+
!Number.isSafeInteger(decision.restoreEpochMs) ||
|
|
295
|
+
decision.restoreEpochMs <= 0 ||
|
|
296
|
+
decision.publicPort !== PLATFORM_HANDOFF_PUBLIC_PORT ||
|
|
297
|
+
typeof decision.viteDepsCacheEnabled !== 'boolean' ||
|
|
298
|
+
decision.nestBundleHandoffEnabled !== true
|
|
299
|
+
) {
|
|
300
|
+
return null;
|
|
301
|
+
}
|
|
302
|
+
return decision;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
const OWNER_KEYS = [
|
|
306
|
+
'schemaVersion',
|
|
307
|
+
'bootId',
|
|
308
|
+
'restoreEpochMs',
|
|
309
|
+
'generationId',
|
|
310
|
+
'pid',
|
|
311
|
+
'processStartTicks',
|
|
312
|
+
'publicPort',
|
|
313
|
+
'createdAtMs',
|
|
314
|
+
];
|
|
315
|
+
|
|
316
|
+
function readStrictHandoffOwner() {
|
|
317
|
+
const opened = readJsonNoFollow(HANDOFF_OWNER_FILE);
|
|
318
|
+
const owner = opened?.payload;
|
|
319
|
+
if (
|
|
320
|
+
!hasExactKeys(owner, OWNER_KEYS) ||
|
|
321
|
+
owner.schemaVersion !== 1 ||
|
|
322
|
+
owner.bootId !== HANDOFF_BOOT_ID ||
|
|
323
|
+
owner.restoreEpochMs !== HANDOFF_RESTORE_EPOCH_MS ||
|
|
324
|
+
!/^[a-f0-9]{64}$/.test(owner.generationId || '') ||
|
|
325
|
+
!Number.isSafeInteger(owner.pid) ||
|
|
326
|
+
owner.pid <= 0 ||
|
|
327
|
+
!/^\d+$/.test(owner.processStartTicks || '') ||
|
|
328
|
+
owner.publicPort !== PUBLIC_SERVER_PORT ||
|
|
329
|
+
!Number.isSafeInteger(owner.createdAtMs) ||
|
|
330
|
+
owner.createdAtMs < HANDOFF_RESTORE_EPOCH_MS ||
|
|
331
|
+
!processIdentityIsLive(owner.pid, owner.processStartTicks)
|
|
332
|
+
)
|
|
333
|
+
return null;
|
|
334
|
+
return owner;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
function publicServerReachable() {
|
|
338
|
+
return new Promise(resolve => {
|
|
339
|
+
let settled = false;
|
|
340
|
+
const finish = reachable => {
|
|
341
|
+
if (settled) return;
|
|
342
|
+
settled = true;
|
|
343
|
+
resolve(reachable);
|
|
344
|
+
};
|
|
345
|
+
const basePath =
|
|
346
|
+
process.env.CLIENT_BASE_PATH?.startsWith('/') &&
|
|
347
|
+
!process.env.CLIENT_BASE_PATH.startsWith('//')
|
|
348
|
+
? process.env.CLIENT_BASE_PATH.replace(/\/+$/, '')
|
|
349
|
+
: '';
|
|
350
|
+
const request = http.get(
|
|
351
|
+
{
|
|
352
|
+
host: '127.0.0.1',
|
|
353
|
+
port: PUBLIC_SERVER_PORT,
|
|
354
|
+
path: `${basePath}/__innerapi__/capability/list`,
|
|
355
|
+
},
|
|
356
|
+
response => {
|
|
357
|
+
if (response.statusCode !== 200) {
|
|
358
|
+
response.resume();
|
|
359
|
+
response.once('end', () => finish(false));
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
const chunks = [];
|
|
363
|
+
let responseBytes = 0;
|
|
364
|
+
response.on('data', chunk => {
|
|
365
|
+
if (settled) return;
|
|
366
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
367
|
+
responseBytes += buffer.length;
|
|
368
|
+
if (responseBytes > MAX_HANDOFF_READINESS_RESPONSE_BYTES) {
|
|
369
|
+
response.destroy();
|
|
370
|
+
finish(false);
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
chunks.push(buffer);
|
|
374
|
+
});
|
|
375
|
+
response.once('end', () => {
|
|
376
|
+
if (settled) return;
|
|
377
|
+
try {
|
|
378
|
+
const payload = JSON.parse(Buffer.concat(chunks).toString('utf8'));
|
|
379
|
+
finish(
|
|
380
|
+
payload?.status_code === '0' &&
|
|
381
|
+
payload.data &&
|
|
382
|
+
typeof payload.data === 'object' &&
|
|
383
|
+
!Array.isArray(payload.data) &&
|
|
384
|
+
Array.isArray(payload.data.capabilities)
|
|
385
|
+
);
|
|
386
|
+
} catch {
|
|
387
|
+
finish(false);
|
|
388
|
+
}
|
|
389
|
+
});
|
|
390
|
+
response.once('error', () => finish(false));
|
|
391
|
+
}
|
|
392
|
+
);
|
|
393
|
+
request.setTimeout(250, () => {
|
|
394
|
+
request.destroy();
|
|
395
|
+
finish(false);
|
|
396
|
+
});
|
|
397
|
+
request.once('error', () => finish(false));
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
function registryMatchesOwner(registry, owner) {
|
|
402
|
+
if (
|
|
403
|
+
!hasExactKeys(registry, [
|
|
404
|
+
'schemaVersion',
|
|
405
|
+
'bootId',
|
|
406
|
+
'restoreEpochMs',
|
|
407
|
+
'runtimePid',
|
|
408
|
+
'runtimeProcessStartTicks',
|
|
409
|
+
'updatedAtMs',
|
|
410
|
+
'processes',
|
|
411
|
+
]) ||
|
|
412
|
+
registry.schemaVersion !== 1 ||
|
|
413
|
+
registry.bootId !== HANDOFF_BOOT_ID ||
|
|
414
|
+
registry.restoreEpochMs !== HANDOFF_RESTORE_EPOCH_MS ||
|
|
415
|
+
registry.runtimePid !== owner.pid ||
|
|
416
|
+
registry.runtimeProcessStartTicks !== owner.processStartTicks ||
|
|
417
|
+
!Number.isSafeInteger(registry.updatedAtMs) ||
|
|
418
|
+
registry.updatedAtMs < HANDOFF_RESTORE_EPOCH_MS ||
|
|
419
|
+
!Array.isArray(registry.processes)
|
|
420
|
+
)
|
|
421
|
+
return false;
|
|
422
|
+
const roles = new Set();
|
|
423
|
+
return registry.processes.every(owned => {
|
|
424
|
+
const valid =
|
|
425
|
+
hasExactKeys(owned, ['role', 'pid', 'processStartTicks']) &&
|
|
426
|
+
['cache', 'source', 'action-plugin-init'].includes(owned.role) &&
|
|
427
|
+
!roles.has(owned.role) &&
|
|
428
|
+
Number.isSafeInteger(owned.pid) &&
|
|
429
|
+
owned.pid > 0 &&
|
|
430
|
+
/^\d+$/.test(owned.processStartTicks || '');
|
|
431
|
+
roles.add(owned.role);
|
|
432
|
+
return valid;
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
function ownedGroupIdentityMatches(owned) {
|
|
437
|
+
const leaderIdentity = readLinuxProcessIdentity(owned.pid);
|
|
438
|
+
if (
|
|
439
|
+
leaderIdentity &&
|
|
440
|
+
(leaderIdentity.processStartTicks !== owned.processStartTicks ||
|
|
441
|
+
leaderIdentity.processGroupId !== owned.pid)
|
|
442
|
+
) {
|
|
443
|
+
return false;
|
|
444
|
+
}
|
|
445
|
+
// Linux reserves a numeric PGID while any member of that group remains, so
|
|
446
|
+
// the dead leader's PID cannot be reused until the leaderless owned group is
|
|
447
|
+
// empty. Re-check both PID identity and group existence immediately before
|
|
448
|
+
// every signal; if a new leader appeared, fail closed above.
|
|
449
|
+
if (!leaderIdentity) {
|
|
450
|
+
try {
|
|
451
|
+
process.kill(owned.pid, 0);
|
|
452
|
+
return false;
|
|
453
|
+
} catch (error) {
|
|
454
|
+
if (error?.code !== 'ESRCH') return false;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
try {
|
|
458
|
+
for (const name of fs.readdirSync('/proc')) {
|
|
459
|
+
if (!/^\d+$/.test(name)) continue;
|
|
460
|
+
const member = readLinuxProcessIdentity(Number(name));
|
|
461
|
+
if (member?.processGroupId === owned.pid && member.state !== 'Z') {
|
|
462
|
+
const currentLeader = readLinuxProcessIdentity(owned.pid);
|
|
463
|
+
return Boolean(
|
|
464
|
+
!currentLeader ||
|
|
465
|
+
(currentLeader.processStartTicks === owned.processStartTicks &&
|
|
466
|
+
currentLeader.processGroupId === owned.pid)
|
|
467
|
+
);
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
} catch {
|
|
471
|
+
return false;
|
|
472
|
+
}
|
|
473
|
+
return false;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
function runtimeGroupIdentityMatches(owner) {
|
|
477
|
+
const identity = readLinuxProcessIdentity(owner.pid);
|
|
478
|
+
return Boolean(
|
|
479
|
+
identity &&
|
|
480
|
+
identity.state !== 'Z' &&
|
|
481
|
+
identity.processStartTicks === owner.processStartTicks &&
|
|
482
|
+
identity.processGroupId === owner.pid
|
|
483
|
+
);
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
async function terminateStrictRuntimeOwner(owner) {
|
|
487
|
+
if (process.platform !== 'linux' || !runtimeGroupIdentityMatches(owner))
|
|
488
|
+
return;
|
|
489
|
+
// Re-check pid birth identity and PGID immediately before every signal. A
|
|
490
|
+
// numeric pid from the marker is never sufficient authority after waiting.
|
|
491
|
+
if (runtimeGroupIdentityMatches(owner)) {
|
|
492
|
+
try {
|
|
493
|
+
process.kill(-owner.pid, 'SIGTERM');
|
|
494
|
+
} catch {}
|
|
495
|
+
}
|
|
496
|
+
const deadline = Date.now() + 2000;
|
|
497
|
+
while (Date.now() < deadline && runtimeGroupIdentityMatches(owner)) {
|
|
498
|
+
await sleep(25);
|
|
499
|
+
}
|
|
500
|
+
if (runtimeGroupIdentityMatches(owner)) {
|
|
501
|
+
try {
|
|
502
|
+
process.kill(-owner.pid, 'SIGKILL');
|
|
503
|
+
} catch {}
|
|
504
|
+
}
|
|
505
|
+
const killDeadline = Date.now() + 1000;
|
|
506
|
+
while (Date.now() < killDeadline && runtimeGroupIdentityMatches(owner)) {
|
|
507
|
+
await sleep(25);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
async function cleanupDeadRuntimeRegistry(owner) {
|
|
512
|
+
if (process.platform !== 'linux') return;
|
|
513
|
+
const opened = readJsonNoFollow(HANDOFF_PROCESS_REGISTRY_FILE);
|
|
514
|
+
const registry = opened?.payload;
|
|
515
|
+
if (!opened || !registryMatchesOwner(registry, owner)) return;
|
|
516
|
+
for (const owned of registry.processes) {
|
|
517
|
+
if (!ownedGroupIdentityMatches(owned)) continue;
|
|
518
|
+
try {
|
|
519
|
+
process.kill(-owned.pid, 'SIGTERM');
|
|
520
|
+
} catch {}
|
|
521
|
+
}
|
|
522
|
+
const deadline = Date.now() + 2000;
|
|
523
|
+
while (
|
|
524
|
+
Date.now() < deadline &&
|
|
525
|
+
registry.processes.some(ownedGroupIdentityMatches)
|
|
526
|
+
) {
|
|
527
|
+
await sleep(25);
|
|
528
|
+
}
|
|
529
|
+
for (const owned of registry.processes) {
|
|
530
|
+
if (!ownedGroupIdentityMatches(owned)) continue;
|
|
531
|
+
try {
|
|
532
|
+
process.kill(-owned.pid, 'SIGKILL');
|
|
533
|
+
} catch {}
|
|
534
|
+
}
|
|
535
|
+
const killDeadline = Date.now() + 1000;
|
|
536
|
+
while (
|
|
537
|
+
Date.now() < killDeadline &&
|
|
538
|
+
registry.processes.some(ownedGroupIdentityMatches)
|
|
539
|
+
) {
|
|
540
|
+
await sleep(25);
|
|
541
|
+
}
|
|
542
|
+
try {
|
|
543
|
+
const after = fs.lstatSync(HANDOFF_PROCESS_REGISTRY_FILE);
|
|
544
|
+
if (
|
|
545
|
+
after.dev === opened.stat.dev &&
|
|
546
|
+
after.ino === opened.stat.ino &&
|
|
547
|
+
!registry.processes.some(ownedGroupIdentityMatches)
|
|
548
|
+
) {
|
|
549
|
+
fs.unlinkSync(HANDOFF_PROCESS_REGISTRY_FILE);
|
|
550
|
+
}
|
|
551
|
+
} catch {}
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
function removeDeadOwnerMarker(owner) {
|
|
555
|
+
if (processIdentityIsLive(owner.pid, owner.processStartTicks)) return;
|
|
556
|
+
const opened = readJsonNoFollow(HANDOFF_OWNER_FILE);
|
|
557
|
+
const payload = opened?.payload;
|
|
558
|
+
if (
|
|
559
|
+
!opened ||
|
|
560
|
+
!hasExactKeys(payload, OWNER_KEYS) ||
|
|
561
|
+
payload.bootId !== owner.bootId ||
|
|
562
|
+
payload.restoreEpochMs !== owner.restoreEpochMs ||
|
|
563
|
+
payload.pid !== owner.pid ||
|
|
564
|
+
payload.processStartTicks !== owner.processStartTicks
|
|
565
|
+
)
|
|
566
|
+
return;
|
|
567
|
+
try {
|
|
568
|
+
const after = fs.lstatSync(HANDOFF_OWNER_FILE);
|
|
569
|
+
if (after.dev === opened.stat.dev && after.ino === opened.stat.ino) {
|
|
570
|
+
fs.unlinkSync(HANDOFF_OWNER_FILE);
|
|
571
|
+
}
|
|
572
|
+
} catch {}
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
function publicPortBindable() {
|
|
576
|
+
return new Promise(resolve => {
|
|
577
|
+
const server = net.createServer();
|
|
578
|
+
let settled = false;
|
|
579
|
+
const finish = bindable => {
|
|
580
|
+
if (settled) return;
|
|
581
|
+
settled = true;
|
|
582
|
+
server.close(() => resolve(bindable));
|
|
583
|
+
};
|
|
584
|
+
server.once('error', () => resolve(false));
|
|
585
|
+
server.listen(PUBLIC_SERVER_PORT, '127.0.0.1', () => finish(true));
|
|
586
|
+
});
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
async function waitForPublicPortRelease() {
|
|
590
|
+
while (!stopping) {
|
|
591
|
+
if (await publicPortBindable()) return true;
|
|
592
|
+
await sleep(50);
|
|
593
|
+
}
|
|
594
|
+
return false;
|
|
115
595
|
}
|
|
116
596
|
|
|
117
597
|
/**
|
|
118
598
|
* Start and supervise a process with auto-restart and log piping.
|
|
119
599
|
* Returns a promise that resolves when the process loop ends.
|
|
120
600
|
*/
|
|
121
|
-
function startProcess({
|
|
601
|
+
function startProcess({
|
|
602
|
+
name,
|
|
603
|
+
command,
|
|
604
|
+
args,
|
|
605
|
+
cleanupPort,
|
|
606
|
+
environment,
|
|
607
|
+
strictOwnership = false,
|
|
608
|
+
}) {
|
|
122
609
|
const logFilePath = path.join(LOG_DIR, `${name}.std.log`);
|
|
123
610
|
const logFd = fs.openSync(logFilePath, 'a');
|
|
124
611
|
|
|
125
|
-
const entry = {
|
|
612
|
+
const entry = {
|
|
613
|
+
name,
|
|
614
|
+
pid: null,
|
|
615
|
+
child: null,
|
|
616
|
+
processStartTicks: null,
|
|
617
|
+
strictOwnership,
|
|
618
|
+
};
|
|
126
619
|
managedProcesses.push(entry);
|
|
127
620
|
|
|
128
621
|
const run = async () => {
|
|
@@ -134,21 +627,41 @@ function startProcess({ name, command, args, cleanupPort }) {
|
|
|
134
627
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
135
628
|
shell: true,
|
|
136
629
|
cwd: PROJECT_ROOT,
|
|
137
|
-
env: { ...process.env },
|
|
630
|
+
env: { ...process.env, ...environment },
|
|
138
631
|
});
|
|
139
632
|
|
|
140
633
|
entry.pid = child.pid;
|
|
141
634
|
entry.child = child;
|
|
635
|
+
entry.processStartTicks = readLinuxProcessIdentity(
|
|
636
|
+
child.pid
|
|
637
|
+
)?.processStartTicks;
|
|
638
|
+
if (
|
|
639
|
+
strictOwnership &&
|
|
640
|
+
process.platform === 'linux' &&
|
|
641
|
+
!entry.processStartTicks
|
|
642
|
+
) {
|
|
643
|
+
child.kill('SIGKILL');
|
|
644
|
+
throw new Error(`Strict process identity unavailable: ${name}`);
|
|
645
|
+
}
|
|
142
646
|
|
|
143
647
|
const startTime = Date.now();
|
|
144
|
-
logEvent(
|
|
648
|
+
logEvent(
|
|
649
|
+
'INFO',
|
|
650
|
+
name,
|
|
651
|
+
`Process started (PGID: ${child.pid}): ${command} ${args.join(' ')}`
|
|
652
|
+
);
|
|
145
653
|
|
|
146
654
|
// Pipe stdout and stderr through readline for timestamped logging
|
|
147
|
-
const pipeLines =
|
|
148
|
-
const rl = readline.createInterface({
|
|
149
|
-
|
|
655
|
+
const pipeLines = stream => {
|
|
656
|
+
const rl = readline.createInterface({
|
|
657
|
+
input: stream,
|
|
658
|
+
crlfDelay: Infinity,
|
|
659
|
+
});
|
|
660
|
+
rl.on('line', line => {
|
|
150
661
|
const msg = `[${timestamp()}] [${name}] ${line}\n`;
|
|
151
|
-
try {
|
|
662
|
+
try {
|
|
663
|
+
fs.writeSync(logFd, msg);
|
|
664
|
+
} catch {}
|
|
152
665
|
writeOutput(msg);
|
|
153
666
|
});
|
|
154
667
|
};
|
|
@@ -159,25 +672,30 @@ function startProcess({ name, command, args, cleanupPort }) {
|
|
|
159
672
|
// NOTE: must use 'exit', not 'close'. With shell:true, grandchild processes
|
|
160
673
|
// (e.g. nest's server) inherit stdout pipes. 'close' won't fire until ALL
|
|
161
674
|
// pipe holders exit, causing dev.js to hang when npm/nest dies but server survives.
|
|
162
|
-
const exitCode = await new Promise(
|
|
163
|
-
child.on('exit',
|
|
675
|
+
const exitCode = await new Promise(resolve => {
|
|
676
|
+
child.on('exit', code => resolve(code ?? 1));
|
|
164
677
|
child.on('error', () => resolve(1));
|
|
165
678
|
});
|
|
166
679
|
|
|
167
680
|
// Kill the entire process group
|
|
168
681
|
if (entry.pid) {
|
|
169
|
-
|
|
682
|
+
signalManagedProcess(entry, 'SIGTERM');
|
|
170
683
|
await sleep(2000);
|
|
171
|
-
|
|
684
|
+
signalManagedProcess(entry, 'SIGKILL');
|
|
172
685
|
}
|
|
173
686
|
entry.pid = null;
|
|
174
687
|
entry.child = null;
|
|
688
|
+
entry.processStartTicks = null;
|
|
175
689
|
|
|
176
690
|
// Port cleanup fallback
|
|
177
691
|
if (cleanupPort) {
|
|
178
692
|
const orphans = killOrphansByPort(cleanupPort);
|
|
179
693
|
if (orphans.length > 0) {
|
|
180
|
-
logEvent(
|
|
694
|
+
logEvent(
|
|
695
|
+
'WARN',
|
|
696
|
+
name,
|
|
697
|
+
`Killed orphan processes on port ${cleanupPort}: ${orphans.join(' ')}`
|
|
698
|
+
);
|
|
181
699
|
await sleep(500);
|
|
182
700
|
}
|
|
183
701
|
}
|
|
@@ -187,26 +705,133 @@ function startProcess({ name, command, args, cleanupPort }) {
|
|
|
187
705
|
const runDuration = (Date.now() - startTime) / 1000;
|
|
188
706
|
if (runDuration >= 60) {
|
|
189
707
|
restartCount = 0;
|
|
190
|
-
logEvent(
|
|
708
|
+
logEvent(
|
|
709
|
+
'INFO',
|
|
710
|
+
name,
|
|
711
|
+
`Ran for ${Math.round(runDuration)}s, resetting restart counter`
|
|
712
|
+
);
|
|
191
713
|
} else {
|
|
192
714
|
restartCount++;
|
|
193
715
|
}
|
|
194
716
|
if (restartCount >= MAX_RESTART_COUNT) {
|
|
195
|
-
logEvent(
|
|
717
|
+
logEvent(
|
|
718
|
+
'ERROR',
|
|
719
|
+
name,
|
|
720
|
+
`Max restart count (${MAX_RESTART_COUNT}) reached, giving up`
|
|
721
|
+
);
|
|
196
722
|
break;
|
|
197
723
|
}
|
|
198
724
|
|
|
199
|
-
const delay = Math.min(
|
|
200
|
-
|
|
725
|
+
const delay = Math.min(
|
|
726
|
+
RESTART_DELAY * (1 << Math.max(0, restartCount - 1)),
|
|
727
|
+
MAX_DELAY
|
|
728
|
+
);
|
|
729
|
+
logEvent(
|
|
730
|
+
'WARN',
|
|
731
|
+
name,
|
|
732
|
+
`Process exited with code ${exitCode}, restarting (${restartCount}/${MAX_RESTART_COUNT}) in ${delay}s...`
|
|
733
|
+
);
|
|
201
734
|
await sleep(delay * 1000);
|
|
202
735
|
}
|
|
203
736
|
|
|
204
|
-
try {
|
|
737
|
+
try {
|
|
738
|
+
fs.closeSync(logFd);
|
|
739
|
+
} catch {}
|
|
205
740
|
};
|
|
206
741
|
|
|
207
742
|
return run();
|
|
208
743
|
}
|
|
209
744
|
|
|
745
|
+
function signalManagedProcess(entry, signal) {
|
|
746
|
+
if (!entry.pid) return;
|
|
747
|
+
if (entry.strictOwnership && process.platform === 'linux') {
|
|
748
|
+
if (
|
|
749
|
+
!ownedGroupIdentityMatches({
|
|
750
|
+
pid: entry.pid,
|
|
751
|
+
processStartTicks: entry.processStartTicks,
|
|
752
|
+
})
|
|
753
|
+
) {
|
|
754
|
+
return;
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
killProcessGroup(entry.pid, signal);
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
function initializeActionPlugins() {
|
|
761
|
+
writeOutput('\n🔌 Initializing action plugins...\n');
|
|
762
|
+
try {
|
|
763
|
+
execSync('fullstack-cli action-plugin init', {
|
|
764
|
+
cwd: PROJECT_ROOT,
|
|
765
|
+
stdio: 'inherit',
|
|
766
|
+
});
|
|
767
|
+
writeOutput('✅ Action plugins initialized\n\n');
|
|
768
|
+
} catch {
|
|
769
|
+
writeOutput(
|
|
770
|
+
'⚠️ Action plugin initialization failed, continuing anyway...\n\n'
|
|
771
|
+
);
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
let handoffFailbackStarted = false;
|
|
776
|
+
|
|
777
|
+
async function startHandoffFailback(owner, reason) {
|
|
778
|
+
if (handoffFailbackStarted || stopping) return;
|
|
779
|
+
handoffFailbackStarted = true;
|
|
780
|
+
logEvent(
|
|
781
|
+
'ERROR',
|
|
782
|
+
'handoff-watchdog',
|
|
783
|
+
`Backend owner lost (${reason}); taking over the public server lifecycle`
|
|
784
|
+
);
|
|
785
|
+
await terminateStrictRuntimeOwner(owner);
|
|
786
|
+
await cleanupDeadRuntimeRegistry(owner);
|
|
787
|
+
removeDeadOwnerMarker(owner);
|
|
788
|
+
if (!(await waitForPublicPortRelease())) return;
|
|
789
|
+
initializeActionPlugins();
|
|
790
|
+
void startProcess({
|
|
791
|
+
name: 'server',
|
|
792
|
+
command: 'npm',
|
|
793
|
+
args: ['run', 'dev:server'],
|
|
794
|
+
environment: {
|
|
795
|
+
SERVER_HOST: '127.0.0.1',
|
|
796
|
+
SERVER_PORT: String(PUBLIC_SERVER_PORT),
|
|
797
|
+
},
|
|
798
|
+
strictOwnership: true,
|
|
799
|
+
// Never use lsof/pkill in handoff failback. The strict per-boot registry is
|
|
800
|
+
// the only authority for removing the dead runtime's children.
|
|
801
|
+
});
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
async function monitorHandoffOwner(initialOwner) {
|
|
805
|
+
let expectedOwner = initialOwner;
|
|
806
|
+
let identityFailures = 0;
|
|
807
|
+
let portFailures = 0;
|
|
808
|
+
let stableSuccesses = 0;
|
|
809
|
+
while (!stopping && !handoffFailbackStarted) {
|
|
810
|
+
const currentOwner = readStrictHandoffOwner();
|
|
811
|
+
const reachable = await publicServerReachable();
|
|
812
|
+
if (currentOwner) {
|
|
813
|
+
expectedOwner = currentOwner;
|
|
814
|
+
identityFailures = 0;
|
|
815
|
+
} else {
|
|
816
|
+
identityFailures += 1;
|
|
817
|
+
}
|
|
818
|
+
portFailures = reachable ? 0 : portFailures + 1;
|
|
819
|
+
stableSuccesses = currentOwner && reachable ? stableSuccesses + 1 : 0;
|
|
820
|
+
if (identityFailures >= 2 || portFailures >= 5) {
|
|
821
|
+
await startHandoffFailback(
|
|
822
|
+
expectedOwner,
|
|
823
|
+
identityFailures >= 2
|
|
824
|
+
? 'owner-identity-stale'
|
|
825
|
+
: 'public-port-unreachable'
|
|
826
|
+
);
|
|
827
|
+
return;
|
|
828
|
+
}
|
|
829
|
+
// Stay responsive while establishing health or after the first failure,
|
|
830
|
+
// then reduce steady-state readiness traffic from 5 QPS to below 1 QPS.
|
|
831
|
+
await sleep(stableSuccesses >= 3 ? 1500 : 200);
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
|
|
210
835
|
// ── Cleanup ───────────────────────────────────────────────────────────────────
|
|
211
836
|
let cleanupDone = false;
|
|
212
837
|
|
|
@@ -221,7 +846,7 @@ async function cleanup() {
|
|
|
221
846
|
for (const entry of managedProcesses) {
|
|
222
847
|
if (entry.pid) {
|
|
223
848
|
logEvent('INFO', 'main', `Stopping process group (PGID: ${entry.pid})`);
|
|
224
|
-
|
|
849
|
+
signalManagedProcess(entry, 'SIGTERM');
|
|
225
850
|
}
|
|
226
851
|
}
|
|
227
852
|
|
|
@@ -231,19 +856,27 @@ async function cleanup() {
|
|
|
231
856
|
// Force kill any remaining
|
|
232
857
|
for (const entry of managedProcesses) {
|
|
233
858
|
if (entry.pid) {
|
|
234
|
-
logEvent(
|
|
235
|
-
|
|
859
|
+
logEvent(
|
|
860
|
+
'WARN',
|
|
861
|
+
'main',
|
|
862
|
+
`Force killing process group (PGID: ${entry.pid})`
|
|
863
|
+
);
|
|
864
|
+
signalManagedProcess(entry, 'SIGKILL');
|
|
236
865
|
}
|
|
237
866
|
}
|
|
238
867
|
|
|
239
868
|
// Port cleanup fallback
|
|
240
|
-
killOrphansByPort(
|
|
869
|
+
if (!BACKEND_OWNED_BY_HANDOFF) killOrphansByPort(LEGACY_SERVER_PORT);
|
|
241
870
|
killOrphansByPort(CLIENT_DEV_PORT);
|
|
242
871
|
|
|
243
872
|
logEvent('INFO', 'main', 'All processes stopped');
|
|
244
873
|
|
|
245
|
-
try {
|
|
246
|
-
|
|
874
|
+
try {
|
|
875
|
+
fs.closeSync(devStdLogFd);
|
|
876
|
+
} catch {}
|
|
877
|
+
try {
|
|
878
|
+
fs.closeSync(devLogFd);
|
|
879
|
+
} catch {}
|
|
247
880
|
|
|
248
881
|
process.exit(0);
|
|
249
882
|
}
|
|
@@ -267,22 +900,35 @@ async function main() {
|
|
|
267
900
|
|
|
268
901
|
cleanStaleDist();
|
|
269
902
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
903
|
+
const initialHandoffOwner = BACKEND_OWNED_BY_HANDOFF
|
|
904
|
+
? readStrictHandoffOwner()
|
|
905
|
+
: null;
|
|
906
|
+
const handoffOwnsBackend = Boolean(initialHandoffOwner);
|
|
907
|
+
if (BACKEND_OWNED_BY_HANDOFF && !handoffOwnsBackend) {
|
|
908
|
+
logEvent(
|
|
909
|
+
'WARN',
|
|
910
|
+
'handoff-watchdog',
|
|
911
|
+
'Static handoff claim has no matching live owner; using legacy backend ownership'
|
|
912
|
+
);
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
if (!handoffOwnsBackend) {
|
|
916
|
+
// Initialize action plugins. In handoff mode the backend runtime owns this
|
|
917
|
+
// predecessor and the source process as one lifecycle.
|
|
918
|
+
initializeActionPlugins();
|
|
277
919
|
}
|
|
278
920
|
|
|
279
921
|
// Start server and client
|
|
280
|
-
const serverPromise =
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
922
|
+
const serverPromise = handoffOwnsBackend
|
|
923
|
+
? Promise.resolve()
|
|
924
|
+
: startProcess({
|
|
925
|
+
name: 'server',
|
|
926
|
+
command: 'npm',
|
|
927
|
+
args: ['run', 'dev:server'],
|
|
928
|
+
cleanupPort: BACKEND_OWNED_BY_HANDOFF ? undefined : LEGACY_SERVER_PORT,
|
|
929
|
+
strictOwnership: BACKEND_OWNED_BY_HANDOFF,
|
|
930
|
+
});
|
|
931
|
+
if (handoffOwnsBackend) void monitorHandoffOwner(initialHandoffOwner);
|
|
286
932
|
|
|
287
933
|
const clientPromise = startProcess({
|
|
288
934
|
name: 'client',
|
|
@@ -302,7 +948,7 @@ async function main() {
|
|
|
302
948
|
}
|
|
303
949
|
}
|
|
304
950
|
|
|
305
|
-
main().catch(
|
|
951
|
+
main().catch(err => {
|
|
306
952
|
console.error('Fatal error:', err);
|
|
307
953
|
process.exit(1);
|
|
308
954
|
});
|