@amalgm/shell 0.1.163 → 0.1.164-main.34182640231
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PURPOSE.md +68 -6
- package/README.md +58 -11
- package/config/releases/trust/runtime/main.root.json +84 -0
- package/config/releases/trust/runtime/preview.root.json +84 -0
- package/dist/boot.js +28 -20
- package/dist/boot.js.map +1 -1
- package/dist/cli.js +7 -1
- package/dist/cli.js.map +1 -1
- package/dist/content-cache-index.js +2 -2
- package/dist/content-cache-index.js.map +1 -1
- package/dist/detection/runtime.js +2 -2
- package/dist/detection/runtime.js.map +1 -1
- package/dist/entity-apply-host.js +2 -2
- package/dist/entity-apply-host.js.map +1 -1
- package/dist/entity-apply-store.d.ts +2 -1
- package/dist/entity-apply-store.js +52 -37
- package/dist/entity-apply-store.js.map +1 -1
- package/dist/installation/lease.js +2 -2
- package/dist/installation/lease.js.map +1 -1
- package/dist/installation/ownership.js +2 -2
- package/dist/installation/ownership.js.map +1 -1
- package/dist/machine-host.d.ts +1 -0
- package/dist/machine-host.js +56 -31
- package/dist/machine-host.js.map +1 -1
- package/dist/process-lifecycle.d.ts +1 -3
- package/dist/process-lifecycle.js +10 -0
- package/dist/process-lifecycle.js.map +1 -1
- package/dist/pty-host.d.ts +1 -1
- package/dist/pty-host.js +70 -21
- package/dist/pty-host.js.map +1 -1
- package/dist/service/cli.js +50 -9
- package/dist/service/cli.js.map +1 -1
- package/dist/service/control.d.ts +10 -1
- package/dist/service/control.js +1 -1
- package/dist/service/control.js.map +1 -1
- package/dist/service/keepalive.d.ts +10 -3
- package/dist/service/keepalive.js +267 -29
- package/dist/service/keepalive.js.map +1 -1
- package/dist/service/recovery.d.ts +27 -0
- package/dist/service/recovery.js +58 -0
- package/dist/service/recovery.js.map +1 -0
- package/dist/service/systemd-unit.d.ts +7 -0
- package/dist/service/systemd-unit.js +43 -0
- package/dist/service/systemd-unit.js.map +1 -0
- package/dist/service/systemd.d.ts +1 -5
- package/dist/service/systemd.js +9 -31
- package/dist/service/systemd.js.map +1 -1
- package/dist/service/windows-action.d.ts +9 -0
- package/dist/service/windows-action.js +55 -0
- package/dist/service/windows-action.js.map +1 -0
- package/dist/service/windows-task.d.ts +1 -8
- package/dist/service/windows-task.js +4 -40
- package/dist/service/windows-task.js.map +1 -1
- package/dist/service/windows.js +3 -3
- package/dist/service/windows.js.map +1 -1
- package/dist/sqlite.d.ts +4 -0
- package/dist/sqlite.js +16 -0
- package/dist/sqlite.js.map +1 -0
- package/dist/updates/artifacts.d.ts +40 -0
- package/dist/updates/artifacts.js +299 -0
- package/dist/updates/artifacts.js.map +1 -0
- package/dist/updates/bootstrap.d.ts +48 -0
- package/dist/updates/bootstrap.js +384 -0
- package/dist/updates/bootstrap.js.map +1 -0
- package/dist/updates/controller.d.ts +66 -0
- package/dist/updates/controller.js +322 -0
- package/dist/updates/controller.js.map +1 -0
- package/dist/updates/isolated-command-lifetime.d.ts +1 -0
- package/dist/updates/isolated-command-lifetime.js +63 -0
- package/dist/updates/isolated-command-lifetime.js.map +1 -0
- package/dist/updates/store.d.ts +41 -0
- package/dist/updates/store.js +80 -0
- package/dist/updates/store.js.map +1 -0
- package/dist/updates/trust.d.ts +20 -0
- package/dist/updates/trust.js +138 -0
- package/dist/updates/trust.js.map +1 -0
- package/dist/user-ground-host.d.ts +1 -1
- package/dist/user-ground-host.js +3 -3
- package/dist/user-ground-host.js.map +1 -1
- package/dist/watching/host.js +4 -2
- package/dist/watching/host.js.map +1 -1
- package/dist/workspace-add-staging.js +2 -2
- package/dist/workspace-add-staging.js.map +1 -1
- package/examinations/fixtures/machine-control-plane.js +57 -0
- package/examinations/fixtures/private-wire-authority.js +1065 -0
- package/examinations/native-service.mjs +92 -0
- package/examinations/network-view.mjs +202 -0
- package/examinations/runtime-upgrade.native-test.mjs +198 -0
- package/npm-shrinkwrap.json +2104 -191
- package/package.json +94 -13
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { randomUUID, createHash } from 'node:crypto';
|
|
3
|
+
import fs from 'node:fs/promises';
|
|
4
|
+
import { spawn } from 'node:child_process';
|
|
5
|
+
import os from 'node:os';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
8
|
+
import { handleMachineControlRequest } from './fixtures/machine-control-plane.js';
|
|
9
|
+
import { createPrivateWireAuthority } from './fixtures/private-wire-authority.js';
|
|
10
|
+
|
|
11
|
+
/** Enroll the actual installed Shell through setup, with test cloud inputs.
|
|
12
|
+
* The process tree and OS service are real; this helper neither supplies a
|
|
13
|
+
* substitute child nor alters the package under examination. */
|
|
14
|
+
export async function nativeInstalledService({ packageRoot = fileURLToPath(new URL('../', import.meta.url)), label }) {
|
|
15
|
+
assert.equal(process.env.CI, 'true', 'Installed service certification requires a disposable native CI account');
|
|
16
|
+
assert.ok(['main', 'preview'].includes(label));
|
|
17
|
+
packageRoot = await fs.realpath(packageRoot);
|
|
18
|
+
const manifest = JSON.parse(await fs.readFile(path.join(packageRoot, 'package.json'), 'utf8'));
|
|
19
|
+
assert.ok(['amalgm', '@amalgm/shell'].includes(manifest.name));
|
|
20
|
+
const load = file => import(pathToFileURL(path.join(packageRoot, 'dist', file)).href);
|
|
21
|
+
const [{ serviceIdentity, detectBackend, readRegistration }, { ownerFor }, { userSystemdAvailable },
|
|
22
|
+
{ keepaliveStatus }, { MachineHost }, { installationDeviceKeyStores }] = await Promise.all([
|
|
23
|
+
load('service/registration.js'), load('service/owner.js'), load('service/systemd.js'),
|
|
24
|
+
load('service/control.js'), load('machine-host.js'), load('authorization/index.js'),
|
|
25
|
+
]);
|
|
26
|
+
const backend = detectBackend('auto', { platform: process.platform, userSystemd: userSystemdAvailable });
|
|
27
|
+
assert.notEqual(backend, 'portable', 'Shell survival certification requires the real OS service manager');
|
|
28
|
+
const owner = ownerFor(backend);
|
|
29
|
+
const home = await fs.mkdtemp(path.join(os.tmpdir(), 'amalgm-installed-service-'));
|
|
30
|
+
const user = `desktop-upgrade-${randomUUID()}@example.test`;
|
|
31
|
+
const identity = serviceIdentity({ runtimeLabel: label, branch: label, user, home });
|
|
32
|
+
let wireUrl; let setupChild; let stateFiles = [];
|
|
33
|
+
const authority = createPrivateWireAuthority({ token: 'desktop-upgrade-examination', onRequest: (request, response) => {
|
|
34
|
+
void handleMachineControlRequest(request, response, { userEmail: user, userId: user,
|
|
35
|
+
wireUrl: () => wireUrl, tunnelToken: 'desktop-upgrade-examination' });
|
|
36
|
+
} });
|
|
37
|
+
const close = async () => {
|
|
38
|
+
if (setupChild?.exitCode === null) setupChild.kill('SIGKILL');
|
|
39
|
+
await owner.uninstall(identity);
|
|
40
|
+
for (const socket of authority.sockets.clients) socket.terminate();
|
|
41
|
+
await new Promise(resolve => authority.sockets.close(resolve));
|
|
42
|
+
authority.server.closeAllConnections();
|
|
43
|
+
await new Promise(resolve => authority.server.close(resolve));
|
|
44
|
+
const stateDir = path.join(identity.amalgmRoot, '.amalgm');
|
|
45
|
+
try {
|
|
46
|
+
const { device_id: deviceId } = JSON.parse(await fs.readFile(path.join(stateDir, 'device.json'), 'utf8'));
|
|
47
|
+
const stores = installationDeviceKeyStores({ stateDir, deviceId });
|
|
48
|
+
await stores.active.remove(); await stores.pending.remove();
|
|
49
|
+
} catch (error) { if (error.code !== 'ENOENT') throw error; }
|
|
50
|
+
await fs.rm(home, { recursive: true, force: true, maxRetries: 5 });
|
|
51
|
+
};
|
|
52
|
+
try {
|
|
53
|
+
await new Promise(resolve => authority.server.listen(0, '127.0.0.1', resolve));
|
|
54
|
+
const appUrl = `http://127.0.0.1:${authority.server.address().port}`;
|
|
55
|
+
wireUrl = appUrl.replace('http:', 'ws:') + '/wire';
|
|
56
|
+
const args = [...(label === 'preview' ? ['preview'] : []), 'setup', '--setup-code', 'ABCD-EFGH-JKLM-NPQR',
|
|
57
|
+
'--home', home, '--app-url', appUrl, '--no-open'];
|
|
58
|
+
await new Promise((resolve, reject) => {
|
|
59
|
+
setupChild = spawn(process.execPath, [path.join(packageRoot, 'dist/main.js'), ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
60
|
+
let output = '';
|
|
61
|
+
const collect = bytes => { output = (output + bytes).slice(-24_000); };
|
|
62
|
+
setupChild.stdout.on('data', collect); setupChild.stderr.on('data', collect);
|
|
63
|
+
const timer = setTimeout(() => { setupChild.kill('SIGTERM'); reject(new Error(`Installed Shell setup timed out: ${output}`)); }, 90_000);
|
|
64
|
+
setupChild.once('error', error => { clearTimeout(timer); reject(error); });
|
|
65
|
+
setupChild.once('exit', code => { clearTimeout(timer); if (code === 0 && /AMALGM_SETUP_COMPLETE/.test(output)) resolve(); else reject(new Error(`Installed Shell setup failed (${code}): ${output}`)); });
|
|
66
|
+
});
|
|
67
|
+
const registration = readRegistration(identity);
|
|
68
|
+
assert.equal(await fs.realpath(registration.package_root), packageRoot, 'OS service must execute the package under examination');
|
|
69
|
+
assert.equal(registration.package_version, manifest.version);
|
|
70
|
+
const host = new MachineHost({ runtimeLabel: label, branch: label, homedir: home, appUrl, userEmail: user,
|
|
71
|
+
runtimeVersion: manifest.version });
|
|
72
|
+
const selected = host.selectedPaths();
|
|
73
|
+
stateFiles = [host.deviceFile, selected.publicRecord, selected.secretAuth, selected.installation];
|
|
74
|
+
await host.close();
|
|
75
|
+
const status = async () => {
|
|
76
|
+
const controller = await keepaliveStatus(identity);
|
|
77
|
+
assert.ok(controller?.childPid && controller.phase === 'running', 'Installed Shell has no authenticated healthy controller and child');
|
|
78
|
+
assert.equal((await owner.status(identity)).pid, controller.bootstrapPid ?? controller.pid);
|
|
79
|
+
process.kill(controller.childPid, 0);
|
|
80
|
+
const current = readRegistration(identity);
|
|
81
|
+
const currentPackage = JSON.parse(await fs.readFile(path.join(current.package_root, 'package.json'), 'utf8'));
|
|
82
|
+
const durable = [];
|
|
83
|
+
for (const file of stateFiles) durable.push({ name: path.basename(file), sha256: createHash('sha256').update(await fs.readFile(file)).digest('hex') });
|
|
84
|
+
return { package: currentPackage.name, version: currentPackage.version, source: currentPackage.amalgmSource ?? null,
|
|
85
|
+
backend, binding: identity.binding, controllerPid: controller.pid, childPid: controller.childPid,
|
|
86
|
+
bootstrapPid: controller.bootstrapPid ?? null, instanceId: controller.instanceId,
|
|
87
|
+
updates: controller.update ?? null, durable };
|
|
88
|
+
};
|
|
89
|
+
await status();
|
|
90
|
+
return { identity, paths: { ...selected, deviceFile: host.deviceFile }, status, close };
|
|
91
|
+
} catch (error) { await close(); throw error; }
|
|
92
|
+
}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
// Disposable CI network transport shared by packaged runtime and desktop gates.
|
|
2
|
+
// Callers supply exact exported authority bytes; this module never signs or
|
|
3
|
+
// interprets release metadata and is never imported by the product runtime.
|
|
4
|
+
import assert from 'node:assert/strict';
|
|
5
|
+
import crypto from 'node:crypto';
|
|
6
|
+
import fs from 'node:fs/promises';
|
|
7
|
+
import { createReadStream, realpathSync } from 'node:fs';
|
|
8
|
+
import https from 'node:https';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
import readline from 'node:readline';
|
|
11
|
+
import { Transform } from 'node:stream';
|
|
12
|
+
import { pipeline } from 'node:stream/promises';
|
|
13
|
+
import { spawn, execFileSync } from 'node:child_process';
|
|
14
|
+
import { fileURLToPath } from 'node:url';
|
|
15
|
+
|
|
16
|
+
export const canonicalHost = 'amalgm-desktop-releases.fly.dev';
|
|
17
|
+
const run = (file, args, options = {}) => execFileSync(file, args, { encoding: 'utf8', timeout: 30_000, ...options });
|
|
18
|
+
const sudo = (file, args) => run('sudo', ['-n', file, ...args]);
|
|
19
|
+
const powershell = (source, env = {}) => run('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', source], { env: { ...process.env, ...env } });
|
|
20
|
+
|
|
21
|
+
export async function startNetworkView({ temporary, immutable, previous = {}, candidate, artifacts = [], events, hint = {}, serviceEnvironment = false }) {
|
|
22
|
+
assert.equal(process.env.CI, 'true');
|
|
23
|
+
assert.equal(process.env.GITHUB_ACTIONS, 'true', 'Never replace DNS or TLS trust on a developer account');
|
|
24
|
+
assert.ok(['darwin', 'linux', 'win32'].includes(process.platform));
|
|
25
|
+
for (const resources of [immutable, previous, candidate]) assert.ok(resources && typeof resources === 'object');
|
|
26
|
+
assert.ok(typeof events === 'string' && events.startsWith('/trust/'));
|
|
27
|
+
const view = { immutable, previous, candidate, artifacts, events, hint };
|
|
28
|
+
const directory = path.join(temporary, 'network'); await fs.mkdir(directory, { mode: 0o700 });
|
|
29
|
+
const ca = path.join(directory, 'ca.crt');
|
|
30
|
+
const key = path.join(directory, 'server.key');
|
|
31
|
+
const certificate = path.join(directory, 'server.crt');
|
|
32
|
+
const name = `Amalgm disposable native examination ${crypto.randomUUID()}`;
|
|
33
|
+
const extension = path.join(directory, 'server.ext');
|
|
34
|
+
await fs.writeFile(extension, `subjectAltName=DNS:${canonicalHost}\nextendedKeyUsage=serverAuth\n`);
|
|
35
|
+
run('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', `/CN=${name}`, '-keyout', path.join(directory, 'ca.key'), '-out', ca], { stdio: 'pipe' });
|
|
36
|
+
run('openssl', ['req', '-newkey', 'rsa:2048', '-nodes', '-subj', `/CN=${canonicalHost}`, '-keyout', key, '-out', path.join(directory, 'server.csr')], { stdio: 'pipe' });
|
|
37
|
+
run('openssl', ['x509', '-req', '-in', path.join(directory, 'server.csr'), '-CA', ca, '-CAkey', path.join(directory, 'ca.key'), '-CAcreateserial', '-days', '1', '-extfile', extension, '-out', certificate], { stdio: 'pipe' });
|
|
38
|
+
// The server never needs even the temporary CA private key after issuing TLS.
|
|
39
|
+
await fs.rm(path.join(directory, 'ca.key'));
|
|
40
|
+
const hostsFile = process.platform === 'win32' ? path.join(process.env.SystemRoot, 'System32/drivers/etc/hosts') : '/etc/hosts';
|
|
41
|
+
const originalHosts = await fs.readFile(hostsFile);
|
|
42
|
+
const hosts = path.join(directory, 'hosts');
|
|
43
|
+
await fs.writeFile(hosts, Buffer.concat([originalHosts, Buffer.from(`\n127.0.0.1 ${canonicalHost}\n`)]));
|
|
44
|
+
const original = path.join(directory, 'hosts.original'); await fs.writeFile(original, originalHosts);
|
|
45
|
+
const linuxCa = `/usr/local/share/ca-certificates/amalgm-examination-${crypto.randomUUID()}.crt`;
|
|
46
|
+
const nss = path.join(process.env.HOME || process.env.USERPROFILE, '.pki/nssdb');
|
|
47
|
+
const fingerprint = run('openssl', ['x509', '-in', ca, '-noout', '-fingerprint', '-sha1']).split('=')[1].trim().replace(/:/g, '');
|
|
48
|
+
let child; let changedHosts = false; let trusted = false; let nssTrusted = false;
|
|
49
|
+
let changedServiceEnvironment = false; let priorServiceCa = null;
|
|
50
|
+
const serviceCa = value => {
|
|
51
|
+
if (process.platform === 'darwin') run('launchctl', value === null ? ['unsetenv', 'NODE_EXTRA_CA_CERTS'] : ['setenv', 'NODE_EXTRA_CA_CERTS', value]);
|
|
52
|
+
else if (process.platform === 'linux') run('systemctl', ['--user', value === null ? 'unset-environment' : 'set-environment', value === null ? 'NODE_EXTRA_CA_CERTS' : `NODE_EXTRA_CA_CERTS=${value}`]);
|
|
53
|
+
else powershell("[Environment]::SetEnvironmentVariable('NODE_EXTRA_CA_CERTS', $env:AMALGM_EXAMINATION_CA, 'User')", { AMALGM_EXAMINATION_CA: value ?? '' });
|
|
54
|
+
};
|
|
55
|
+
const copyHosts = async source => {
|
|
56
|
+
if (process.platform === 'win32') await fs.copyFile(source, hostsFile);
|
|
57
|
+
else sudo('cp', [source, hostsFile]);
|
|
58
|
+
};
|
|
59
|
+
const close = async () => {
|
|
60
|
+
const errors = [];
|
|
61
|
+
try {
|
|
62
|
+
if (child && child.exitCode === null) {
|
|
63
|
+
child.stdin.end();
|
|
64
|
+
await new Promise((resolve, reject) => {
|
|
65
|
+
const timeout = setTimeout(() => reject(new Error('Privileged examination transport did not stop')), 10_000);
|
|
66
|
+
child.once('exit', () => { clearTimeout(timeout); resolve(); });
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
} catch (error) { errors.push(error); }
|
|
70
|
+
if (changedHosts) await copyHosts(original);
|
|
71
|
+
if (changedServiceEnvironment) serviceCa(priorServiceCa);
|
|
72
|
+
if (process.platform === 'darwin') {
|
|
73
|
+
if (trusted) sudo('security', ['remove-trusted-cert', '-d', ca]);
|
|
74
|
+
sudo('dscacheutil', ['-flushcache']);
|
|
75
|
+
sudo('killall', ['-HUP', 'mDNSResponder']);
|
|
76
|
+
} else if (process.platform === 'linux') {
|
|
77
|
+
if (nssTrusted) run('certutil', ['-D', '-d', `sql:${nss}`, '-n', name]);
|
|
78
|
+
if (trusted) { sudo('rm', ['-f', linuxCa]); sudo('update-ca-certificates', []); }
|
|
79
|
+
} else {
|
|
80
|
+
if (trusted) run('certutil.exe', ['-user', '-delstore', 'Root', fingerprint]);
|
|
81
|
+
run('ipconfig.exe', ['/flushdns']);
|
|
82
|
+
}
|
|
83
|
+
if (errors.length) throw new AggregateError(errors, 'Examination transport cleanup failed');
|
|
84
|
+
};
|
|
85
|
+
try {
|
|
86
|
+
if (process.platform === 'darwin') {
|
|
87
|
+
sudo('security', ['add-trusted-cert', '-d', '-r', 'trustRoot', '-k', '/Library/Keychains/System.keychain', ca]); trusted = true;
|
|
88
|
+
} else if (process.platform === 'linux') {
|
|
89
|
+
sudo('cp', [ca, linuxCa]); trusted = true; sudo('update-ca-certificates', []);
|
|
90
|
+
await fs.mkdir(nss, { recursive: true });
|
|
91
|
+
try { await fs.access(path.join(nss, 'cert9.db')); }
|
|
92
|
+
catch { run('certutil', ['-N', '--empty-password', '-d', `sql:${nss}`]); }
|
|
93
|
+
run('certutil', ['-A', '-d', `sql:${nss}`, '-n', name, '-t', 'C,,', '-i', ca]); nssTrusted = true;
|
|
94
|
+
} else {
|
|
95
|
+
powershell("if (-not ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { throw 'Disposable Windows CI account must be an administrator' }");
|
|
96
|
+
run('certutil.exe', ['-user', '-addstore', 'Root', ca]); trusted = true;
|
|
97
|
+
}
|
|
98
|
+
if (serviceEnvironment) {
|
|
99
|
+
if (process.platform === 'darwin') { try { priorServiceCa = run('launchctl', ['getenv', 'NODE_EXTRA_CA_CERTS']).trim() || null; } catch { /* unset */ } }
|
|
100
|
+
else if (process.platform === 'linux') priorServiceCa = run('systemctl', ['--user', 'show-environment']).split('\n').find(line => line.startsWith('NODE_EXTRA_CA_CERTS='))?.slice('NODE_EXTRA_CA_CERTS='.length) ?? null;
|
|
101
|
+
else priorServiceCa = JSON.parse(powershell("ConvertTo-Json -Compress -InputObject ([Environment]::GetEnvironmentVariable('NODE_EXTRA_CA_CERTS', 'User'))"));
|
|
102
|
+
serviceCa(ca); changedServiceEnvironment = true;
|
|
103
|
+
}
|
|
104
|
+
await copyHosts(hosts); changedHosts = true;
|
|
105
|
+
if (process.platform === 'darwin') { sudo('dscacheutil', ['-flushcache']); sudo('killall', ['-HUP', 'mDNSResponder']); }
|
|
106
|
+
else if (process.platform === 'win32') run('ipconfig.exe', ['/flushdns']);
|
|
107
|
+
const config = path.join(directory, 'config.json');
|
|
108
|
+
await fs.writeFile(config, JSON.stringify({ ...view, key, certificate }), { mode: 0o600 });
|
|
109
|
+
child = spawn(process.platform === 'win32' ? process.execPath : 'sudo', [
|
|
110
|
+
...(process.platform === 'win32' ? [] : ['-n', process.execPath]), fileURLToPath(import.meta.url), '--serve', config], {
|
|
111
|
+
env: { ...process.env, CI: 'true', GITHUB_ACTIONS: 'true' }, stdio: ['pipe', 'pipe', 'pipe'],
|
|
112
|
+
});
|
|
113
|
+
let stderr = ''; let sequence = 0; const pending = new Map();
|
|
114
|
+
child.stderr.on('data', data => { stderr = (stderr + data).slice(-8000); });
|
|
115
|
+
const messages = readline.createInterface({ input: child.stdout });
|
|
116
|
+
messages.on('line', line => {
|
|
117
|
+
const value = JSON.parse(line); const callback = pending.get(value.id);
|
|
118
|
+
if (callback) { pending.delete(value.id); value.error ? callback.reject(new Error(value.error)) : callback.resolve(value); }
|
|
119
|
+
});
|
|
120
|
+
child.on('error', error => { for (const callback of pending.values()) callback.reject(error); pending.clear(); });
|
|
121
|
+
child.on('exit', () => { for (const callback of pending.values()) callback.reject(new Error(`Examination transport stopped: ${stderr}`)); pending.clear(); });
|
|
122
|
+
const command = async operation => {
|
|
123
|
+
const id = ++sequence;
|
|
124
|
+
return new Promise((resolve, reject) => {
|
|
125
|
+
const timeout = setTimeout(() => { pending.delete(id); reject(new Error(`Examination transport timed out: ${stderr}`)); }, 15_000);
|
|
126
|
+
pending.set(id, { resolve: value => { clearTimeout(timeout); resolve(value); }, reject: error => { clearTimeout(timeout); reject(error); } });
|
|
127
|
+
child.stdin.write(JSON.stringify({ id, ...operation }) + '\n');
|
|
128
|
+
});
|
|
129
|
+
};
|
|
130
|
+
await command({ operation: 'status' });
|
|
131
|
+
return { ca, close, command };
|
|
132
|
+
} catch (error) { await close(); throw error; }
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
async function serve(configFile) {
|
|
136
|
+
// Only this disposable CI process needs the privileged HTTPS listening port.
|
|
137
|
+
if (process.platform !== 'win32') assert.equal(process.getuid(), 0);
|
|
138
|
+
const config = JSON.parse(await fs.readFile(configFile, 'utf8'));
|
|
139
|
+
const immutable = config.immutable;
|
|
140
|
+
let selected = config.previous; let unavailable = false; let fault = false; let failures = 0; let artifactRequests = 0;
|
|
141
|
+
const streams = new Set(); const sockets = new Set();
|
|
142
|
+
const artifacts = new Map(config.artifacts.map(entry => [entry.path, entry]));
|
|
143
|
+
const server = https.createServer({ key: await fs.readFile(config.key), cert: await fs.readFile(config.certificate) }, async (req, res) => {
|
|
144
|
+
try {
|
|
145
|
+
const url = new URL(req.url, `https://${canonicalHost}`);
|
|
146
|
+
if (req.headers.host !== canonicalHost || !['GET', 'HEAD'].includes(req.method)) { res.writeHead(400).end(); return; }
|
|
147
|
+
if (url.pathname === config.events) {
|
|
148
|
+
if (unavailable) { res.writeHead(503).end(); return; }
|
|
149
|
+
res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-store' });
|
|
150
|
+
streams.add(res); res.on('close', () => streams.delete(res)); res.write('event: ready\ndata: {}\n\n'); return;
|
|
151
|
+
}
|
|
152
|
+
const resource = selected[url.pathname] ?? immutable[url.pathname];
|
|
153
|
+
if (resource) {
|
|
154
|
+
const bytes = Buffer.from(resource.body, 'base64');
|
|
155
|
+
res.writeHead(200, { 'content-type': resource.contentType || 'application/json', 'content-length': bytes.length });
|
|
156
|
+
res.end(req.method === 'HEAD' ? undefined : bytes); return;
|
|
157
|
+
}
|
|
158
|
+
const artifact = artifacts.get(url.pathname);
|
|
159
|
+
if (artifact) {
|
|
160
|
+
artifactRequests++;
|
|
161
|
+
const corrupt = fault && artifact.faultable !== false;
|
|
162
|
+
if (corrupt) failures++;
|
|
163
|
+
res.writeHead(200, { 'content-length': artifact.size, 'content-type': 'application/octet-stream' });
|
|
164
|
+
if (req.method === 'HEAD') { res.end(); return; }
|
|
165
|
+
let first = true;
|
|
166
|
+
const transport = new Transform({ transform(chunk, _encoding, callback) {
|
|
167
|
+
if (first && corrupt) chunk[0] ^= 1;
|
|
168
|
+
first = false; callback(null, chunk);
|
|
169
|
+
} });
|
|
170
|
+
await pipeline(createReadStream(artifact.file), transport, res);
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
res.writeHead(404).end();
|
|
174
|
+
} catch (error) { res.destroy(error); }
|
|
175
|
+
});
|
|
176
|
+
server.on('connection', socket => { sockets.add(socket); socket.on('close', () => sockets.delete(socket)); });
|
|
177
|
+
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(443, '127.0.0.1', resolve); });
|
|
178
|
+
const heartbeat = setInterval(() => { for (const stream of streams) stream.write(': keepalive\n\n'); }, 15_000);
|
|
179
|
+
for await (const line of readline.createInterface({ input: process.stdin })) {
|
|
180
|
+
const command = JSON.parse(line);
|
|
181
|
+
try {
|
|
182
|
+
if (command.operation === 'publish') {
|
|
183
|
+
selected = config.candidate; unavailable = false; fault = command.corrupt === true;
|
|
184
|
+
for (const stream of streams) stream.write(`event: promotion\ndata: ${JSON.stringify(config.hint)}\n\n`);
|
|
185
|
+
} else if (command.operation === 'reset') { selected = config.previous; unavailable = false; fault = false; }
|
|
186
|
+
else if (command.operation === 'unavailable') {
|
|
187
|
+
selected = {}; unavailable = true; fault = false;
|
|
188
|
+
for (const stream of streams) stream.end();
|
|
189
|
+
}
|
|
190
|
+
else assert.equal(command.operation, 'status');
|
|
191
|
+
process.stdout.write(JSON.stringify({ id: command.id, subscribers: streams.size, failures, artifactRequests }) + '\n');
|
|
192
|
+
} catch (error) { process.stdout.write(JSON.stringify({ id: command.id, error: error.message }) + '\n'); }
|
|
193
|
+
}
|
|
194
|
+
clearInterval(heartbeat);
|
|
195
|
+
for (const stream of streams) stream.end();
|
|
196
|
+
for (const socket of sockets) socket.destroy();
|
|
197
|
+
await new Promise(resolve => server.close(resolve));
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)) && process.argv[2] === '--serve') {
|
|
201
|
+
await serve(process.argv[3]);
|
|
202
|
+
}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import test from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import crypto from 'node:crypto';
|
|
4
|
+
import fs from 'node:fs/promises';
|
|
5
|
+
import { createWriteStream } from 'node:fs';
|
|
6
|
+
import os from 'node:os';
|
|
7
|
+
import path from 'node:path';
|
|
8
|
+
import { createRequire } from 'node:module';
|
|
9
|
+
import { pathToFileURL } from 'node:url';
|
|
10
|
+
import { pipeline } from 'node:stream/promises';
|
|
11
|
+
import { Readable } from 'node:stream';
|
|
12
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
13
|
+
import { Metadata, MetadataKind } from '@tufjs/models';
|
|
14
|
+
import { admitManagedRuntimeRelease } from '@amalgm/core/supervision';
|
|
15
|
+
import { runtimeRoot } from '../dist/updates/trust.js';
|
|
16
|
+
import { examineArchive, hashFile, runIsolated } from '../dist/updates/artifacts.js';
|
|
17
|
+
import { runtimeHost } from '../dist/updates/store.js';
|
|
18
|
+
import { canonicalHost, startNetworkView } from './network-view.mjs';
|
|
19
|
+
|
|
20
|
+
const encode = value => Buffer.from(JSON.stringify(value) + '\n');
|
|
21
|
+
const digest = value => crypto.createHash('sha256').update(encode(value)).digest('hex');
|
|
22
|
+
const resource = value => ({ body: encode(value).toString('base64') });
|
|
23
|
+
const readJson = async file => JSON.parse(await fs.readFile(file, 'utf8'));
|
|
24
|
+
const pause = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
25
|
+
async function until(description, action, timeout = 600_000) {
|
|
26
|
+
const deadline = Date.now() + timeout; let last;
|
|
27
|
+
do {
|
|
28
|
+
try { const value = await action(); if (value) return value; }
|
|
29
|
+
catch (error) { last = error; }
|
|
30
|
+
await pause(250);
|
|
31
|
+
} while (Date.now() < deadline);
|
|
32
|
+
throw new Error(`${description} did not complete${last ? `: ${last.message}` : ''}`);
|
|
33
|
+
}
|
|
34
|
+
function journal(identity) {
|
|
35
|
+
const db = new DatabaseSync(path.toNamespacedPath(path.join(identity.serviceDir, 'updates.sqlite')), { readOnly: true });
|
|
36
|
+
try {
|
|
37
|
+
const read = key => {
|
|
38
|
+
const row = db.prepare('SELECT value FROM update_state WHERE key = ?').get(key);
|
|
39
|
+
return row ? JSON.parse(row.value) : null;
|
|
40
|
+
};
|
|
41
|
+
return { launch: read('launch'), policy: read('policy') };
|
|
42
|
+
} finally { db.close(); }
|
|
43
|
+
}
|
|
44
|
+
function validateReceipt(receipt, roots, lane) {
|
|
45
|
+
assert.ok(receipt?.target && receipt.authorization, 'An exact signed installed receipt is required');
|
|
46
|
+
const target = receipt.target;
|
|
47
|
+
const authorization = Metadata.fromJSON(MetadataKind.Targets, receipt.authorization);
|
|
48
|
+
let verified = false;
|
|
49
|
+
for (const root of roots) {
|
|
50
|
+
try { root.verifyDelegate('targets', authorization); verified = true; break; } catch { /* Another historical root may own the receipt. */ }
|
|
51
|
+
}
|
|
52
|
+
assert.ok(verified, 'Installed receipt has no signature from its scoped public root history');
|
|
53
|
+
const entry = authorization.signed.targets['current.json'];
|
|
54
|
+
assert.equal(Object.keys(authorization.signed.targets).length, 1);
|
|
55
|
+
assert.equal(authorization.signed.delegations, undefined);
|
|
56
|
+
assert.equal(entry.custom.amalgm, `runtime/${lane}`);
|
|
57
|
+
assert.equal(entry.length, encode(target).length); assert.equal(entry.hashes.sha256, digest(target));
|
|
58
|
+
return admitManagedRuntimeRelease(target, digest(target), { runtimeLabel: lane, branch: lane }, runtimeHost());
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The producer's public roots and signed prepared generation are input bytes.
|
|
62
|
+
* No native job receives a release key or advances the actual selected head. */
|
|
63
|
+
export function runtimeExaminationView(examination, baselineReceipt) {
|
|
64
|
+
assert.equal(examination.schema, 1);
|
|
65
|
+
const lane = examination.lane;
|
|
66
|
+
assert.ok(['main', 'preview'].includes(lane));
|
|
67
|
+
assert.equal(examination.id, examination.candidate.target.id);
|
|
68
|
+
const scope = `/trust/runtime/${lane}`;
|
|
69
|
+
const immutable = {};
|
|
70
|
+
assert.ok(Array.isArray(examination.roots) && examination.roots.length);
|
|
71
|
+
const roots = examination.roots.map((value, index) => {
|
|
72
|
+
const root = runtimeRoot(value, lane);
|
|
73
|
+
assert.equal(root.signed.version, index + 1);
|
|
74
|
+
if (index) runtimeRoot(examination.roots[index - 1], lane).verifyDelegate('root', root);
|
|
75
|
+
immutable[`${scope}/metadata/${index + 1}.root.json`] = resource(value);
|
|
76
|
+
return root;
|
|
77
|
+
});
|
|
78
|
+
for (const generation of [examination.previous, examination.candidate].filter(Boolean)) {
|
|
79
|
+
validateReceipt(generation, roots, lane);
|
|
80
|
+
const root = roots[generation.rootVersion - 1]; assert.ok(root);
|
|
81
|
+
root.verifyDelegate('targets', Metadata.fromJSON(MetadataKind.Targets, generation.authorization));
|
|
82
|
+
const snapshot = Metadata.fromJSON(MetadataKind.Snapshot, generation.snapshot);
|
|
83
|
+
const timestamp = Metadata.fromJSON(MetadataKind.Timestamp, generation.timestamp);
|
|
84
|
+
root.verifyDelegate('snapshot', snapshot); root.verifyDelegate('timestamp', timestamp);
|
|
85
|
+
for (const [parent, child, filename] of [[generation.snapshot, generation.authorization, 'targets.json'], [generation.timestamp, generation.snapshot, 'snapshot.json']]) {
|
|
86
|
+
const link = parent.signed.meta[filename];
|
|
87
|
+
assert.equal(link.version, child.signed.version); assert.equal(link.length, encode(child).length);
|
|
88
|
+
assert.equal(link.hashes.sha256, digest(child));
|
|
89
|
+
}
|
|
90
|
+
if (generation === examination.candidate) {
|
|
91
|
+
for (const value of [generation.authorization, generation.snapshot, generation.timestamp, examination.roots[generation.rootVersion - 1]]) {
|
|
92
|
+
assert.ok(Date.parse(value.signed.expires) > Date.now(), 'The candidate authority must remain fresh throughout certification');
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
immutable[`${scope}/metadata/${generation.authorization.signed.version}.targets.json`] = resource(generation.authorization);
|
|
96
|
+
immutable[`${scope}/metadata/${generation.version}.snapshot.json`] = resource(generation.snapshot);
|
|
97
|
+
immutable[`${scope}/targets/${digest(generation.target)}.current.json`] = resource(generation.target);
|
|
98
|
+
immutable[`${scope}/releases/${generation.target.id}/receipt.json`] = resource({ target: generation.target, authorization: generation.authorization });
|
|
99
|
+
}
|
|
100
|
+
const receipt = examination.previous ?? baselineReceipt;
|
|
101
|
+
const baseline = validateReceipt(receipt, roots, lane);
|
|
102
|
+
const candidate = validateReceipt(examination.candidate, roots, lane);
|
|
103
|
+
assert.ok(candidate.target.sequence > baseline.target.sequence);
|
|
104
|
+
assert.notEqual(candidate.target.id, baseline.target.id);
|
|
105
|
+
immutable[`${scope}/releases/${baseline.target.id}/receipt.json`] = resource({ target: receipt.target, authorization: receipt.authorization });
|
|
106
|
+
const view = generation => generation ? { [`${scope}/metadata/timestamp.json`]: resource(generation.timestamp) } : {};
|
|
107
|
+
return { baseline, candidate, immutable, previous: view(examination.previous), candidateView: view(examination.candidate), events: `${scope}/events` };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
test('the real native Shell service replaces its authorized package and survives OS restart with identity and state intact', {
|
|
111
|
+
skip: process.env.AMALGM_TEST_NATIVE_RUNTIME_UPDATES !== '1', timeout: 1_800_000,
|
|
112
|
+
}, async t => {
|
|
113
|
+
assert.equal(process.env.CI, 'true'); assert.equal(process.env.GITHUB_ACTIONS, 'true');
|
|
114
|
+
assert.ok(['darwin', 'linux', 'win32'].includes(process.platform));
|
|
115
|
+
assert.ok(process.env.AMALGM_RUNTIME_EXAMINATION_DIR, 'The producer must supply signed examination inputs');
|
|
116
|
+
const directory = path.resolve(process.env.AMALGM_RUNTIME_EXAMINATION_DIR);
|
|
117
|
+
const examination = await readJson(path.join(directory, 'examination.json'));
|
|
118
|
+
const baselineReceipt = examination.previous ? null : await readJson(path.join(directory, 'baseline.json'));
|
|
119
|
+
const view = runtimeExaminationView(examination, baselineReceipt);
|
|
120
|
+
const artifact = view.baseline.target.release.units[0].artifact;
|
|
121
|
+
const nextArtifact = view.candidate.target.release.units[0].artifact;
|
|
122
|
+
const proof = { schema: 1, lane: examination.lane, platform: process.platform, architecture: process.arch,
|
|
123
|
+
previous: view.baseline.target.id, candidate: view.candidate.target.id,
|
|
124
|
+
previousDigest: view.baseline.digest, candidateDigest: view.candidate.digest,
|
|
125
|
+
bootstrapProof: examination.previous === null, legacyAdoptionCertified: false, passed: false,
|
|
126
|
+
previousArtifact: artifact, candidateArtifact: nextArtifact, checkpoints: [] };
|
|
127
|
+
const temporary = await fs.mkdtemp(path.join(os.tmpdir(), 'amalgm-native-runtime-update-'));
|
|
128
|
+
let network; let service;
|
|
129
|
+
t.after(async () => {
|
|
130
|
+
const errors = [];
|
|
131
|
+
try { await service?.close(); } catch (error) { errors.push(error); }
|
|
132
|
+
try { await network?.close(); } catch (error) { errors.push(error); }
|
|
133
|
+
if (errors.length) { proof.passed = false; proof.cleanupErrors = errors.map(error => error.message); }
|
|
134
|
+
await fs.writeFile(path.join(directory, `runtime-update-${process.platform}.json`), JSON.stringify(proof, null, 2) + '\n');
|
|
135
|
+
await fs.rm(temporary, { recursive: true, force: true, maxRetries: 5 });
|
|
136
|
+
if (errors.length) throw new AggregateError(errors, 'Native runtime examination cleanup failed');
|
|
137
|
+
});
|
|
138
|
+
const archive = path.join(temporary, 'baseline.tgz');
|
|
139
|
+
const response = await fetch(`https://registry.npmjs.org/amalgm/-/amalgm-${artifact.version}.tgz`, { redirect: 'error', signal: AbortSignal.timeout(180_000) });
|
|
140
|
+
assert.equal(response.status, 200, 'The exact baseline public package must be available');
|
|
141
|
+
await pipeline(Readable.fromWeb(response.body), createWriteStream(archive, { flags: 'wx' }));
|
|
142
|
+
assert.equal(await hashFile(archive), artifact.sha512);
|
|
143
|
+
await examineArchive(archive, temporary);
|
|
144
|
+
const packageRoot = path.join(temporary, 'package');
|
|
145
|
+
const manifest = await readJson(path.join(packageRoot, 'package.json'));
|
|
146
|
+
assert.equal(manifest.name, 'amalgm'); assert.equal(manifest.version, artifact.version);
|
|
147
|
+
for (const name of ['repository', 'branch', 'commit']) assert.equal(manifest.amalgmSource[name], artifact.source[name]);
|
|
148
|
+
assert.equal(await hashFile(path.join(packageRoot, 'npm-shrinkwrap.json')), artifact.lockSha512);
|
|
149
|
+
const shippedRoot = await readJson(path.join(packageRoot, 'config/releases/trust/runtime', `${examination.lane}.root.json`));
|
|
150
|
+
assert.deepEqual(shippedRoot, examination.roots[shippedRoot.signed.version - 1], 'The package must carry the actual reviewed trust root');
|
|
151
|
+
const npm = path.join(path.dirname(createRequire(import.meta.url).resolve('npm/package.json')), 'bin/npm-cli.js');
|
|
152
|
+
await runIsolated(process.execPath, [npm, 'ci', '--omit=dev', '--strict-allow-scripts', '--no-audit', '--no-fund'], packageRoot, new AbortController().signal);
|
|
153
|
+
network = await startNetworkView({ temporary, immutable: view.immutable, previous: view.previous, candidate: view.candidateView,
|
|
154
|
+
events: view.events, hint: { id: examination.id, sequence: examination.candidate.target.sequence }, serviceEnvironment: true });
|
|
155
|
+
const { nativeInstalledService } = await import(pathToFileURL(path.join(packageRoot, 'examinations/native-service.mjs')).href);
|
|
156
|
+
service = await nativeInstalledService({ packageRoot, label: examination.lane });
|
|
157
|
+
const baseline = await until('Verified installed baseline and native service health', async () => {
|
|
158
|
+
const state = journal(service.identity); const status = await service.status();
|
|
159
|
+
return state.policy?.active?.digest === view.baseline.digest && state.policy.activeHealthy
|
|
160
|
+
&& !state.policy.transaction && status.version === artifact.version ? status : null;
|
|
161
|
+
});
|
|
162
|
+
assert.ok(baseline.bootstrapPid && baseline.bootstrapPid !== baseline.controllerPid, 'The OS must own the stable launcher independently of the controller package');
|
|
163
|
+
const marker = crypto.randomUUID();
|
|
164
|
+
const workspace = path.join(service.paths.userRoot, 'workspaces'); await fs.mkdir(workspace, { recursive: true });
|
|
165
|
+
const markerFile = path.join(workspace, 'native-runtime-upgrade-marker.txt'); await fs.writeFile(markerFile, marker);
|
|
166
|
+
await until('Runtime promotion event subscription', async () => (await network.command({ operation: 'status' })).subscribers > 0);
|
|
167
|
+
proof.checkpoints.push({ phase: 'baseline', bootstrapPid: baseline.bootstrapPid, controllerPid: baseline.controllerPid, childPid: baseline.childPid });
|
|
168
|
+
await network.command({ operation: 'publish' });
|
|
169
|
+
const current = await until('Event-driven staging, controller replacement, and native runtime healthy commit', async () => {
|
|
170
|
+
const state = journal(service.identity); const status = await service.status();
|
|
171
|
+
return state.policy?.active?.digest === view.candidate.digest && state.policy.activeHealthy
|
|
172
|
+
&& !state.policy.transaction && state.launch.pending === null && status.version === nextArtifact.version ? status : null;
|
|
173
|
+
});
|
|
174
|
+
assert.equal(current.bootstrapPid, baseline.bootstrapPid, 'A package handover must preserve the stable OS-owned launcher');
|
|
175
|
+
assert.notEqual(current.controllerPid, baseline.controllerPid); assert.notEqual(current.childPid, baseline.childPid);
|
|
176
|
+
assert.deepEqual(current.binding, baseline.binding);
|
|
177
|
+
assert.deepEqual(current.durable, baseline.durable, 'Native replacement must preserve registration and credentials byte for byte');
|
|
178
|
+
assert.equal(await fs.readFile(markerFile, 'utf8'), marker);
|
|
179
|
+
for (const name of ['repository', 'branch', 'commit']) assert.equal(current.source[name], nextArtifact.source[name]);
|
|
180
|
+
const selected = journal(service.identity).launch.current;
|
|
181
|
+
const actual = await readJson(path.resolve(selected.bin, '../../package.json'));
|
|
182
|
+
assert.equal(actual.version, nextArtifact.version);
|
|
183
|
+
assert.equal(await hashFile(path.resolve(selected.bin, '../../npm-shrinkwrap.json')), nextArtifact.lockSha512);
|
|
184
|
+
proof.checkpoints.push({ phase: 'committed-candidate', bootstrapPid: current.bootstrapPid, controllerPid: current.controllerPid, childPid: current.childPid });
|
|
185
|
+
const selectors = ['--user', service.identity.user, '--home', service.identity.home, '--amalgm-root', service.identity.amalgmRoot, '--branch', examination.lane];
|
|
186
|
+
const prefix = examination.lane === 'preview' ? ['preview'] : [];
|
|
187
|
+
const invoke = verb => runIsolated(selected.node, [selected.bin, ...prefix, 'service', verb, ...selectors], temporary, new AbortController().signal, 90_000);
|
|
188
|
+
await invoke('stop');
|
|
189
|
+
await invoke('start');
|
|
190
|
+
const restarted = await until('Native OS restart preserves committed candidate', async () => {
|
|
191
|
+
const state = journal(service.identity); const status = await service.status();
|
|
192
|
+
return status.version === nextArtifact.version && status.bootstrapPid !== current.bootstrapPid
|
|
193
|
+
&& state.policy?.active?.digest === view.candidate.digest && !state.policy.transaction && state.launch.pending === null ? status : null;
|
|
194
|
+
});
|
|
195
|
+
assert.deepEqual(restarted.durable, baseline.durable); assert.equal(await fs.readFile(markerFile, 'utf8'), marker);
|
|
196
|
+
proof.checkpoints.push({ phase: 'native-os-restart', bootstrapPid: restarted.bootstrapPid, controllerPid: restarted.controllerPid, childPid: restarted.childPid });
|
|
197
|
+
proof.passed = true;
|
|
198
|
+
});
|