@celilo/e2e 0.9.3 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/cpanel-host/cpanel-host-ca.crt +20 -0
- package/config/cpanel-host/docroot/.htaccess +14 -0
- package/config/cpanel-host/docroot/cms.html +4 -0
- package/config/cpanel-host/site-tls.crt +20 -0
- package/config/cpanel-host/site-tls.key +28 -0
- package/config/cpanel-host/site.conf +27 -0
- package/config/dns/com.zone +7 -0
- package/config/dns/knot-namecheap.conf +6 -0
- package/config/dns/tangohost.com.zone +14 -0
- package/config/resolver/unbound.conf +26 -0
- package/config/routing/fw-ext-routes.sh +16 -1
- package/config/routing/management-routes.sh +14 -4
- package/config/routing/minio-startup.sh +9 -0
- package/config/routing/public-sim-entrypoint.sh +24 -0
- package/config/routing/resolver-internal-routes.sh +10 -0
- package/docker/Dockerfile.apt-repo-sim +8 -0
- package/docker/Dockerfile.cpanel-host-sim +75 -0
- package/docker/Dockerfile.firewall +12 -0
- package/docker/Dockerfile.management +6 -0
- package/docker/Dockerfile.minio +7 -0
- package/docker/Dockerfile.npm-registry-sim +8 -0
- package/docker/Dockerfile.observer +11 -0
- package/docker/Dockerfile.registry +8 -0
- package/docker/Dockerfile.signal-cli +60 -0
- package/docker/Dockerfile.signal-release +44 -0
- package/docker/Dockerfile.signal-sim +17 -0
- package/docker/Dockerfile.target-machine +9 -0
- package/package.json +3 -3
- package/registry-server/package.json +17 -0
- package/registry-server/src/auth.test.ts +76 -0
- package/registry-server/src/auth.ts +105 -0
- package/registry-server/src/bootstrap-packaging.test.ts +71 -0
- package/registry-server/src/bootstrap.ts +250 -0
- package/registry-server/src/index.ts +16 -0
- package/registry-server/src/introspection.test.ts +247 -0
- package/registry-server/src/introspection.ts +204 -0
- package/registry-server/src/landing.ts +90 -0
- package/registry-server/src/module-owner-store.test.ts +85 -0
- package/registry-server/src/module-owner-store.ts +108 -0
- package/registry-server/src/rate-limit.test.ts +62 -0
- package/registry-server/src/rate-limit.ts +83 -0
- package/registry-server/src/scoped-token-store.test.ts +93 -0
- package/registry-server/src/scoped-token-store.ts +110 -0
- package/registry-server/src/server.test.ts +771 -0
- package/registry-server/src/server.ts +701 -0
- package/registry-server/src/storage.test.ts +148 -0
- package/registry-server/src/storage.ts +150 -0
- package/registry-server/src/validation.test.ts +86 -0
- package/registry-server/src/validation.ts +60 -0
- package/registry-server/tsconfig.json +15 -0
- package/scripts/stage-libsignal.ts +116 -0
- package/simulators/signal-cli/server.ts +328 -0
- package/src/cli/build.ts +7 -0
- package/src/cli/index.ts +6 -0
- package/src/container-manager.ts +47 -10
- package/src/docker-compose-generator.ts +120 -16
- package/src/exit-cleanup.test.ts +56 -0
- package/src/index.ts +7 -0
- package/src/network-builder.ts +66 -0
- package/src/public-sim-routes.test.ts +117 -0
- package/src/simulator-ips.ts +9 -0
- package/src/types.ts +71 -1
|
@@ -73,7 +73,16 @@ RUN apt-get update && apt-get install -y \
|
|
|
73
73
|
debian-keyring \
|
|
74
74
|
debian-archive-keyring \
|
|
75
75
|
apt-transport-https \
|
|
76
|
+
wireguard-tools \
|
|
77
|
+
wireguard-go \
|
|
76
78
|
&& rm -rf /var/lib/apt/lists/*
|
|
79
|
+
|
|
80
|
+
# A provisioned box may host the `wireguard` module (the VPN does not have to
|
|
81
|
+
# terminate on the firewall). The module's validate_config requires wg to be
|
|
82
|
+
# present and fails loudly when it is not — so the stand-in for an operator's
|
|
83
|
+
# box carries it, exactly as a real one would. Userspace implementation named
|
|
84
|
+
# for the binary Ubuntu actually installs (/usr/bin/wireguard), not the package.
|
|
85
|
+
ENV WG_QUICK_USERSPACE_IMPLEMENTATION=wireguard
|
|
77
86
|
RUN curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | \
|
|
78
87
|
gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
|
|
79
88
|
RUN curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | \
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celilo/e2e",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.2",
|
|
4
4
|
"description": "E2E test infrastructure for Celilo-deployed applications. Provides a simulated internet with DNS hierarchy, ACME server, firewalls, and target machines in Docker.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.ts",
|
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
"README.md"
|
|
38
38
|
],
|
|
39
39
|
"dependencies": {
|
|
40
|
-
"@celilo/cli-display": "
|
|
41
|
-
"@celilo/event-bus": "
|
|
40
|
+
"@celilo/cli-display": "^0.1.9",
|
|
41
|
+
"@celilo/event-bus": "^0.1.9",
|
|
42
42
|
"yaml": "^2.8.0",
|
|
43
43
|
"zod": "^3.24.1"
|
|
44
44
|
},
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@celilo/registry-server",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"private": true,
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "src/index.ts",
|
|
7
|
+
"scripts": {
|
|
8
|
+
"start": "bun run src/index.ts",
|
|
9
|
+
"dev": "bun --watch src/index.ts",
|
|
10
|
+
"build:binary": "bun build --compile --target=bun-linux-x64 --outfile=dist/celilo-registry-server src/index.ts",
|
|
11
|
+
"test": "bun test"
|
|
12
|
+
},
|
|
13
|
+
"devDependencies": {
|
|
14
|
+
"bun-types": "latest",
|
|
15
|
+
"typescript": "^5"
|
|
16
|
+
}
|
|
17
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { ADMIN_SCOPE, TokenAuth, hashToken } from './auth';
|
|
3
|
+
|
|
4
|
+
describe('TokenAuth (package-scoped — ISS-0140)', () => {
|
|
5
|
+
test('empty specs → no tokens; nothing authorizes', () => {
|
|
6
|
+
const auth = new TokenAuth([]);
|
|
7
|
+
expect(auth.hasTokens()).toBe(false);
|
|
8
|
+
expect(auth.authorize('anything', 'caddy')).toBe(false);
|
|
9
|
+
expect(auth.isAdmin('anything')).toBe(false);
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
test('admin-scoped token publishes ANY package', () => {
|
|
13
|
+
const auth = new TokenAuth([{ token: 'admin-tok', scope: ADMIN_SCOPE }]);
|
|
14
|
+
expect(auth.authorize('admin-tok', 'caddy')).toBe(true);
|
|
15
|
+
expect(auth.authorize('admin-tok', 'lunacycle')).toBe(true);
|
|
16
|
+
expect(auth.isAdmin('admin-tok')).toBe(true);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
test('package-scoped token publishes ONLY its package', () => {
|
|
20
|
+
const auth = new TokenAuth([{ token: 'luna-tok', scope: 'lunacycle' }]);
|
|
21
|
+
expect(auth.authorize('luna-tok', 'lunacycle')).toBe(true);
|
|
22
|
+
expect(auth.authorize('luna-tok', 'caddy')).toBe(false);
|
|
23
|
+
// A scoped token is NOT an admin token — it cannot mint.
|
|
24
|
+
expect(auth.isAdmin('luna-tok')).toBe(false);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test('unknown token never authorizes', () => {
|
|
28
|
+
const auth = new TokenAuth([{ token: 'known', scope: 'lunacycle' }]);
|
|
29
|
+
expect(auth.authorize('unknown', 'lunacycle')).toBe(false);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
test('tokens are stored hashed, not in cleartext', () => {
|
|
33
|
+
const auth = new TokenAuth([{ token: 'secret-token', scope: ADMIN_SCOPE }]);
|
|
34
|
+
expect(JSON.stringify(auth)).not.toContain('secret-token');
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test('tolerates a Bearer prefix and surrounding whitespace', () => {
|
|
38
|
+
const auth = new TokenAuth([{ token: 'clean', scope: ADMIN_SCOPE }]);
|
|
39
|
+
expect(auth.authorize(' clean ', 'caddy')).toBe(true);
|
|
40
|
+
expect(auth.authorize('Bearer clean', 'caddy')).toBe(true);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test('empty header rejected', () => {
|
|
44
|
+
const auth = new TokenAuth([{ token: 'secret', scope: ADMIN_SCOPE }]);
|
|
45
|
+
expect(auth.authorize('', 'caddy')).toBe(false);
|
|
46
|
+
expect(auth.scopeOf('')).toBe(null);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test('addHashed / removeHashed manage minted tokens at runtime', () => {
|
|
50
|
+
const auth = new TokenAuth([]);
|
|
51
|
+
const hash = hashToken('minted-tok');
|
|
52
|
+
auth.addHashed(hash, 'lunacycle');
|
|
53
|
+
expect(auth.authorize('minted-tok', 'lunacycle')).toBe(true);
|
|
54
|
+
auth.removeHashed(hash);
|
|
55
|
+
expect(auth.authorize('minted-tok', 'lunacycle')).toBe(false);
|
|
56
|
+
});
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
describe('TokenAuth.fromEnv (PUBLISH_TOKENS format)', () => {
|
|
60
|
+
test('bare line → admin scope; "token pkg" line → scoped', () => {
|
|
61
|
+
process.env.PUBLISH_TOKENS = 'admin-tok\nluna-tok lunacycle\n';
|
|
62
|
+
const auth = TokenAuth.fromEnv();
|
|
63
|
+
expect(auth.isAdmin('admin-tok')).toBe(true);
|
|
64
|
+
expect(auth.authorize('luna-tok', 'lunacycle')).toBe(true);
|
|
65
|
+
expect(auth.authorize('luna-tok', 'caddy')).toBe(false);
|
|
66
|
+
process.env.PUBLISH_TOKENS = '';
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
test('blank lines ignored', () => {
|
|
70
|
+
process.env.PUBLISH_TOKENS = '\n \nt1\n';
|
|
71
|
+
const auth = TokenAuth.fromEnv();
|
|
72
|
+
expect(auth.hasTokens()).toBe(true);
|
|
73
|
+
expect(auth.isAdmin('t1')).toBe(true);
|
|
74
|
+
process.env.PUBLISH_TOKENS = '';
|
|
75
|
+
});
|
|
76
|
+
});
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Package-scoped token auth (build-bus Phase 3, ISS-0140).
|
|
5
|
+
*
|
|
6
|
+
* Each token carries a SCOPE — either a single package name (the token may
|
|
7
|
+
* publish only that module) or the admin scope `*` (publish anything + mint
|
|
8
|
+
* scoped tokens). Tokens are stored SHA-256 hashed; cleartext never lives in
|
|
9
|
+
* the auth set once the server starts.
|
|
10
|
+
*
|
|
11
|
+
* PUBLISH_TOKENS env (newline-separated), per line:
|
|
12
|
+
* <token> → admin scope `*` (publish any package, mint tokens)
|
|
13
|
+
* <token> <package> → scoped to that one package
|
|
14
|
+
*
|
|
15
|
+
* A bare `<token>` line is the admin/bootstrap token — this preserves the
|
|
16
|
+
* pre-scoping behavior (any configured token publishes anything) AND is the
|
|
17
|
+
* privilege that the `registry_publish` capability uses to mint per-repo
|
|
18
|
+
* scoped tokens. Minted scoped tokens are added at runtime via {@link addHashed}
|
|
19
|
+
* (loaded from the persisted store) and removed via {@link removeHashed}.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/** Admin scope: publish any package + mint scoped tokens. */
|
|
23
|
+
export const ADMIN_SCOPE = '*';
|
|
24
|
+
|
|
25
|
+
export interface TokenSpec {
|
|
26
|
+
token: string;
|
|
27
|
+
/** Package name, or {@link ADMIN_SCOPE}. */
|
|
28
|
+
scope: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** SHA-256 of a token (trimmed). The persisted scoped-token store hashes too. */
|
|
32
|
+
export function hashToken(token: string): string {
|
|
33
|
+
return createHash('sha256').update(token.trim()).digest('hex');
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Strip an optional `Bearer ` prefix and surrounding whitespace from a header. */
|
|
37
|
+
export function bearerToken(header: string): string {
|
|
38
|
+
return header.replace(/^Bearer\s+/i, '').trim();
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export class TokenAuth {
|
|
42
|
+
/** hash → scope. */
|
|
43
|
+
private readonly scopeByHash = new Map<string, string>();
|
|
44
|
+
|
|
45
|
+
constructor(specs: TokenSpec[]) {
|
|
46
|
+
for (const s of specs) this.addRaw(s.token, s.scope);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Parse the `PUBLISH_TOKENS` env format into the scoped auth set. Each line is
|
|
51
|
+
* `<token>` (admin scope) or `<token> <package>` (scoped to one package).
|
|
52
|
+
*/
|
|
53
|
+
static fromEnv(): TokenAuth {
|
|
54
|
+
const raw = process.env.PUBLISH_TOKENS ?? '';
|
|
55
|
+
const specs: TokenSpec[] = [];
|
|
56
|
+
for (const line of raw.split('\n')) {
|
|
57
|
+
const trimmed = line.trim();
|
|
58
|
+
if (!trimmed) continue;
|
|
59
|
+
const [token, scope] = trimmed.split(/\s+/, 2);
|
|
60
|
+
specs.push({ token, scope: scope || ADMIN_SCOPE });
|
|
61
|
+
}
|
|
62
|
+
return new TokenAuth(specs);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Add a cleartext token (hashes it). Empty tokens are ignored. */
|
|
66
|
+
addRaw(rawToken: string, scope: string): void {
|
|
67
|
+
const t = rawToken.trim();
|
|
68
|
+
if (!t) return;
|
|
69
|
+
this.scopeByHash.set(hashToken(t), scope.trim() || ADMIN_SCOPE);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Add an already-hashed token (used when loading the persisted minted-token store). */
|
|
73
|
+
addHashed(hash: string, scope: string): void {
|
|
74
|
+
this.scopeByHash.set(hash, scope);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Remove a token by its hash (token revocation). */
|
|
78
|
+
removeHashed(hash: string): void {
|
|
79
|
+
this.scopeByHash.delete(hash);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** If no tokens are configured, the server treats every write as unauthorized. */
|
|
83
|
+
hasTokens(): boolean {
|
|
84
|
+
return this.scopeByHash.size > 0;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** The scope of a token (from an Authorization header value), or null if unknown. */
|
|
88
|
+
scopeOf(header: string): string | null {
|
|
89
|
+
const t = bearerToken(header);
|
|
90
|
+
if (!t) return null;
|
|
91
|
+
return this.scopeByHash.get(hashToken(t)) ?? null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** True when the token may publish/yank the given package (admin or exact scope). */
|
|
95
|
+
authorize(header: string, pkg: string): boolean {
|
|
96
|
+
const scope = this.scopeOf(header);
|
|
97
|
+
if (scope === null) return false;
|
|
98
|
+
return scope === ADMIN_SCOPE || scope === pkg;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** True when the token is an admin token (scope `*`) — required to mint scoped tokens. */
|
|
102
|
+
isAdmin(header: string): boolean {
|
|
103
|
+
return this.scopeOf(header) === ADMIN_SCOPE;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lockstep gate (ISS-0046): the registry-server's on-demand `walkDir` packager
|
|
3
|
+
* can't import the canonical node_modules-bundling rule at runtime (this server
|
|
4
|
+
* ships standalone, with no @celilo deps), so it carries a structural copy.
|
|
5
|
+
* This test imports the canonical rule directly (zero-dependency module, safe to
|
|
6
|
+
* pull in at test time from the workspace) and asserts walkDir's actual output
|
|
7
|
+
* conforms to it — so a drift in either copy fails here instead of shipping a
|
|
8
|
+
* broken `.netapp`.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
12
|
+
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
13
|
+
import { tmpdir } from 'node:os';
|
|
14
|
+
import { join } from 'node:path';
|
|
15
|
+
// Canonical rule lives in apps/celilo; importing it cross-package is fine in a
|
|
16
|
+
// workspace test (it's not bundled into the standalone registry-sim image).
|
|
17
|
+
import { includeNodeModulesPath } from '../../../apps/celilo/src/module/packaging/package-rules';
|
|
18
|
+
import { walkDir } from './bootstrap';
|
|
19
|
+
|
|
20
|
+
describe('bootstrap walkDir stays in lockstep with the canonical packaging rule', () => {
|
|
21
|
+
let dir: string;
|
|
22
|
+
|
|
23
|
+
// Representative node_modules paths spanning both branches of the rule.
|
|
24
|
+
const nmFiles = [
|
|
25
|
+
'scripts/node_modules/@celilo/capabilities/src/dns-internal.ts',
|
|
26
|
+
'scripts/node_modules/tldts/index.js',
|
|
27
|
+
'scripts/node_modules/drizzle-orm/index.js',
|
|
28
|
+
'scripts/node_modules/.bin/tldts',
|
|
29
|
+
'node_modules/@celilo/capabilities/index.js',
|
|
30
|
+
'node_modules/lodash/index.js',
|
|
31
|
+
'node_modules/@celilo/e2e/index.js',
|
|
32
|
+
];
|
|
33
|
+
|
|
34
|
+
function write(rel: string): void {
|
|
35
|
+
const full = join(dir, rel);
|
|
36
|
+
mkdirSync(join(full, '..'), { recursive: true });
|
|
37
|
+
writeFileSync(full, 'x');
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
beforeEach(() => {
|
|
41
|
+
dir = mkdtempSync(join(tmpdir(), 'celilo-bootstrap-rule-'));
|
|
42
|
+
write('manifest.yml');
|
|
43
|
+
write('scripts/on-install.ts');
|
|
44
|
+
for (const f of nmFiles) write(f);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
afterEach(() => {
|
|
48
|
+
try {
|
|
49
|
+
rmSync(dir, { recursive: true, force: true });
|
|
50
|
+
} catch {
|
|
51
|
+
/* ignore */
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it("walkDir's node_modules decisions match includeNodeModulesPath", () => {
|
|
56
|
+
const bundled = new Set(walkDir(dir));
|
|
57
|
+
for (const rel of nmFiles) {
|
|
58
|
+
expect(bundled.has(rel)).toBe(includeNodeModulesPath(rel));
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('bundles scripts deps (tldts), drops .bin shims and non-scripts cruft', () => {
|
|
63
|
+
const bundled = new Set(walkDir(dir));
|
|
64
|
+
expect(bundled.has('scripts/node_modules/tldts/index.js')).toBe(true);
|
|
65
|
+
expect(bundled.has('scripts/node_modules/drizzle-orm/index.js')).toBe(true);
|
|
66
|
+
expect(bundled.has('scripts/node_modules/.bin/tldts')).toBe(false);
|
|
67
|
+
expect(bundled.has('node_modules/lodash/index.js')).toBe(false);
|
|
68
|
+
expect(bundled.has('node_modules/@celilo/e2e/index.js')).toBe(false);
|
|
69
|
+
expect(bundled.has('node_modules/@celilo/capabilities/index.js')).toBe(true);
|
|
70
|
+
});
|
|
71
|
+
});
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bootstrap mode — optional behavior for the e2e stand-in registry.
|
|
3
|
+
*
|
|
4
|
+
* When BOOTSTRAP_MODULES_DIR is set, the server also serves modules whose
|
|
5
|
+
* source lives in that directory. Each immediate subdirectory containing a
|
|
6
|
+
* manifest.yml is treated as a module; its manifest's `id` and `version`
|
|
7
|
+
* become the sparse index entry. On download, the directory is packaged
|
|
8
|
+
* into a .netapp tarball on the fly (cached on disk).
|
|
9
|
+
*
|
|
10
|
+
* This preserves the existing packages/e2e/docker/Dockerfile.registry
|
|
11
|
+
* behavior where tests can import any module from the repo without having
|
|
12
|
+
* to explicitly publish each dependency first.
|
|
13
|
+
*
|
|
14
|
+
* In production, BOOTSTRAP_MODULES_DIR is unset and this whole layer is
|
|
15
|
+
* inert — storage.ts is the only code that touches disk.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
|
|
19
|
+
import { basename, dirname, join, relative } from 'node:path';
|
|
20
|
+
import { createHash } from 'node:crypto';
|
|
21
|
+
import type { IndexEntry } from './storage';
|
|
22
|
+
|
|
23
|
+
export interface BootstrapEntry {
|
|
24
|
+
name: string;
|
|
25
|
+
version: string;
|
|
26
|
+
/** Source directory to package on demand (source mode). */
|
|
27
|
+
sourceDir?: string;
|
|
28
|
+
/** Pre-built .netapp to serve directly (upload mode). */
|
|
29
|
+
netappPath?: string;
|
|
30
|
+
/**
|
|
31
|
+
* Manifest's `description` field, when present. Used by the search
|
|
32
|
+
* endpoint to populate the module-list payload — see
|
|
33
|
+
* apps/celilo/designs/REGISTRY_BROWSE_UI.md (Phase 2 step 0).
|
|
34
|
+
*/
|
|
35
|
+
description?: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Scan a directory for modules. Each subdirectory is a module iff it contains
|
|
40
|
+
* a manifest.yml. Manifest parsing is line-regex (not full YAML) so we don't
|
|
41
|
+
* need a YAML dependency at runtime — this matches packages/e2e/src/registry-server.ts.
|
|
42
|
+
*/
|
|
43
|
+
export function scanBootstrapDir(bootstrapDir: string): Map<string, BootstrapEntry> {
|
|
44
|
+
const entries = new Map<string, BootstrapEntry>();
|
|
45
|
+
if (!existsSync(bootstrapDir)) return entries;
|
|
46
|
+
|
|
47
|
+
for (const entry of readdirSync(bootstrapDir, { withFileTypes: true })) {
|
|
48
|
+
if (!entry.isDirectory()) continue;
|
|
49
|
+
const manifestPath = join(bootstrapDir, entry.name, 'manifest.yml');
|
|
50
|
+
if (!existsSync(manifestPath)) continue;
|
|
51
|
+
try {
|
|
52
|
+
const yaml = readFileSync(manifestPath, 'utf-8');
|
|
53
|
+
const idMatch = yaml.match(/^id:\s*['"]?([^\s'"#]+)['"]?/m);
|
|
54
|
+
const versionMatch = yaml.match(/^version:\s*['"]?([^\s'"#]+)['"]?/m);
|
|
55
|
+
// Description is the rest-of-line up to (but not including) the
|
|
56
|
+
// newline. The manifest schema allows multi-line scalars, but
|
|
57
|
+
// existing modules use the single-line form so the line-regex is
|
|
58
|
+
// sufficient. We strip surrounding quotes if present.
|
|
59
|
+
const descriptionMatch = yaml.match(/^description:\s*(.+?)\s*$/m);
|
|
60
|
+
const name = idMatch?.[1]?.trim() ?? entry.name;
|
|
61
|
+
const version = versionMatch?.[1]?.trim() ?? '0.0.0';
|
|
62
|
+
const description = descriptionMatch?.[1]?.trim().replace(/^['"]|['"]$/g, '');
|
|
63
|
+
entries.set(name, {
|
|
64
|
+
name,
|
|
65
|
+
version,
|
|
66
|
+
sourceDir: join(bootstrapDir, entry.name),
|
|
67
|
+
description,
|
|
68
|
+
});
|
|
69
|
+
} catch {
|
|
70
|
+
// skip malformed manifests — matches e2e server behavior
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return entries;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Scan a directory for pre-built .netapp files. Filename (minus suffix) is
|
|
78
|
+
* the module name. Version defaults to the filename's version suffix or to
|
|
79
|
+
* 0.0.0+1 — real version data is in the archive's manifest but parsing that
|
|
80
|
+
* at startup is overkill for the e2e bootstrap use case.
|
|
81
|
+
*
|
|
82
|
+
* Used by the shared-infra stand-in: the e2e harness's publishModule helper
|
|
83
|
+
* copies .netapps into BOOTSTRAP_UPLOADS_DIR with `docker cp`, and the
|
|
84
|
+
* server picks them up without needing the full HTTP publish protocol.
|
|
85
|
+
*/
|
|
86
|
+
export function scanUploadsDir(uploadsDir: string): Map<string, BootstrapEntry> {
|
|
87
|
+
const entries = new Map<string, BootstrapEntry>();
|
|
88
|
+
if (!existsSync(uploadsDir)) return entries;
|
|
89
|
+
|
|
90
|
+
for (const file of readdirSync(uploadsDir)) {
|
|
91
|
+
if (!file.endsWith('.netapp')) continue;
|
|
92
|
+
const name = file.slice(0, -7);
|
|
93
|
+
entries.set(name, {
|
|
94
|
+
name,
|
|
95
|
+
version: '0.0.0',
|
|
96
|
+
netappPath: join(uploadsDir, file),
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
return entries;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Render a bootstrap entry as an IndexEntry suitable for the sparse index. */
|
|
103
|
+
export function bootstrapIndexEntry(entry: BootstrapEntry): IndexEntry {
|
|
104
|
+
const vers = entry.version.includes('+') ? entry.version : `${entry.version}+1`;
|
|
105
|
+
return {
|
|
106
|
+
name: entry.name,
|
|
107
|
+
vers,
|
|
108
|
+
deps: [],
|
|
109
|
+
cksum: 'bootstrap',
|
|
110
|
+
yanked: false,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Walk a directory, skipping hidden entries, *.test.ts files, the
|
|
116
|
+
* module's e2e/ tree (tests + their deps don't ship with the module),
|
|
117
|
+
* and most of node_modules.
|
|
118
|
+
*
|
|
119
|
+
* `scripts/node_modules` (the module's hook runtime) is bundled IN FULL
|
|
120
|
+
* (ISS-0046): hook scripts run on the target, where the package registry can
|
|
121
|
+
* be unreachable, so the `.netapp` must carry @celilo/capabilities AND its
|
|
122
|
+
* third-party deps (tldts, drizzle-orm, …). Any OTHER node_modules ships only
|
|
123
|
+
* `@celilo/capabilities` — the framework SDK the module was authored against.
|
|
124
|
+
*
|
|
125
|
+
* This is the structural twin of the canonical `includeNodeModulesPath` rule in
|
|
126
|
+
* apps/celilo/src/module/packaging/package-rules.ts. We can't import that at
|
|
127
|
+
* runtime (this server ships in a standalone Docker image with no @celilo deps),
|
|
128
|
+
* so the two are held in lockstep by bootstrap-packaging.test.ts, which asserts
|
|
129
|
+
* `walkDir`'s output conforms to the canonical rule. `walkDir` is exported for
|
|
130
|
+
* that test.
|
|
131
|
+
*/
|
|
132
|
+
export function walkDir(dir: string, base = dir, inNodeModules = false): string[] {
|
|
133
|
+
const results: string[] = [];
|
|
134
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
135
|
+
if (entry.name.startsWith('.')) continue;
|
|
136
|
+
if (entry.name.endsWith('.test.ts')) continue;
|
|
137
|
+
// The module's own tsconfig.json is dev-only (CI typechecks hooks with it);
|
|
138
|
+
// build.ts drops it too. A dependency's tsconfig.json inside node_modules is
|
|
139
|
+
// that package's published content and stays — build.ts keeps those as well.
|
|
140
|
+
if (entry.name === 'tsconfig.json' && !inNodeModules) continue;
|
|
141
|
+
|
|
142
|
+
const fullPath = join(dir, entry.name);
|
|
143
|
+
|
|
144
|
+
// Module's e2e/ directory at the root is tests + their devDependencies.
|
|
145
|
+
// Detected by being a sibling of manifest.yml.
|
|
146
|
+
if (entry.name === 'e2e' && existsSync(join(dir, 'manifest.yml'))) {
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
if (entry.name === 'node_modules') {
|
|
151
|
+
if (basename(dir) === 'scripts') {
|
|
152
|
+
// The hook-script runtime closure — bundle it whole (incl. nested deps).
|
|
153
|
+
// walkAll keeps the hidden-entry / *.test.ts skips, so .bin shims drop.
|
|
154
|
+
results.push(...walkAll(fullPath, base));
|
|
155
|
+
} else {
|
|
156
|
+
// Other node_modules: collect only @celilo/capabilities.
|
|
157
|
+
const capDir = join(fullPath, '@celilo', 'capabilities');
|
|
158
|
+
if (existsSync(capDir)) {
|
|
159
|
+
results.push(...walkDir(capDir, base, true));
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
if (entry.isDirectory()) {
|
|
166
|
+
results.push(...walkDir(fullPath, base, inNodeModules));
|
|
167
|
+
} else {
|
|
168
|
+
results.push(relative(base, fullPath));
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
return results;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Collect every file under `dir` (recursing into nested node_modules too),
|
|
176
|
+
* skipping only hidden entries (`.bin` symlink shims) and `*.test.ts`. Used to
|
|
177
|
+
* bundle a module's full `scripts/node_modules` runtime closure.
|
|
178
|
+
*/
|
|
179
|
+
function walkAll(dir: string, base: string): string[] {
|
|
180
|
+
const results: string[] = [];
|
|
181
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
182
|
+
if (entry.name.startsWith('.')) continue;
|
|
183
|
+
if (entry.name.endsWith('.test.ts')) continue;
|
|
184
|
+
const fullPath = join(dir, entry.name);
|
|
185
|
+
if (entry.isDirectory()) {
|
|
186
|
+
results.push(...walkAll(fullPath, base));
|
|
187
|
+
} else {
|
|
188
|
+
results.push(relative(base, fullPath));
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return results;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Package a module directory into a .netapp tarball. Cached by module name
|
|
196
|
+
* in cacheDir; cache is keyed by name only (assume the source is immutable
|
|
197
|
+
* for the lifetime of the server — fine for e2e).
|
|
198
|
+
*/
|
|
199
|
+
export async function packageBootstrapModule(
|
|
200
|
+
entry: BootstrapEntry,
|
|
201
|
+
cacheDir: string,
|
|
202
|
+
): Promise<string> {
|
|
203
|
+
// Upload-mode entries are already .netapp files — serve as-is.
|
|
204
|
+
if (entry.netappPath) return entry.netappPath;
|
|
205
|
+
const sourceDir = entry.sourceDir;
|
|
206
|
+
if (!sourceDir) {
|
|
207
|
+
throw new Error(`bootstrap entry ${entry.name} has neither sourceDir nor netappPath`);
|
|
208
|
+
}
|
|
209
|
+
mkdirSync(cacheDir, { recursive: true });
|
|
210
|
+
const cachePath = join(cacheDir, `${entry.name}.netapp`);
|
|
211
|
+
if (existsSync(cachePath)) return cachePath;
|
|
212
|
+
|
|
213
|
+
const filePaths = walkDir(sourceDir);
|
|
214
|
+
const fileHashes: Record<string, string> = {};
|
|
215
|
+
for (const relPath of filePaths) {
|
|
216
|
+
const data = readFileSync(join(sourceDir, relPath));
|
|
217
|
+
fileHashes[relPath] = createHash('sha256').update(data).digest('hex');
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const checksumsJson = JSON.stringify(
|
|
221
|
+
{ version: '1', generated: new Date().toISOString(), files: fileHashes },
|
|
222
|
+
null,
|
|
223
|
+
2,
|
|
224
|
+
);
|
|
225
|
+
|
|
226
|
+
// Source dir may be read-only (volume-mounted) — stage in /tmp first.
|
|
227
|
+
const stagingDir = join('/tmp', `celilo-bootstrap-${entry.name}-${Date.now()}`);
|
|
228
|
+
mkdirSync(stagingDir, { recursive: true });
|
|
229
|
+
try {
|
|
230
|
+
for (const relPath of filePaths) {
|
|
231
|
+
mkdirSync(dirname(join(stagingDir, relPath)), { recursive: true });
|
|
232
|
+
writeFileSync(join(stagingDir, relPath), readFileSync(join(sourceDir, relPath)));
|
|
233
|
+
}
|
|
234
|
+
writeFileSync(join(stagingDir, 'checksums.json'), checksumsJson);
|
|
235
|
+
|
|
236
|
+
const proc = Bun.spawn(['tar', '-czf', cachePath, '-C', stagingDir, '.'], {
|
|
237
|
+
stderr: 'inherit',
|
|
238
|
+
});
|
|
239
|
+
const exitCode = await proc.exited;
|
|
240
|
+
if (exitCode !== 0) throw new Error(`tar exited with code ${exitCode}`);
|
|
241
|
+
} finally {
|
|
242
|
+
try {
|
|
243
|
+
Bun.spawnSync(['rm', '-rf', stagingDir]);
|
|
244
|
+
} catch {
|
|
245
|
+
// best-effort cleanup
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
return cachePath;
|
|
250
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export { RegistryStorage } from './storage';
|
|
2
|
+
export type { IndexEntry, ModuleVersion } from './storage';
|
|
3
|
+
export { TokenAuth } from './auth';
|
|
4
|
+
export { startServer, startFromEnv } from './server';
|
|
5
|
+
export {
|
|
6
|
+
scanBootstrapDir,
|
|
7
|
+
bootstrapIndexEntry,
|
|
8
|
+
packageBootstrapModule,
|
|
9
|
+
} from './bootstrap';
|
|
10
|
+
export type { BootstrapEntry } from './bootstrap';
|
|
11
|
+
|
|
12
|
+
import { startFromEnv } from './server';
|
|
13
|
+
|
|
14
|
+
if (import.meta.main) {
|
|
15
|
+
startFromEnv();
|
|
16
|
+
}
|