@cldmv/gh-broker 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +134 -40
- package/config.example.json +21 -8
- package/config.sample.json +45 -0
- package/deploy/README.md +73 -15
- package/deploy/install.mjs +166 -101
- package/deploy/install.sh +0 -0
- package/deploy/lib/config-merge.mjs +174 -16
- package/deploy/lib/credentials.mjs +143 -0
- package/deploy/lib/install-args.mjs +144 -0
- package/deploy/lib/npm-setup.mjs +164 -0
- package/deploy/lib/secret-store.mjs +20 -2
- package/deploy/lib/system.mjs +26 -2
- package/dist/index.mjs +1638 -202
- package/dist/index.mjs.map +4 -4
- package/docs/TOOLS.md +192 -1
- package/docs/changelog/v0/v0.8.0.md +115 -0
- package/docs/deploy-architecture.md +13 -11
- package/package.json +7 -7
- package/src/availability.ts +42 -0
- package/src/catalog.ts +18 -1
- package/src/config.ts +29 -2
- package/src/router.ts +43 -2
- package/src/secrets.ts +34 -4
- package/src/types.ts +25 -1
- package/src/verbs/ghcr/archive.ts +314 -0
- package/src/verbs/ghcr/config.ts +142 -0
- package/src/verbs/ghcr/jobs.ts +141 -0
- package/src/verbs/ghcr/push.ts +192 -0
- package/src/verbs/ghcr/registry.ts +275 -0
- package/src/verbs/ghcr/tar.ts +203 -0
- package/src/verbs/ghcr_push.ts +264 -0
- package/src/verbs/npm-registry.ts +168 -0
- package/src/verbs/npm-tarball.ts +76 -0
- package/src/verbs/npm_scaffold.ts +325 -0
- package/src/verbs/release-analysis.ts +3 -1
- package/tests/catalog.test.vitest.mts +55 -1
- package/tests/config-merge.test.vitest.mts +275 -8
- package/tests/config.test.vitest.mts +75 -1
- package/tests/credentials.test.vitest.mts +127 -0
- package/tests/ghcr-archive.test.vitest.mts +439 -0
- package/tests/ghcr-config.test.vitest.mts +96 -0
- package/tests/ghcr-push.test.vitest.mts +526 -0
- package/tests/ghcr-registry.test.vitest.mts +206 -0
- package/tests/helpers/ghcr-fixtures.mts +373 -0
- package/tests/install-args.test.vitest.mts +152 -0
- package/tests/issue-labels.test.vitest.mts +15 -0
- package/tests/npm-scaffold.test.vitest.mts +479 -0
- package/tests/npm-setup.test.vitest.mts +196 -0
- package/tests/paths.test.vitest.mts +5 -0
- package/tests/prompt.test.vitest.mts +128 -0
- package/tests/release-prs.test.vitest.mts +10 -0
- package/tests/router.test.vitest.mts +108 -2
- package/tests/secret-store.test.vitest.mts +64 -3
- package/tests/secrets.test.vitest.mts +53 -0
- package/tests/security-alerts.test.vitest.mts +10 -0
- package/tests/server.test.vitest.mts +187 -7
- package/tests/smoke.test.vitest.mts +177 -0
- package/tests/system.test.vitest.mts +355 -0
- package/tests/verb-helpers.test.vitest.mts +46 -0
- package/.configs/fix-headers.json +0 -3
- package/.configs/vitest.config.mjs +0 -44
- package/.github/dependabot.yml +0 -111
- package/.github/workflows/branch-retention.yml +0 -44
- package/.github/workflows/bundle-size.yml +0 -52
- package/.github/workflows/ci.yml +0 -389
- package/.github/workflows/cla.yml +0 -66
- package/.github/workflows/codeql.yml +0 -87
- package/.github/workflows/dependabot-auto-merge.yml +0 -62
- package/.github/workflows/dependabot-recreate.yml +0 -67
- package/.github/workflows/dependency-review.yml +0 -46
- package/.github/workflows/feature-pr.yml +0 -75
- package/.github/workflows/hotfix-redirector.yml +0 -67
- package/.github/workflows/hotfixes-release.yml +0 -70
- package/.github/workflows/labeler.yml +0 -48
- package/.github/workflows/master-commit-audit.yml +0 -56
- package/.github/workflows/member-auto-merge.yml +0 -75
- package/.github/workflows/next-release.yml +0 -76
- package/.github/workflows/next-reset.yml +0 -50
- package/.github/workflows/pr-notify.yml +0 -44
- package/.github/workflows/pr-title-normalizer.yml +0 -52
- package/.github/workflows/provenance.yml +0 -51
- package/.github/workflows/publish.yml +0 -146
- package/.github/workflows/release-merge.yml +0 -104
- package/.github/workflows/release-notify.yml +0 -41
- package/.github/workflows/scorecard.yml +0 -64
- package/.github/workflows/stale.yml +0 -50
- package/.github/workflows/tag-health.yml +0 -70
- package/.github/workflows/update-major-version-tags.yml +0 -94
- package/.github/workflows/v4-bootstrap.yml +0 -115
- package/.github/workflows/welcome.yml +0 -42
package/src/router.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* @Email: <Shinrai@users.noreply.github.com>
|
|
8
8
|
* -----
|
|
9
9
|
* @Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
|
|
10
|
-
* @Last modified time: 2026-10-
|
|
10
|
+
* @Last modified time: 2026-10-31T12:12:11-07:00 (1793473931)
|
|
11
11
|
* -----
|
|
12
12
|
* @Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
|
|
13
13
|
*
|
|
@@ -19,6 +19,7 @@ import type { AppAuth } from "./appauth.ts";
|
|
|
19
19
|
import { Octokit } from "@octokit/core";
|
|
20
20
|
import { indexTools } from "./config.ts";
|
|
21
21
|
import { audit } from "./audit.ts";
|
|
22
|
+
import { unavailableReason } from "./availability.ts";
|
|
22
23
|
// DEFERRED: link_issue is App-token-only and not wired in — see docs.
|
|
23
24
|
// import { linkIssue, type LinkIssueInput } from "./verbs/link_issue.ts";
|
|
24
25
|
import {
|
|
@@ -32,6 +33,11 @@ import { reportSecurityAlerts, type ReportSecurityAlertsInput } from "./verbs/re
|
|
|
32
33
|
import { reportReleasePrs, type ReportReleasePrsInput } from "./verbs/report_release_prs.ts";
|
|
33
34
|
import { issueLabels, type IssueLabelsInput } from "./verbs/issue_labels.ts";
|
|
34
35
|
import { reportCiStatus, ciStatusForHeadSha, type ReportCiStatusInput, type FailedJobLogsFetcher } from "./verbs/report_ci_status.ts";
|
|
36
|
+
import { npmScaffold } from "./verbs/npm_scaffold.ts";
|
|
37
|
+
import { NpmRegistry } from "./verbs/npm-registry.ts";
|
|
38
|
+
import { GhcrService } from "./verbs/ghcr_push.ts";
|
|
39
|
+
|
|
40
|
+
const GHCR_TOOLS = new Set(["ghcr_push", "ghcr_push_status", "ghcr_tags"]);
|
|
35
41
|
|
|
36
42
|
/** What the router needs from the upstream (hosted-MCP) client pool. */
|
|
37
43
|
export interface Upstream {
|
|
@@ -48,6 +54,8 @@ export class Router {
|
|
|
48
54
|
private readonly upstream: Upstream;
|
|
49
55
|
private readonly appAuth?: AppAuth;
|
|
50
56
|
private readonly rules: Map<string, ToolRule>;
|
|
57
|
+
/** GHCR verbs run on config.ghcr's own credential, not the owner's GitHub PAT (created on first use). */
|
|
58
|
+
private ghcr?: GhcrService;
|
|
51
59
|
|
|
52
60
|
constructor(config: Config, secrets: SecretStore, upstream: Upstream, appAuth?: AppAuth) {
|
|
53
61
|
this.config = config;
|
|
@@ -59,7 +67,7 @@ export class Router {
|
|
|
59
67
|
|
|
60
68
|
/** Tools advertised to the agent (used to build tools/list). */
|
|
61
69
|
advertisedTools(): ToolRule[] {
|
|
62
|
-
return this.config.tools;
|
|
70
|
+
return this.config.tools.filter((rule) => !unavailableReason(rule, this.config, this.secrets));
|
|
63
71
|
}
|
|
64
72
|
|
|
65
73
|
async call(tool: string, args: Record<string, unknown>): Promise<CallToolResult> {
|
|
@@ -68,6 +76,15 @@ export class Router {
|
|
|
68
76
|
audit({ tool, decision: "deny", detail: "not in allowlist" });
|
|
69
77
|
throw new Error(`tool "${tool}" is not allowed`);
|
|
70
78
|
}
|
|
79
|
+
// An optional integration that isn't set up hides its tools: refuse them the same way.
|
|
80
|
+
const unavailable = unavailableReason(rule, this.config, this.secrets);
|
|
81
|
+
if (unavailable) {
|
|
82
|
+
audit({ tool, decision: "deny", detail: `unavailable: ${unavailable}` });
|
|
83
|
+
throw new Error(`tool "${tool}" is not available on this broker: ${unavailable}`);
|
|
84
|
+
}
|
|
85
|
+
// The GHCR verbs authenticate with config.ghcr's own credential, never an owner's GitHub
|
|
86
|
+
// PAT, so they don't need (or touch) a config.owners entry.
|
|
87
|
+
if (GHCR_TOOLS.has(tool)) return this.callGhcr(tool, args, rule);
|
|
71
88
|
// Owner-less reads (global advisory DB, get_me, notifications) fall back to
|
|
72
89
|
// the configured defaultOwner's READ PAT. Writes never fall back — a write
|
|
73
90
|
// must always name the owner it targets.
|
|
@@ -102,6 +119,21 @@ export class Router {
|
|
|
102
119
|
}
|
|
103
120
|
}
|
|
104
121
|
|
|
122
|
+
/** GHCR verbs: config.ghcr checks the owner and image itself; no GitHub PAT is resolved. */
|
|
123
|
+
private async callGhcr(tool: string, args: Record<string, unknown>, rule: ToolRule): Promise<CallToolResult> {
|
|
124
|
+
const owner = extractOwner(args) ?? "";
|
|
125
|
+
const repo = typeof args.image === "string" ? `${owner}/${args.image}` : owner;
|
|
126
|
+
try {
|
|
127
|
+
this.ghcr ??= new GhcrService(this.config.ghcr, this.secrets);
|
|
128
|
+
const out = tool === "ghcr_push" ? await this.ghcr.push(args) : tool === "ghcr_tags" ? await this.ghcr.tags(args) : this.ghcr.status(args);
|
|
129
|
+
audit({ tool, owner, mode: rule.mode, repo, decision: "custom", ok: true });
|
|
130
|
+
return { content: [{ type: "text", text: JSON.stringify(out) }] };
|
|
131
|
+
} catch (err) {
|
|
132
|
+
audit({ tool, owner, mode: rule.mode, repo, decision: "custom", ok: false, detail: (err as Error).message });
|
|
133
|
+
throw err;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
105
137
|
private async dispatchCustom(
|
|
106
138
|
tool: string,
|
|
107
139
|
args: Record<string, unknown>,
|
|
@@ -158,6 +190,15 @@ export class Router {
|
|
|
158
190
|
const out = await reportCiStatus(octokit, reportArgs, this.makeJobLogsFetcher(owner, reportArgs.repo, token));
|
|
159
191
|
return { content: [{ type: "text", text: JSON.stringify(out) }] };
|
|
160
192
|
}
|
|
193
|
+
case "npm_scaffold": {
|
|
194
|
+
const npm = this.config.npm!; // availability already required it
|
|
195
|
+
// The GitHub side only reads (repo metadata, package.json, workflow file), so it
|
|
196
|
+
// runs on the read PAT even though the verb is a write.
|
|
197
|
+
const octokit = new Octokit({ auth: this.secrets.resolve(this.config.owners[owner]!.read) });
|
|
198
|
+
const registry = new NpmRegistry({ token: this.secrets.resolve(npm.token), ...(npm.registry ? { registry: npm.registry } : {}) });
|
|
199
|
+
const out = await npmScaffold(args, { octokit, registry, scopes: npm.scopes });
|
|
200
|
+
return { content: [{ type: "text", text: JSON.stringify(out) }] };
|
|
201
|
+
}
|
|
161
202
|
default:
|
|
162
203
|
throw new Error(`custom verb "${tool}" not implemented`);
|
|
163
204
|
}
|
package/src/secrets.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* @Email: <Shinrai@users.noreply.github.com>
|
|
8
8
|
* -----
|
|
9
9
|
* @Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
|
|
10
|
-
* @Last modified time: 2026-10-
|
|
10
|
+
* @Last modified time: 2026-10-31T12:12:11-07:00 (1793473931)
|
|
11
11
|
* -----
|
|
12
12
|
* @Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
|
|
13
13
|
*
|
|
@@ -17,6 +17,7 @@ import { readFileSync } from "node:fs";
|
|
|
17
17
|
import { join } from "node:path";
|
|
18
18
|
import type { Config, SecretStore } from "./types.ts";
|
|
19
19
|
import { resolveSecretsPath } from "./paths.ts";
|
|
20
|
+
import { ghcrCredRefs } from "./verbs/ghcr/config.ts";
|
|
20
21
|
|
|
21
22
|
const CRED_PREFIX = "cred:";
|
|
22
23
|
|
|
@@ -48,6 +49,22 @@ export function loadSecretStore(config: Config): SecretStore {
|
|
|
48
49
|
|
|
49
50
|
const SAFE_CRED_NAME = /^[A-Za-z0-9._-]+$/;
|
|
50
51
|
|
|
52
|
+
/** A store over a reader that throws for a missing/empty credential. */
|
|
53
|
+
function storeFrom(read: (name: string) => string): SecretStore {
|
|
54
|
+
return {
|
|
55
|
+
resolve: (ref) => read(credName(ref)),
|
|
56
|
+
has: (ref) => {
|
|
57
|
+
const name = credName(ref);
|
|
58
|
+
try {
|
|
59
|
+
read(name);
|
|
60
|
+
return true;
|
|
61
|
+
} catch {
|
|
62
|
+
return false;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
51
68
|
/**
|
|
52
69
|
* Credential names become filesystem path components (`join(dir, name)`) and are
|
|
53
70
|
* interpolated into systemd drop-in lines, so constrain them to a safe charset —
|
|
@@ -66,7 +83,13 @@ function credName(ref: string): string {
|
|
|
66
83
|
return assertSafeName(ref.slice(CRED_PREFIX.length));
|
|
67
84
|
}
|
|
68
85
|
|
|
69
|
-
/**
|
|
86
|
+
/**
|
|
87
|
+
* Eagerly read every REQUIRED credential (the owners' PATs, the App key) so a missing token
|
|
88
|
+
* fails at startup, not on first request. Optional credentials (the npm token, the GHCR
|
|
89
|
+
* credentials) are only
|
|
90
|
+
* format-checked here: when one isn't stored, the tools that need it are hidden instead
|
|
91
|
+
* (see availability.ts), and the rest of the broker runs normally.
|
|
92
|
+
*/
|
|
70
93
|
function validateRefs(config: Config, read: (name: string) => string): void {
|
|
71
94
|
for (const [owner, creds] of Object.entries(config.owners)) {
|
|
72
95
|
for (const ref of [creds.read, creds.write]) {
|
|
@@ -82,6 +105,13 @@ function validateRefs(config: Config, read: (name: string) => string): void {
|
|
|
82
105
|
}
|
|
83
106
|
read(assertSafeName(config.app.privateKey.slice(CRED_PREFIX.length)));
|
|
84
107
|
}
|
|
108
|
+
if (config.npm) {
|
|
109
|
+
if (!config.npm.token.startsWith(CRED_PREFIX)) throw new Error(`secrets: config.npm.token must be a "cred:<name>" ref, got "${config.npm.token}"`);
|
|
110
|
+
assertSafeName(config.npm.token.slice(CRED_PREFIX.length));
|
|
111
|
+
}
|
|
112
|
+
// GHCR credentials are optional too: format-checked here, and the ghcr_* tools are hidden
|
|
113
|
+
// while none is stored (availability.ts).
|
|
114
|
+
for (const ref of ghcrCredRefs(config.ghcr)) credName(ref);
|
|
85
115
|
}
|
|
86
116
|
|
|
87
117
|
/** systemd-creds provider: read decrypted creds from $CREDENTIALS_DIRECTORY. */
|
|
@@ -105,7 +135,7 @@ function systemdCredsStore(config: Config): SecretStore {
|
|
|
105
135
|
return val;
|
|
106
136
|
};
|
|
107
137
|
validateRefs(config, read);
|
|
108
|
-
return
|
|
138
|
+
return storeFrom(read);
|
|
109
139
|
}
|
|
110
140
|
|
|
111
141
|
/** file provider: read tokens from a JSON `{ "<name>": "<token>" }` map. */
|
|
@@ -133,5 +163,5 @@ function fileStore(config: Config): SecretStore {
|
|
|
133
163
|
return token;
|
|
134
164
|
};
|
|
135
165
|
validateRefs(config, read);
|
|
136
|
-
return
|
|
166
|
+
return storeFrom(read);
|
|
137
167
|
}
|
package/src/types.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* @Email: <Shinrai@users.noreply.github.com>
|
|
8
8
|
* -----
|
|
9
9
|
* @Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
|
|
10
|
-
* @Last modified time: 2026-10-
|
|
10
|
+
* @Last modified time: 2026-10-31T12:12:12-07:00 (1793473932)
|
|
11
11
|
* -----
|
|
12
12
|
* @Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
|
|
13
13
|
*
|
|
@@ -23,6 +23,8 @@
|
|
|
23
23
|
* safe to copy between machines.
|
|
24
24
|
*/
|
|
25
25
|
|
|
26
|
+
import type { GhcrConfig } from "./verbs/ghcr/config.ts";
|
|
27
|
+
|
|
26
28
|
export type Mode = "read" | "write";
|
|
27
29
|
|
|
28
30
|
/**
|
|
@@ -73,6 +75,22 @@ export interface SecretsConfig {
|
|
|
73
75
|
path?: string;
|
|
74
76
|
}
|
|
75
77
|
|
|
78
|
+
/**
|
|
79
|
+
* npm integration for `npm_scaffold`. Optional: without it the verb refuses. The token is a
|
|
80
|
+
* granular access token created by the installer without 2FA bypass, so every write it makes
|
|
81
|
+
* still needs a user-supplied one-time password.
|
|
82
|
+
*/
|
|
83
|
+
export interface NpmConfig {
|
|
84
|
+
/** Credential reference (e.g. "cred:npm-token") for the granular access token. */
|
|
85
|
+
token: string;
|
|
86
|
+
/** Scopes the verb may create packages in, e.g. ["@cldmv"]. */
|
|
87
|
+
scopes: string[];
|
|
88
|
+
/** Registry base URL. Defaults to https://registry.npmjs.org. */
|
|
89
|
+
registry?: string;
|
|
90
|
+
/** Installer only: npm orgs the token is also granted org read-write on when it is created. */
|
|
91
|
+
tokenOrgs?: string[];
|
|
92
|
+
}
|
|
93
|
+
|
|
76
94
|
export interface Config {
|
|
77
95
|
/** e.g. "unix:/run/gh-broker/gh-broker.sock" */
|
|
78
96
|
listen: string;
|
|
@@ -96,9 +114,15 @@ export interface Config {
|
|
|
96
114
|
app?: AppConfig;
|
|
97
115
|
/** Optional secret-store backend selection. Defaults to the systemd-creds provider. */
|
|
98
116
|
secrets?: SecretsConfig;
|
|
117
|
+
/** Optional npm integration (`npm_scaffold`). */
|
|
118
|
+
npm?: NpmConfig;
|
|
119
|
+
/** Optional GHCR push policy + credentials for the ghcr_* verbs (src/verbs/ghcr/config.ts). */
|
|
120
|
+
ghcr?: GhcrConfig;
|
|
99
121
|
}
|
|
100
122
|
|
|
101
123
|
/** Resolves a "cred:<name>" reference to a token, never logging it. */
|
|
102
124
|
export interface SecretStore {
|
|
103
125
|
resolve(ref: string): string;
|
|
126
|
+
/** Whether an optional credential is stored (non-empty). Never throws for a missing one. */
|
|
127
|
+
has(ref: string): boolean;
|
|
104
128
|
}
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
/**
|
|
2
|
+
*
|
|
3
|
+
* @Project: @cldmv/gh-broker
|
|
4
|
+
* @Filename: /src/verbs/ghcr/archive.ts
|
|
5
|
+
* @Date: 2026-10-31T10:59:13-07:00 (1793469553)
|
|
6
|
+
* @Author: Nate Corcoran <CLDMV>
|
|
7
|
+
* @Email: <Shinrai@users.noreply.github.com>
|
|
8
|
+
* -----
|
|
9
|
+
* @Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
|
|
10
|
+
* @Last modified time: 2026-10-31T23:26:33-07:00 (1793514393)
|
|
11
|
+
* -----
|
|
12
|
+
* @Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
|
|
13
|
+
*
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Opens an image archive from the GHCR drop directory and works out what to push.
|
|
18
|
+
*
|
|
19
|
+
* The archive path comes from the agent, and the broker can read files the agent can't
|
|
20
|
+
* (its own credentials among them). A path that resolves outside the drop directory would
|
|
21
|
+
* let the agent have the broker upload a secret into a registry the agent can then pull
|
|
22
|
+
* from. So the file is opened exactly once, by descriptor, and every check runs against
|
|
23
|
+
* that descriptor rather than the path:
|
|
24
|
+
*
|
|
25
|
+
* 1. lexical: the normalized path must sit under `dropDir`;
|
|
26
|
+
* 2. the real (symlink-resolved) parent directory must be the real `dropDir` or under it,
|
|
27
|
+
* so a symlinked subdirectory pointing elsewhere is refused before anything is opened;
|
|
28
|
+
* steps 1 and 2 give the same answer for every refusal, so a path elsewhere reveals
|
|
29
|
+
* nothing about what exists there;
|
|
30
|
+
* 3. `O_NOFOLLOW | O_NONBLOCK`: the last component can't be a symlink, and a FIFO can't
|
|
31
|
+
* hang the open;
|
|
32
|
+
* 4. `/proc/self/fd/<fd>`: the file actually opened lives under the real `dropDir`. This
|
|
33
|
+
* runs before any check that describes the file, and catches a parent directory swapped
|
|
34
|
+
* between step 2 and the open;
|
|
35
|
+
* 5. `fstat`: a regular file, exactly one link (no hard link to a file elsewhere), not
|
|
36
|
+
* empty, not over `maxArchiveBytes`.
|
|
37
|
+
*
|
|
38
|
+
* All later reads use positional reads on that descriptor; the path is never reopened.
|
|
39
|
+
*
|
|
40
|
+
* Two archive formats are read:
|
|
41
|
+
* - OCI image layout (`index.json` + `blobs/sha256/…`): `docker save` on Docker 25+,
|
|
42
|
+
* `docker buildx build --output type=oci`, `skopeo copy … oci-archive:`.
|
|
43
|
+
* - docker-archive (`manifest.json` only): `docker save` on older Docker.
|
|
44
|
+
*
|
|
45
|
+
* Layers that are already gzip/zstd-compressed are pushed byte-for-byte. Uncompressed
|
|
46
|
+
* layers (what `docker save` writes for the classic image store) are gzipped on the fly at
|
|
47
|
+
* push time, and the manifest is then rebuilt as an OCI image manifest.
|
|
48
|
+
*/
|
|
49
|
+
|
|
50
|
+
import { closeSync, constants, fstatSync, openSync, readlinkSync, realpathSync } from "node:fs";
|
|
51
|
+
import { createHash } from "node:crypto";
|
|
52
|
+
import { dirname, isAbsolute, join, normalize } from "node:path";
|
|
53
|
+
import { indexTar, peekMember, readMember, type TarIndex, type TarMember } from "./tar.ts";
|
|
54
|
+
|
|
55
|
+
export const MEDIA = {
|
|
56
|
+
ociIndex: "application/vnd.oci.image.index.v1+json",
|
|
57
|
+
ociManifest: "application/vnd.oci.image.manifest.v1+json",
|
|
58
|
+
ociConfig: "application/vnd.oci.image.config.v1+json",
|
|
59
|
+
ociLayer: "application/vnd.oci.image.layer.v1.tar",
|
|
60
|
+
ociLayerGzip: "application/vnd.oci.image.layer.v1.tar+gzip",
|
|
61
|
+
ociLayerZstd: "application/vnd.oci.image.layer.v1.tar+zstd",
|
|
62
|
+
dockerList: "application/vnd.docker.distribution.manifest.list.v2+json",
|
|
63
|
+
dockerManifest: "application/vnd.docker.distribution.manifest.v2+json",
|
|
64
|
+
dockerConfig: "application/vnd.docker.container.image.v1+json",
|
|
65
|
+
dockerLayer: "application/vnd.docker.image.rootfs.diff.tar",
|
|
66
|
+
dockerLayerGzip: "application/vnd.docker.image.rootfs.diff.tar.gzip"
|
|
67
|
+
} as const;
|
|
68
|
+
|
|
69
|
+
const MANIFEST_TYPES = new Set<string>([MEDIA.ociManifest, MEDIA.dockerManifest]);
|
|
70
|
+
const INDEX_TYPES = new Set<string>([MEDIA.ociIndex, MEDIA.dockerList]);
|
|
71
|
+
const LAYER_TYPES = new Set<string>([MEDIA.ociLayer, MEDIA.ociLayerGzip, MEDIA.ociLayerZstd, MEDIA.dockerLayer, MEDIA.dockerLayerGzip]);
|
|
72
|
+
const CONFIG_TYPES = new Set<string>([MEDIA.ociConfig, MEDIA.dockerConfig]);
|
|
73
|
+
const DIGEST = /^sha256:[a-f0-9]{64}$/;
|
|
74
|
+
const MAX_JSON = 4 * 1024 * 1024;
|
|
75
|
+
export const MAX_LAYERS = 256;
|
|
76
|
+
|
|
77
|
+
export class ArchiveError extends Error {}
|
|
78
|
+
|
|
79
|
+
const fail = (msg: string): never => {
|
|
80
|
+
throw new ArchiveError(`archive: ${msg}`);
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
export interface OpenedArchive {
|
|
84
|
+
fd: number;
|
|
85
|
+
/** The real path of the opened file (inside dropDir). */
|
|
86
|
+
path: string;
|
|
87
|
+
size: number;
|
|
88
|
+
close(): void;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Securely open `input` (absolute, or relative to dropDir) — see the file comment for the checks. */
|
|
92
|
+
export function openArchive(input: unknown, dropDir: string, maxBytes: number): OpenedArchive {
|
|
93
|
+
if (typeof input !== "string" || !input || input.length > 4096 || input.includes("\0")) fail('"archive" must be a file path inside the drop directory');
|
|
94
|
+
const candidate = normalize(isAbsolute(input as string) ? (input as string) : join(dropDir, input as string));
|
|
95
|
+
// One answer for every path that can't be confirmed to be in the drop dir (outside it,
|
|
96
|
+
// behind a symlinked or missing subdirectory), so none of them reveals anything.
|
|
97
|
+
const outside = `"archive" must be an existing file inside the drop directory ${dropDir}`;
|
|
98
|
+
if (!candidate.startsWith(`${dropDir}/`)) fail(outside);
|
|
99
|
+
|
|
100
|
+
let realDrop: string;
|
|
101
|
+
try {
|
|
102
|
+
realDrop = realpathSync(dropDir);
|
|
103
|
+
} catch {
|
|
104
|
+
return fail(`drop directory ${dropDir} does not exist — the installer creates it`);
|
|
105
|
+
}
|
|
106
|
+
// A symlinked subdirectory would make the open below follow it out of the drop dir.
|
|
107
|
+
let realParent: string;
|
|
108
|
+
try {
|
|
109
|
+
realParent = realpathSync(dirname(candidate));
|
|
110
|
+
} catch {
|
|
111
|
+
return fail(outside);
|
|
112
|
+
}
|
|
113
|
+
if (realParent !== realDrop && !realParent.startsWith(`${realDrop}/`)) fail(outside);
|
|
114
|
+
|
|
115
|
+
let fd: number;
|
|
116
|
+
try {
|
|
117
|
+
fd = openSync(candidate, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
118
|
+
} catch (err) {
|
|
119
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
120
|
+
if (code === "ELOOP") return fail(`${candidate} is a symlink — put the archive file itself in the drop directory`);
|
|
121
|
+
if (code === "ENOENT" || code === "ENOTDIR") return fail(`${candidate} not found`);
|
|
122
|
+
if (code === "EACCES") return fail(`${candidate} is not readable by the broker — make it group-readable (chmod g+r)`);
|
|
123
|
+
// openSync errors always carry an errno code, so the "error" fallback is unreachable; it only satisfies the type.
|
|
124
|
+
/* v8 ignore next */
|
|
125
|
+
return fail(`cannot open ${candidate} (${code ?? "error"})`);
|
|
126
|
+
}
|
|
127
|
+
try {
|
|
128
|
+
// Containment first: nothing about a file outside the drop dir is described.
|
|
129
|
+
const real = readlinkSync(`/proc/self/fd/${fd}`);
|
|
130
|
+
// Reachable when a parent directory is swapped for a symlink between the realpath check
|
|
131
|
+
// above and the open (a race the suite cannot drive deterministically); guarded because
|
|
132
|
+
// that race is exactly how a broker-only file could otherwise be read.
|
|
133
|
+
/* v8 ignore next */
|
|
134
|
+
if (!real.startsWith(`${realDrop}/`)) fail(outside);
|
|
135
|
+
const st = fstatSync(fd);
|
|
136
|
+
if (!st.isFile()) fail(`${candidate} is not a regular file`);
|
|
137
|
+
if (st.nlink !== 1) fail(`${candidate} has ${st.nlink} hard links — copy the archive instead of linking it`);
|
|
138
|
+
if (st.size === 0) fail(`${candidate} is empty`);
|
|
139
|
+
if (st.size > maxBytes) fail(`${candidate} is ${st.size} bytes — over the ${maxBytes}-byte limit (ghcr.maxArchiveBytes)`);
|
|
140
|
+
let closed = false;
|
|
141
|
+
return {
|
|
142
|
+
fd,
|
|
143
|
+
path: real,
|
|
144
|
+
size: st.size,
|
|
145
|
+
close() {
|
|
146
|
+
if (closed) return;
|
|
147
|
+
closed = true;
|
|
148
|
+
closeSync(fd);
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
} catch (err) {
|
|
152
|
+
closeSync(fd);
|
|
153
|
+
throw err;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** One blob to push. `digest`/`size` are what the registry will hold (known up front unless `compress`). */
|
|
158
|
+
export interface BlobPlan {
|
|
159
|
+
member: TarMember;
|
|
160
|
+
mediaType: string;
|
|
161
|
+
/** Digest + size in the archive (for a compressed-on-push layer: of the uncompressed tar). */
|
|
162
|
+
sourceDigest?: string;
|
|
163
|
+
sourceSize: number;
|
|
164
|
+
/** Gzip this layer while pushing it (archive holds it uncompressed). */
|
|
165
|
+
compress: boolean;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export interface ImagePlan {
|
|
169
|
+
format: "oci-layout" | "docker-archive";
|
|
170
|
+
config: BlobPlan & { digest: string };
|
|
171
|
+
layers: BlobPlan[];
|
|
172
|
+
/** Set when the archive's own manifest can be pushed unchanged (nothing recompressed or retyped). */
|
|
173
|
+
verbatimManifest?: { bytes: Buffer; mediaType: string; digest: string };
|
|
174
|
+
annotations?: Record<string, string>;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const sha256 = (b: Buffer): string => `sha256:${createHash("sha256").update(b).digest("hex")}`;
|
|
178
|
+
|
|
179
|
+
function parseJson(buf: Buffer, what: string): unknown {
|
|
180
|
+
try {
|
|
181
|
+
return JSON.parse(buf.toString("utf8"));
|
|
182
|
+
} catch {
|
|
183
|
+
return fail(`${what} is not valid JSON`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
interface Descriptor {
|
|
188
|
+
mediaType?: string;
|
|
189
|
+
digest?: string;
|
|
190
|
+
size?: number;
|
|
191
|
+
annotations?: Record<string, string>;
|
|
192
|
+
platform?: { os?: string; architecture?: string };
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function blobMember(index: TarIndex, digest: unknown, what: string): TarMember {
|
|
196
|
+
if (typeof digest !== "string" || !DIGEST.test(digest)) fail(`${what} has an unsupported digest (only sha256 is accepted)`);
|
|
197
|
+
const m = index.files.get(`blobs/sha256/${(digest as string).slice("sha256:".length)}`);
|
|
198
|
+
if (!m) fail(`${what} ${digest} is missing from the archive`);
|
|
199
|
+
return m as TarMember;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function readBlobJson(fd: number, index: TarIndex, d: Descriptor, what: string): { bytes: Buffer; json: unknown } {
|
|
203
|
+
const m = blobMember(index, d.digest, what);
|
|
204
|
+
const bytes = readMember(fd, m, MAX_JSON, what);
|
|
205
|
+
if (sha256(bytes) !== d.digest) fail(`${what} does not match its digest`);
|
|
206
|
+
return { bytes, json: parseJson(bytes, what) };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* The image entries of an index: buildx attestation manifests and unknown/unknown platform
|
|
211
|
+
* entries are not images. When several remain, keep only those whose manifest blob is in
|
|
212
|
+
* the archive: with the containerd image store, `docker save` writes the image's whole
|
|
213
|
+
* multi-platform index but only the blobs of the platform that was pulled.
|
|
214
|
+
*/
|
|
215
|
+
function images(manifests: unknown, index: TarIndex): Descriptor[] {
|
|
216
|
+
if (!Array.isArray(manifests)) return fail("index has no manifests list");
|
|
217
|
+
const list = (manifests as Descriptor[]).filter(
|
|
218
|
+
(d) => d?.annotations?.["vnd.docker.reference.type"] !== "attestation-manifest" && !(d?.platform?.os === "unknown" && d?.platform?.architecture === "unknown")
|
|
219
|
+
);
|
|
220
|
+
if (list.length <= 1) return list;
|
|
221
|
+
return list.filter((d) => typeof d?.digest === "string" && DIGEST.test(d.digest) && index.files.has(`blobs/sha256/${d.digest.slice("sha256:".length)}`));
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function sniff(fd: number, m: TarMember): "gzip" | "zstd" | "none" {
|
|
225
|
+
const head = peekMember(fd, m, 4);
|
|
226
|
+
if (head.length >= 2 && head[0] === 0x1f && head[1] === 0x8b) return "gzip";
|
|
227
|
+
if (head.length >= 4 && head.readUInt32LE(0) === 0xfd2fb528) return "zstd";
|
|
228
|
+
return "none";
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Decide how a layer is pushed: verbatim when already compressed, gzipped on the fly otherwise. */
|
|
232
|
+
function layerPlan(fd: number, m: TarMember, declared: string | undefined, sourceDigest: string | undefined): BlobPlan {
|
|
233
|
+
if (declared !== undefined && !LAYER_TYPES.has(declared)) fail(`layer media type "${declared}" is not supported (non-distributable and foreign layers are refused)`);
|
|
234
|
+
const kind = sniff(fd, m);
|
|
235
|
+
if (kind === "gzip") {
|
|
236
|
+
const mediaType = declared === MEDIA.ociLayerGzip || declared === MEDIA.dockerLayerGzip ? declared : MEDIA.ociLayerGzip;
|
|
237
|
+
return { member: m, mediaType, sourceDigest, sourceSize: m.size, compress: false };
|
|
238
|
+
}
|
|
239
|
+
if (kind === "zstd") return { member: m, mediaType: MEDIA.ociLayerZstd, sourceDigest, sourceSize: m.size, compress: false };
|
|
240
|
+
return { member: m, mediaType: MEDIA.ociLayerGzip, sourceDigest, sourceSize: m.size, compress: true };
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Read the archive's index/manifest and plan the push. Reads only small JSON members and 4-byte layer heads. */
|
|
244
|
+
export function planImage(fd: number, size: number): ImagePlan {
|
|
245
|
+
const index = indexTar(fd, size);
|
|
246
|
+
const top = index.files.get("index.json");
|
|
247
|
+
if (top) return planOciLayout(fd, index, top);
|
|
248
|
+
const legacy = index.files.get("manifest.json");
|
|
249
|
+
if (legacy) return planDockerArchive(fd, index, legacy);
|
|
250
|
+
return fail("not an image archive (no index.json or manifest.json) — create it with `docker save <image> -o <file>`");
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function planOciLayout(fd: number, index: TarIndex, top: TarMember): ImagePlan {
|
|
254
|
+
const root = parseJson(readMember(fd, top, MAX_JSON, "index.json"), "index.json") as { manifests?: unknown };
|
|
255
|
+
let list = images(root.manifests, index);
|
|
256
|
+
let desc: Descriptor | undefined = list.length === 1 ? list[0] : undefined;
|
|
257
|
+
// docker save / buildx often wrap the image manifest in one more index level.
|
|
258
|
+
if (desc && INDEX_TYPES.has(desc.mediaType ?? "")) {
|
|
259
|
+
const nested = readBlobJson(fd, index, desc, "nested index").json as { manifests?: unknown };
|
|
260
|
+
list = images(nested.manifests, index);
|
|
261
|
+
desc = list.length === 1 ? list[0] : undefined;
|
|
262
|
+
}
|
|
263
|
+
if (!desc) fail(`archive holds ${list.length} images — save exactly one single-platform image per archive`);
|
|
264
|
+
const d = desc as Descriptor;
|
|
265
|
+
if (!MANIFEST_TYPES.has(d.mediaType ?? "")) fail(`manifest media type "${d.mediaType}" is not supported`);
|
|
266
|
+
const { bytes, json } = readBlobJson(fd, index, d, "image manifest");
|
|
267
|
+
const man = json as { config?: Descriptor; layers?: Descriptor[]; annotations?: Record<string, string> };
|
|
268
|
+
if (!man.config || !Array.isArray(man.layers)) fail("image manifest has no config or layers");
|
|
269
|
+
if (man.layers!.length > MAX_LAYERS) fail(`image has ${man.layers!.length} layers — at most ${MAX_LAYERS}`);
|
|
270
|
+
|
|
271
|
+
const cfg = man.config as Descriptor;
|
|
272
|
+
if (!CONFIG_TYPES.has(cfg.mediaType ?? "")) fail(`config media type "${cfg.mediaType}" is not supported`);
|
|
273
|
+
const cfgMember = blobMember(index, cfg.digest, "image config");
|
|
274
|
+
if (cfgMember.size !== cfg.size) fail("image config size does not match its descriptor");
|
|
275
|
+
|
|
276
|
+
let changed = false;
|
|
277
|
+
const layers = man.layers!.map((l, i) => {
|
|
278
|
+
const m = blobMember(index, l.digest, `layer ${i + 1}`);
|
|
279
|
+
if (m.size !== l.size) fail(`layer ${i + 1} size does not match its descriptor`);
|
|
280
|
+
const plan = layerPlan(fd, m, l.mediaType, l.digest);
|
|
281
|
+
if (plan.compress || plan.mediaType !== l.mediaType) changed = true;
|
|
282
|
+
return plan;
|
|
283
|
+
});
|
|
284
|
+
return {
|
|
285
|
+
format: "oci-layout",
|
|
286
|
+
config: { member: cfgMember, mediaType: cfg.mediaType!, digest: cfg.digest!, sourceDigest: cfg.digest, sourceSize: cfgMember.size, compress: false },
|
|
287
|
+
layers,
|
|
288
|
+
...(changed ? {} : { verbatimManifest: { bytes, mediaType: d.mediaType!, digest: d.digest! } }),
|
|
289
|
+
...(man.annotations ? { annotations: man.annotations } : {})
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function planDockerArchive(fd: number, index: TarIndex, legacy: TarMember): ImagePlan {
|
|
294
|
+
const list = parseJson(readMember(fd, legacy, MAX_JSON, "manifest.json"), "manifest.json");
|
|
295
|
+
if (!Array.isArray(list) || list.length !== 1) fail(`archive holds ${Array.isArray(list) ? list.length : 0} images — save exactly one image per archive`);
|
|
296
|
+
const entry = (list as Array<{ Config?: unknown; Layers?: unknown }>)[0]!;
|
|
297
|
+
if (typeof entry.Config !== "string" || !Array.isArray(entry.Layers)) fail("manifest.json entry has no Config or Layers");
|
|
298
|
+
if ((entry.Layers as unknown[]).length > MAX_LAYERS) fail(`image has ${(entry.Layers as unknown[]).length} layers — at most ${MAX_LAYERS}`);
|
|
299
|
+
|
|
300
|
+
const member = (name: unknown, what: string): TarMember => {
|
|
301
|
+
const m = typeof name === "string" ? index.files.get(name.replace(/^\.?\//, "")) : undefined;
|
|
302
|
+
if (!m) fail(`${what} "${String(name)}" is missing from the archive`);
|
|
303
|
+
return m as TarMember;
|
|
304
|
+
};
|
|
305
|
+
const cfgMember = member(entry.Config, "image config");
|
|
306
|
+
const cfgBytes = readMember(fd, cfgMember, MAX_JSON, "image config");
|
|
307
|
+
parseJson(cfgBytes, "image config");
|
|
308
|
+
const layers = (entry.Layers as unknown[]).map((name, i) => layerPlan(fd, member(name, `layer ${i + 1}`), undefined, undefined));
|
|
309
|
+
return {
|
|
310
|
+
format: "docker-archive",
|
|
311
|
+
config: { member: cfgMember, mediaType: MEDIA.ociConfig, digest: sha256(cfgBytes), sourceSize: cfgMember.size, compress: false },
|
|
312
|
+
layers
|
|
313
|
+
};
|
|
314
|
+
}
|