@cldmv/gh-broker 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/README.md +48 -13
  2. package/config.example.json +86 -3
  3. package/config.sample.json +1 -1
  4. package/deploy/README.md +58 -18
  5. package/deploy/cli.mjs +38 -0
  6. package/deploy/gh-broker.service +9 -2
  7. package/deploy/install.mjs +100 -41
  8. package/deploy/lib/agent-cli.mjs +166 -0
  9. package/deploy/lib/agents.mjs +128 -0
  10. package/deploy/lib/cli.mjs +72 -0
  11. package/deploy/lib/config-merge.mjs +29 -8
  12. package/deploy/lib/install-args.mjs +3 -2
  13. package/deploy/lib/mcp-config.mjs +211 -0
  14. package/deploy/lib/otp-cli.mjs +136 -0
  15. package/deploy/lib/otp.mjs +325 -0
  16. package/deploy/lib/secret-store.mjs +1 -0
  17. package/deploy/lib/system.mjs +24 -12
  18. package/deploy/real-deps.mjs +102 -0
  19. package/dist/index.mjs +5171 -1686
  20. package/dist/index.mjs.map +4 -4
  21. package/package.json +14 -3
  22. package/docs/TOOLS.md +0 -538
  23. package/docs/agent-user-model.md +0 -89
  24. package/docs/changelog/v0/v0.2.0.md +0 -26
  25. package/docs/changelog/v0/v0.2.1.md +0 -23
  26. package/docs/changelog/v0/v0.2.2.md +0 -23
  27. package/docs/changelog/v0/v0.2.3.md +0 -33
  28. package/docs/changelog/v0/v0.2.4.md +0 -25
  29. package/docs/changelog/v0/v0.2.5.md +0 -29
  30. package/docs/changelog/v0/v0.3.0.md +0 -84
  31. package/docs/changelog/v0/v0.3.1.md +0 -30
  32. package/docs/changelog/v0/v0.3.2.md +0 -24
  33. package/docs/changelog/v0/v0.3.3.md +0 -28
  34. package/docs/changelog/v0/v0.4.0.md +0 -58
  35. package/docs/changelog/v0/v0.4.1.md +0 -33
  36. package/docs/changelog/v0/v0.4.2.md +0 -45
  37. package/docs/changelog/v0/v0.5.0.md +0 -39
  38. package/docs/changelog/v0/v0.5.1.md +0 -28
  39. package/docs/changelog/v0/v0.6.0.md +0 -52
  40. package/docs/changelog/v0/v0.6.1.md +0 -48
  41. package/docs/changelog/v0/v0.7.0.md +0 -71
  42. package/docs/changelog/v0/v0.8.0.md +0 -115
  43. package/docs/deploy-architecture.md +0 -84
  44. package/src/appauth.ts +0 -56
  45. package/src/audit.ts +0 -34
  46. package/src/availability.ts +0 -42
  47. package/src/catalog.ts +0 -182
  48. package/src/config.ts +0 -157
  49. package/src/index.ts +0 -107
  50. package/src/paths.ts +0 -60
  51. package/src/router.ts +0 -299
  52. package/src/secrets.ts +0 -167
  53. package/src/server.ts +0 -91
  54. package/src/types.ts +0 -128
  55. package/src/upstream.ts +0 -85
  56. package/src/verbs/ghcr/archive.ts +0 -314
  57. package/src/verbs/ghcr/config.ts +0 -142
  58. package/src/verbs/ghcr/jobs.ts +0 -141
  59. package/src/verbs/ghcr/push.ts +0 -192
  60. package/src/verbs/ghcr/registry.ts +0 -275
  61. package/src/verbs/ghcr/tar.ts +0 -203
  62. package/src/verbs/ghcr_push.ts +0 -264
  63. package/src/verbs/issue_labels.ts +0 -329
  64. package/src/verbs/link_issue.ts +0 -84
  65. package/src/verbs/npm-registry.ts +0 -168
  66. package/src/verbs/npm-tarball.ts +0 -76
  67. package/src/verbs/npm_scaffold.ts +0 -325
  68. package/src/verbs/release-analysis.ts +0 -212
  69. package/src/verbs/repo-tier.ts +0 -90
  70. package/src/verbs/report_ci_status.ts +0 -358
  71. package/src/verbs/report_release_prs.ts +0 -238
  72. package/src/verbs/report_security_alerts.ts +0 -442
  73. package/src/verbs/report_unlabeled.ts +0 -194
  74. package/src/verbs/rest-client.ts +0 -181
  75. package/src/verbs/review_thread.ts +0 -138
  76. package/tests/appauth.test.vitest.mts +0 -76
  77. package/tests/audit.test.vitest.mts +0 -48
  78. package/tests/catalog.test.vitest.mts +0 -234
  79. package/tests/config-merge.test.vitest.mts +0 -325
  80. package/tests/config.test.vitest.mts +0 -388
  81. package/tests/credentials.test.vitest.mts +0 -127
  82. package/tests/ghcr-archive.test.vitest.mts +0 -439
  83. package/tests/ghcr-config.test.vitest.mts +0 -96
  84. package/tests/ghcr-push.test.vitest.mts +0 -526
  85. package/tests/ghcr-registry.test.vitest.mts +0 -206
  86. package/tests/helpers/ghcr-fixtures.mts +0 -373
  87. package/tests/install-args.test.vitest.mts +0 -152
  88. package/tests/issue-labels.test.vitest.mts +0 -362
  89. package/tests/npm-scaffold.test.vitest.mts +0 -479
  90. package/tests/npm-setup.test.vitest.mts +0 -196
  91. package/tests/paths.test.vitest.mts +0 -57
  92. package/tests/prompt.test.vitest.mts +0 -128
  93. package/tests/release-prs.test.vitest.mts +0 -247
  94. package/tests/rest-client.test.vitest.mts +0 -115
  95. package/tests/router.test.vitest.mts +0 -505
  96. package/tests/run-vitest.mjs +0 -51
  97. package/tests/secret-store.test.vitest.mts +0 -167
  98. package/tests/secrets.test.vitest.mts +0 -208
  99. package/tests/security-alerts.test.vitest.mts +0 -338
  100. package/tests/server.test.vitest.mts +0 -200
  101. package/tests/smoke.test.vitest.mts +0 -177
  102. package/tests/system.test.vitest.mts +0 -355
  103. package/tests/upstream.test.vitest.mts +0 -91
  104. package/tests/verb-helpers.test.vitest.mts +0 -377
  105. package/tests/verbs.test.vitest.mts +0 -624
  106. package/tsconfig.json +0 -21
@@ -1,192 +0,0 @@
1
- /**
2
- *
3
- * @Project: @cldmv/gh-broker
4
- * @Filename: /src/verbs/ghcr/push.ts
5
- * @Date: 2026-10-31T11:00:50-07:00 (1793469650)
6
- * @Author: Nate Corcoran <CLDMV>
7
- * @Email: <Shinrai@users.noreply.github.com>
8
- * -----
9
- * @Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
10
- * @Last modified time: 2026-10-31T11:13:29-07:00 (1793470409)
11
- * -----
12
- * @Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
13
- *
14
- */
15
-
16
- /**
17
- * Pushes one planned image (see archive.ts) to a registry repository.
18
- *
19
- * Phases:
20
- * 1. preparing — for every layer the archive holds uncompressed, gzip it once just to
21
- * learn the compressed digest and size. This costs a second pass of CPU, but it means
22
- * the blob `HEAD` check works for those layers too (a re-push of an unchanged image
23
- * uploads nothing), and nothing has to be buffered or written to disk: the broker's
24
- * unit has no writable scratch space.
25
- * 2. uploading — config then layers: `HEAD` each blob and skip it when present,
26
- * otherwise stream it. Compressed-on-push layers are gzipped again with the same
27
- * settings; zlib is deterministic, and the registry rejects the upload if the bytes
28
- * don't match the digest computed in phase 1.
29
- * 3. manifest — re-check a protected release tag (another push may have landed it while
30
- * this one waited in the queue), then `PUT` the manifest under the tag.
31
- */
32
-
33
- import { createReadStream } from "node:fs";
34
- import { createHash } from "node:crypto";
35
- import { createGzip } from "node:zlib";
36
- import { PassThrough, Writable, type Readable } from "node:stream";
37
- import { pipeline } from "node:stream/promises";
38
- import { MEDIA, type BlobPlan, type ImagePlan } from "./archive.ts";
39
- import type { Registry } from "./registry.ts";
40
-
41
- export const RELEASE_TAG = /^v?\d+\.\d+\.\d+$/;
42
- const GZIP_LEVEL = 6;
43
-
44
- export type Phase = "queued" | "preparing" | "uploading" | "manifest" | "done";
45
-
46
- export interface PushProgress {
47
- phase: Phase;
48
- layersTotal: number;
49
- blobsUploaded: number;
50
- blobsSkipped: number;
51
- /** Bytes of every blob the image references, as stored in the registry. Known once preparing ends. */
52
- bytesTotal: number;
53
- bytesSent: number;
54
- }
55
-
56
- export interface PushResult {
57
- digest: string;
58
- mediaType: string;
59
- /** Total size of config + layers in the registry. */
60
- size: number;
61
- blobsUploaded: number;
62
- blobsSkipped: number;
63
- }
64
-
65
- interface Descriptor {
66
- mediaType: string;
67
- digest: string;
68
- size: number;
69
- }
70
-
71
- export class PushRefused extends Error {}
72
-
73
- function memberStream(fd: number, plan: BlobPlan): Readable {
74
- if (plan.member.size === 0) {
75
- const empty = new PassThrough();
76
- empty.end();
77
- return empty;
78
- }
79
- return createReadStream("", { fd, start: plan.member.offset, end: plan.member.offset + plan.member.size - 1, autoClose: false });
80
- }
81
-
82
- function gzipStream(fd: number, plan: BlobPlan): Readable {
83
- const src = memberStream(fd, plan);
84
- const gz = createGzip({ level: GZIP_LEVEL });
85
- src.on("error", (e) => gz.destroy(e));
86
- return src.pipe(gz);
87
- }
88
-
89
- async function gzipDigest(fd: number, plan: BlobPlan): Promise<{ digest: string; size: number }> {
90
- const hash = createHash("sha256");
91
- let size = 0;
92
- await pipeline(
93
- gzipStream(fd, plan),
94
- new Writable({
95
- write(chunk: Buffer, _enc, cb) {
96
- hash.update(chunk);
97
- size += chunk.length;
98
- cb();
99
- }
100
- })
101
- );
102
- return { digest: `sha256:${hash.digest("hex")}`, size };
103
- }
104
-
105
- const sha256 = (b: Buffer): string => `sha256:${createHash("sha256").update(b).digest("hex")}`;
106
-
107
- export async function pushImage(
108
- registry: Registry,
109
- fd: number,
110
- plan: ImagePlan,
111
- tag: string,
112
- opts: { protectReleaseTags: boolean; onProgress?: (p: PushProgress) => void }
113
- ): Promise<PushResult> {
114
- const progress: PushProgress = { phase: "preparing", layersTotal: plan.layers.length, blobsUploaded: 0, blobsSkipped: 0, bytesTotal: 0, bytesSent: 0 };
115
- const report = () => opts.onProgress?.({ ...progress });
116
- report();
117
-
118
- const layers: Descriptor[] = [];
119
- for (const l of plan.layers) {
120
- if (l.compress) {
121
- const { digest, size } = await gzipDigest(fd, l);
122
- layers.push({ mediaType: l.mediaType, digest, size });
123
- } else {
124
- layers.push({ mediaType: l.mediaType, digest: l.sourceDigest ?? (await rawDigest(fd, l)), size: l.sourceSize });
125
- }
126
- }
127
- const config: Descriptor = { mediaType: plan.config.mediaType, digest: plan.config.digest, size: plan.config.sourceSize };
128
- progress.bytesTotal = config.size + layers.reduce((n, d) => n + d.size, 0);
129
- progress.phase = "uploading";
130
- report();
131
-
132
- const blobs: Array<{ desc: Descriptor; plan: BlobPlan }> = [{ desc: config, plan: plan.config }, ...layers.map((desc, i) => ({ desc, plan: plan.layers[i]! }))];
133
- const seen = new Set<string>();
134
- for (const { desc, plan: p } of blobs) {
135
- if (seen.has(desc.digest) || (await registry.blobExists(desc.digest))) {
136
- seen.add(desc.digest);
137
- progress.blobsSkipped++;
138
- report();
139
- continue;
140
- }
141
- seen.add(desc.digest);
142
- await registry.uploadBlob(desc.digest, desc.size, () => (p.compress ? gzipStream(fd, p) : memberStream(fd, p)), (n) => {
143
- progress.bytesSent += n;
144
- report();
145
- });
146
- progress.blobsUploaded++;
147
- report();
148
- }
149
-
150
- progress.phase = "manifest";
151
- report();
152
- let bytes: Buffer;
153
- let mediaType: string;
154
- if (plan.verbatimManifest) {
155
- bytes = plan.verbatimManifest.bytes;
156
- mediaType = plan.verbatimManifest.mediaType;
157
- } else {
158
- mediaType = MEDIA.ociManifest;
159
- const cfg = { ...config, mediaType: config.mediaType === MEDIA.dockerConfig ? MEDIA.ociConfig : config.mediaType };
160
- bytes = Buffer.from(
161
- JSON.stringify({
162
- schemaVersion: 2,
163
- mediaType,
164
- config: cfg,
165
- layers: layers.map((l) => ({ ...l, mediaType: l.mediaType === MEDIA.dockerLayerGzip ? MEDIA.ociLayerGzip : l.mediaType })),
166
- ...(plan.annotations ? { annotations: plan.annotations } : {})
167
- })
168
- );
169
- }
170
- if (opts.protectReleaseTags && RELEASE_TAG.test(tag) && (await registry.manifestExists(tag))) {
171
- throw new PushRefused(`ghcr_push: tag ${tag} already exists on ${registry.repository} and release tags are protected — push a new version instead`);
172
- }
173
- await registry.putManifest(tag, mediaType, bytes);
174
- progress.phase = "done";
175
- report();
176
- return { digest: sha256(bytes), mediaType, size: progress.bytesTotal, blobsUploaded: progress.blobsUploaded, blobsSkipped: progress.blobsSkipped };
177
- }
178
-
179
- /** sha256 of a member streamed from the archive (docker-archive layers that are already compressed). */
180
- async function rawDigest(fd: number, plan: BlobPlan): Promise<string> {
181
- const hash = createHash("sha256");
182
- await pipeline(
183
- memberStream(fd, plan),
184
- new Writable({
185
- write(chunk: Buffer, _enc, cb) {
186
- hash.update(chunk);
187
- cb();
188
- }
189
- })
190
- );
191
- return `sha256:${hash.digest("hex")}`;
192
- }
@@ -1,275 +0,0 @@
1
- /**
2
- *
3
- * @Project: @cldmv/gh-broker
4
- * @Filename: /src/verbs/ghcr/registry.ts
5
- * @Date: 2026-10-31T11:00:07-07:00 (1793469607)
6
- * @Author: Nate Corcoran <CLDMV>
7
- * @Email: <Shinrai@users.noreply.github.com>
8
- * -----
9
- * @Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
10
- * @Last modified time: 2026-10-31T23:26:34-07:00 (1793514394)
11
- * -----
12
- * @Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
13
- *
14
- */
15
-
16
- /**
17
- * OCI distribution API client for ghcr.io, driven with node:https (no new dependency, no
18
- * docker socket, no skopeo).
19
- *
20
- * Credential handling:
21
- * - The classic PAT is used for exactly one request: the Basic-auth exchange at
22
- * `<base>/token` for a short-lived registry token scoped to one repository.
23
- * - The registry token goes only to `<base>` itself. Every URL the registry hands back
24
- * (upload `Location`, pagination `Link`) is resolved and refused unless its origin is
25
- * `<base>`'s, and redirects are never followed, so neither secret can be carried to
26
- * another host.
27
- * - Every error message leaving this module goes through `clean()`, which replaces both
28
- * secret values and anything shaped like an auth header or a GitHub token. Registry
29
- * error bodies are reduced to their `code: message` pairs; response headers are never
30
- * included.
31
- *
32
- * Upload flow per blob (the one go-containerregistry uses against GHCR):
33
- * `POST /v2/<name>/blobs/uploads/` → `PATCH <location>` streaming the bytes with a known
34
- * Content-Length → `PUT <location>?digest=…`. A 401 mid-flow (expired registry token)
35
- * re-runs the token exchange once and restarts that blob.
36
- */
37
-
38
- import { request as httpRequest, type IncomingHttpHeaders } from "node:http";
39
- import { request as httpsRequest } from "node:https";
40
- import type { Readable } from "node:stream";
41
-
42
- export const GHCR_BASE = "https://ghcr.io";
43
- const MAX_BODY = 1024 * 1024;
44
- const IDLE_TIMEOUT_MS = 10 * 60 * 1000;
45
- const MAX_TAG_PAGES = 20;
46
- export const MANIFEST_ACCEPT = [
47
- "application/vnd.oci.image.manifest.v1+json",
48
- "application/vnd.oci.image.index.v1+json",
49
- "application/vnd.docker.distribution.manifest.v2+json",
50
- "application/vnd.docker.distribution.manifest.list.v2+json"
51
- ].join(", ");
52
-
53
- export class RegistryError extends Error {
54
- readonly status?: number;
55
- constructor(message: string, status?: number) {
56
- super(message);
57
- this.status = status;
58
- }
59
- }
60
-
61
- interface Res {
62
- status: number;
63
- headers: IncomingHttpHeaders;
64
- body: Buffer;
65
- }
66
-
67
- type Body = Buffer | (() => Readable) | undefined;
68
-
69
- export interface RegistryOptions {
70
- /** Registry origin. Always GHCR_BASE in production; tests point it at a local fake. */
71
- base?: string;
72
- username: string;
73
- password: string;
74
- /** "<owner>/<image>" */
75
- repository: string;
76
- }
77
-
78
- export class Registry {
79
- private readonly base: URL;
80
- private readonly username: string;
81
- private readonly password: string;
82
- readonly repository: string;
83
- private token?: string;
84
- private scope?: string;
85
-
86
- constructor(opts: RegistryOptions) {
87
- this.base = new URL(opts.base ?? GHCR_BASE);
88
- this.username = opts.username;
89
- this.password = opts.password;
90
- this.repository = opts.repository;
91
- }
92
-
93
- /** The registry host name as used in image references (e.g. "ghcr.io"). */
94
- get host(): string {
95
- return this.base.host;
96
- }
97
-
98
- /** Replace every secret (and anything that looks like one) in a message. */
99
- clean(msg: string): string {
100
- let out = msg;
101
- for (const s of [this.password, this.token]) if (s && s.length >= 4) out = out.split(s).join("[redacted]");
102
- return out
103
- .replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/gi, "$1 [redacted]")
104
- .replace(/\bgh[pousr]_[A-Za-z0-9]{20,}\b/g, "[redacted]")
105
- .replace(/\bgithub_pat_[A-Za-z0-9_]{20,}\b/g, "[redacted]");
106
- }
107
-
108
- private err(step: string, res: Res): RegistryError {
109
- let detail = "";
110
- try {
111
- const j = JSON.parse(res.body.toString("utf8")) as { errors?: Array<{ code?: unknown; message?: unknown }> };
112
- if (Array.isArray(j.errors)) detail = j.errors.map((e) => `${String(e.code ?? "")}: ${String(e.message ?? "")}`).join("; ");
113
- } catch {
114
- detail = res.body.toString("utf8").replace(/\s+/g, " ").trim();
115
- }
116
- const hint = res.status === 401 || res.status === 403 ? " — check the ghcr credential is a classic PAT with write:packages, not expired, and allowed for this package" : "";
117
- // Scrub before truncating: a secret cut at the 300-character boundary would no longer
118
- // match the exact-value replacement.
119
- const shown = this.clean(detail).slice(0, 300);
120
- return new RegistryError(this.clean(`ghcr: ${step} failed: HTTP ${res.status}${shown ? ` (${shown})` : ""}${hint}`), res.status);
121
- }
122
-
123
- /** Resolve a registry-provided URL and refuse anything that would leave the registry's origin. */
124
- private sameOrigin(raw: string, step: string): URL {
125
- let u: URL;
126
- try {
127
- u = new URL(raw, this.base);
128
- } catch {
129
- throw new RegistryError(`ghcr: ${step}: registry returned an unparseable URL`);
130
- }
131
- if (u.origin !== this.base.origin) throw new RegistryError(`ghcr: ${step}: registry pointed at another host (${u.host}) — refused`);
132
- return u;
133
- }
134
-
135
- private url(path: string): URL {
136
- return new URL(path, this.base);
137
- }
138
-
139
- private send(method: string, url: URL, headers: Record<string, string>, body: Body, onBytes?: (n: number) => void): Promise<Res> {
140
- const fn = url.protocol === "https:" ? httpsRequest : httpRequest;
141
- return new Promise<Res>((resolve, reject) => {
142
- const req = fn(url, { method, headers }, (res) => {
143
- const chunks: Buffer[] = [];
144
- let len = 0;
145
- res.on("data", (c: Buffer) => {
146
- if (len < MAX_BODY) chunks.push(c);
147
- len += c.length;
148
- });
149
- res.on("end", () => resolve({ status: res.statusCode ?? 0, headers: res.headers, body: Buffer.concat(chunks).subarray(0, MAX_BODY) }));
150
- res.on("error", reject);
151
- });
152
- req.setTimeout(IDLE_TIMEOUT_MS, () => req.destroy(new Error(`${method} timed out`)));
153
- req.on("error", (e) => reject(new RegistryError(this.clean(`ghcr: ${method} ${url.pathname} failed: ${e.message}`))));
154
- if (body === undefined) req.end();
155
- else if (Buffer.isBuffer(body)) req.end(body);
156
- else {
157
- const src = body();
158
- src.on("error", (e) => req.destroy(e));
159
- if (onBytes) src.on("data", (c: Buffer) => onBytes(c.length));
160
- src.pipe(req);
161
- }
162
- });
163
- }
164
-
165
- /** Exchange the PAT for a registry token scoped to this repository. */
166
- async login(actions: "pull" | "push,pull"): Promise<void> {
167
- const u = this.url("/token");
168
- u.searchParams.set("service", this.base.host);
169
- u.searchParams.set("scope", `repository:${this.repository}:${actions}`);
170
- const basic = Buffer.from(`${this.username}:${this.password}`).toString("base64");
171
- const res = await this.send("GET", u, { Authorization: `Basic ${basic}`, Accept: "application/json" }, undefined);
172
- if (res.status !== 200) throw this.err("token exchange", res);
173
- let token: unknown;
174
- try {
175
- const j = JSON.parse(res.body.toString("utf8")) as { token?: unknown; access_token?: unknown };
176
- token = j.token ?? j.access_token;
177
- } catch {
178
- /* handled below */
179
- }
180
- if (typeof token !== "string" || !token) throw new RegistryError("ghcr: token exchange returned no token");
181
- this.token = token;
182
- this.scope = actions;
183
- }
184
-
185
- /** One request with the registry token; on 401 re-login once and retry. */
186
- private async authed(method: string, url: URL, headers: Record<string, string>, body: Body, onBytes?: (n: number) => void): Promise<Res> {
187
- if (!this.token) await this.login("pull");
188
- const go = () => this.send(method, url, { ...headers, Authorization: `Bearer ${this.token}` }, body, onBytes);
189
- const res = await go();
190
- if (res.status !== 401) return res;
191
- // login() sets scope together with token and a token is always present here, so the "pull" fallback is unreachable; it only satisfies the type.
192
- /* v8 ignore next */
193
- await this.login((this.scope as "pull" | "push,pull") ?? "pull");
194
- return go();
195
- }
196
-
197
- async blobExists(digest: string): Promise<boolean> {
198
- const res = await this.authed("HEAD", this.url(`/v2/${this.repository}/blobs/${digest}`), {}, undefined);
199
- if (res.status >= 200 && res.status < 400) return true; // a redirect to storage still means "present"
200
- if (res.status === 404) return false;
201
- throw this.err(`checking blob ${digest}`, res);
202
- }
203
-
204
- /**
205
- * Upload one blob whose digest and size are known. `body` must produce a fresh stream
206
- * each call (a retry after a 401 restarts the upload). `onBytes` reports progress.
207
- */
208
- async uploadBlob(digest: string, size: number, body: () => Readable, onBytes?: (n: number) => void): Promise<void> {
209
- for (let attempt = 0; ; attempt++) {
210
- let sent = 0;
211
- const count = (n: number) => {
212
- sent += n;
213
- onBytes?.(n);
214
- };
215
- const res = await this.tryUpload(digest, size, body, count);
216
- if (res === "ok") return;
217
- if (attempt >= 1) throw this.err(`uploading blob ${digest}`, res);
218
- onBytes?.(-sent); // roll the progress counter back before the retry
219
- await this.login("push,pull");
220
- }
221
- }
222
-
223
- private async tryUpload(digest: string, size: number, body: () => Readable, onBytes: (n: number) => void): Promise<"ok" | Res> {
224
- const start = await this.authed("POST", this.url(`/v2/${this.repository}/blobs/uploads/`), { "Content-Length": "0" }, Buffer.alloc(0));
225
- if (start.status === 401) return start;
226
- if (start.status !== 202 || typeof start.headers.location !== "string") throw this.err("starting a blob upload", start);
227
- const patchUrl = this.sameOrigin(start.headers.location, "blob upload");
228
-
229
- const auth = { Authorization: `Bearer ${this.token}` };
230
- const patch = await this.send("PATCH", patchUrl, { ...auth, "Content-Type": "application/octet-stream", "Content-Length": String(size) }, body, onBytes);
231
- if (patch.status === 401) return patch;
232
- if (patch.status !== 202 && patch.status !== 204) throw this.err(`uploading blob ${digest}`, patch);
233
- const putUrl = this.sameOrigin(typeof patch.headers.location === "string" ? patch.headers.location : patchUrl.href, "blob upload");
234
- putUrl.searchParams.set("digest", digest);
235
-
236
- const done = await this.send("PUT", putUrl, { ...auth, "Content-Length": "0" }, Buffer.alloc(0));
237
- if (done.status === 401) return done;
238
- if (done.status !== 201) throw this.err(`finishing blob ${digest}`, done);
239
- return "ok";
240
- }
241
-
242
- /** True when `reference` (tag or digest) already names a manifest in this repository. */
243
- async manifestExists(reference: string): Promise<boolean> {
244
- const res = await this.authed("HEAD", this.url(`/v2/${this.repository}/manifests/${reference}`), { Accept: MANIFEST_ACCEPT }, undefined);
245
- if (res.status === 200) return true;
246
- if (res.status === 404) return false;
247
- throw this.err(`checking tag ${reference}`, res);
248
- }
249
-
250
- async putManifest(tag: string, mediaType: string, bytes: Buffer): Promise<void> {
251
- const res = await this.authed("PUT", this.url(`/v2/${this.repository}/manifests/${tag}`), { "Content-Type": mediaType, "Content-Length": String(bytes.length) }, bytes);
252
- if (res.status !== 201 && res.status !== 200) throw this.err(`pushing manifest for tag ${tag}`, res);
253
- }
254
-
255
- /** Every tag in the repository; `undefined` when the repository doesn't exist yet. */
256
- async listTags(): Promise<string[] | undefined> {
257
- const tags: string[] = [];
258
- let url: URL | undefined = this.url(`/v2/${this.repository}/tags/list?n=1000`);
259
- for (let page = 0; url && page < MAX_TAG_PAGES; page++) {
260
- const res = await this.authed("GET", url, { Accept: "application/json" }, undefined);
261
- if (res.status === 404) return undefined;
262
- if (res.status !== 200) throw this.err("listing tags", res);
263
- let j: { tags?: unknown };
264
- try {
265
- j = JSON.parse(res.body.toString("utf8")) as { tags?: unknown };
266
- } catch {
267
- throw new RegistryError("ghcr: listing tags returned invalid JSON");
268
- }
269
- if (Array.isArray(j.tags)) for (const t of j.tags) if (typeof t === "string") tags.push(t);
270
- const link = typeof res.headers.link === "string" ? /<([^>]+)>;\s*rel="next"/.exec(res.headers.link) : null;
271
- url = link ? this.sameOrigin(link[1]!, "listing tags") : undefined;
272
- }
273
- return tags;
274
- }
275
- }
@@ -1,203 +0,0 @@
1
- /**
2
- *
3
- * @Project: @cldmv/gh-broker
4
- * @Filename: /src/verbs/ghcr/tar.ts
5
- * @Date: 2026-10-31T10:58:00-07:00 (1793469480)
6
- * @Author: Nate Corcoran <CLDMV>
7
- * @Email: <Shinrai@users.noreply.github.com>
8
- * -----
9
- * @Last modified by: Nate Corcoran <CLDMV> (Shinrai@users.noreply.github.com)
10
- * @Last modified time: 2026-10-31T11:13:30-07:00 (1793470410)
11
- * -----
12
- * @Copyright: Copyright (c) 2013-2026 Catalyzed Motivation Inc. All rights reserved.
13
- *
14
- */
15
-
16
- /**
17
- * Minimal read-only tar index for `docker save` / OCI-layout archives.
18
- *
19
- * Only headers are read: the index records where each member's data lives in the file
20
- * (offset + size) so blobs can be streamed straight from the already-open descriptor
21
- * with positional reads. Nothing is extracted to disk, and links inside the archive are
22
- * resolved against the archive's own member names, never the filesystem.
23
- *
24
- * Supported: ustar/POSIX headers, PAX extended headers (`path`, `linkpath`, `size`), GNU
25
- * long names (`L`/`K`), base-256 sizes, regular files, hard links and symlinks. Global PAX
26
- * headers, directories and every other type are skipped.
27
- */
28
-
29
- import { readSync } from "node:fs";
30
- import { posix } from "node:path";
31
-
32
- export interface TarMember {
33
- /** Byte offset of the member's data in the archive. */
34
- offset: number;
35
- size: number;
36
- }
37
-
38
- export interface TarIndex {
39
- /** Regular members by normalized name (links already resolved). */
40
- files: Map<string, TarMember>;
41
- }
42
-
43
- const BLOCK = 512;
44
- /** Bounds the header scan so a hostile archive can't make the index unbounded. */
45
- export const MAX_TAR_MEMBERS = 100_000;
46
- const MAX_EXT_HEADER = 1024 * 1024;
47
- const MAX_LINK_DEPTH = 16;
48
-
49
- export class TarError extends Error {}
50
-
51
- const fail = (msg: string): never => {
52
- throw new TarError(`archive: ${msg}`);
53
- };
54
-
55
- function readAt(fd: number, position: number, length: number): Buffer {
56
- const buf = Buffer.alloc(length);
57
- let got = 0;
58
- while (got < length) {
59
- const n = readSync(fd, buf, got, length - got, position + got);
60
- if (n === 0) break;
61
- got += n;
62
- }
63
- if (got !== length) fail(`truncated at byte ${position + got}`);
64
- return buf;
65
- }
66
-
67
- function cstr(buf: Buffer, start: number, len: number): string {
68
- const slice = buf.subarray(start, start + len);
69
- const nul = slice.indexOf(0);
70
- return slice.subarray(0, nul === -1 ? len : nul).toString("utf8");
71
- }
72
-
73
- function parseSize(buf: Buffer): number {
74
- const field = buf.subarray(124, 136);
75
- if (field[0]! & 0x80) {
76
- // base-256 (GNU) for members over 8 GiB
77
- let n = 0;
78
- for (let i = 1; i < field.length; i++) n = n * 256 + field[i]!;
79
- if (!Number.isSafeInteger(n)) fail("member size out of range");
80
- return n;
81
- }
82
- const s = cstr(field, 0, 12).trim();
83
- if (!/^[0-7]*$/.test(s)) fail("bad size field in tar header");
84
- return s ? parseInt(s, 8) : 0;
85
- }
86
-
87
- function checksumOk(buf: Buffer): boolean {
88
- const stored = cstr(buf, 148, 8).trim();
89
- if (!/^[0-7]+$/.test(stored)) return false;
90
- let sum = 0;
91
- for (let i = 0; i < BLOCK; i++) sum += i >= 148 && i < 156 ? 0x20 : buf[i]!;
92
- return sum === parseInt(stored, 8);
93
- }
94
-
95
- function parsePax(data: Buffer): Record<string, string> {
96
- const out: Record<string, string> = {};
97
- let i = 0;
98
- while (i < data.length) {
99
- const sp = data.indexOf(0x20, i);
100
- if (sp === -1) break;
101
- const len = parseInt(data.subarray(i, sp).toString("ascii"), 10);
102
- if (!Number.isInteger(len) || len <= 0 || i + len > data.length) fail("bad PAX header record");
103
- const rec = data.subarray(sp + 1, i + len - 1).toString("utf8");
104
- const eq = rec.indexOf("=");
105
- if (eq > 0) out[rec.slice(0, eq)] = rec.slice(eq + 1);
106
- i += len;
107
- }
108
- return out;
109
- }
110
-
111
- /** Normalize a member name: strip leading "./" and "/", collapse "..". Returns "" for the root. */
112
- export function normName(name: string): string {
113
- const n = posix.normalize(`/${name}`).slice(1);
114
- return n.replace(/\/+$/, "");
115
- }
116
-
117
- /** Index every member of the tar archive open at `fd` (`size` bytes long). */
118
- export function indexTar(fd: number, size: number): TarIndex {
119
- const files = new Map<string, TarMember>();
120
- const links = new Map<string, { target: string; hard: boolean }>();
121
- let pos = 0;
122
- let members = 0;
123
- let pax: Record<string, string> = {};
124
- let longName: string | undefined;
125
- let longLink: string | undefined;
126
-
127
- while (pos + BLOCK <= size) {
128
- const hdr = readAt(fd, pos, BLOCK);
129
- if (hdr.every((b) => b === 0)) break; // end-of-archive marker
130
- if (!checksumOk(hdr)) fail(`bad tar header checksum at byte ${pos}`);
131
- if (++members > MAX_TAR_MEMBERS) fail(`more than ${MAX_TAR_MEMBERS} members`);
132
-
133
- const type = String.fromCharCode(hdr[156]!);
134
- let memberSize = parseSize(hdr);
135
- const dataAt = pos + BLOCK;
136
- const next = dataAt + Math.ceil(memberSize / BLOCK) * BLOCK;
137
-
138
- if (type === "x" || type === "g" || type === "L" || type === "K") {
139
- if (memberSize > MAX_EXT_HEADER) fail("extended header too large");
140
- const data = readAt(fd, dataAt, memberSize);
141
- if (type === "x") pax = parsePax(data);
142
- else if (type === "L") longName = cstr(data, 0, data.length);
143
- else if (type === "K") longLink = cstr(data, 0, data.length);
144
- // "g" (global PAX) carries nothing this reader needs.
145
- pos = next;
146
- continue;
147
- }
148
-
149
- const prefix = cstr(hdr, 345, 155);
150
- const rawName = pax.path ?? longName ?? (prefix ? `${prefix}/${cstr(hdr, 0, 100)}` : cstr(hdr, 0, 100));
151
- const linkName = pax.linkpath ?? longLink ?? cstr(hdr, 157, 100);
152
- if (pax.size !== undefined) {
153
- if (!/^\d+$/.test(pax.size)) fail("bad PAX size");
154
- memberSize = Number(pax.size);
155
- }
156
- const dataEnd = dataAt + Math.ceil(memberSize / BLOCK) * BLOCK;
157
- pax = {};
158
- longName = undefined;
159
- longLink = undefined;
160
-
161
- const name = normName(rawName);
162
- if (type === "0" || type === "\0" || type === "7") {
163
- if (dataAt + memberSize > size) fail(`member "${name}" runs past the end of the file`);
164
- files.set(name, { offset: dataAt, size: memberSize });
165
- links.delete(name);
166
- } else if (type === "1") {
167
- links.set(name, { target: normName(linkName), hard: true });
168
- files.delete(name);
169
- } else if (type === "2") {
170
- links.set(name, { target: normName(posix.join(posix.dirname(name), linkName)), hard: false });
171
- files.delete(name);
172
- }
173
- pos = dataEnd;
174
- }
175
-
176
- // Resolve links to the regular member they finally point at, within the archive only.
177
- for (const [name, link] of links) {
178
- let target = link.target;
179
- for (let depth = 0; ; depth++) {
180
- if (depth > MAX_LINK_DEPTH) fail(`link loop at "${name}"`);
181
- const hit = files.get(target);
182
- if (hit) {
183
- files.set(name, hit);
184
- break;
185
- }
186
- const hop = links.get(target);
187
- if (!hop) break; // dangling link: simply not a readable member
188
- target = hop.target;
189
- }
190
- }
191
- return { files };
192
- }
193
-
194
- /** Read one (small) member fully, refusing anything larger than `max`. */
195
- export function readMember(fd: number, m: TarMember, max: number, what: string): Buffer {
196
- if (m.size > max) fail(`${what} is ${m.size} bytes — larger than the ${max}-byte limit`);
197
- return readAt(fd, m.offset, m.size);
198
- }
199
-
200
- /** Read the first `n` bytes of a member (for magic-number sniffing). */
201
- export function peekMember(fd: number, m: TarMember, n: number): Buffer {
202
- return readAt(fd, m.offset, Math.min(n, m.size));
203
- }