@magland/mochi 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +108 -0
- package/dist/ansi.js +174 -0
- package/dist/api/admin.js +416 -0
- package/dist/api/auth.js +166 -0
- package/dist/api/backup.js +598 -0
- package/dist/api/ci.js +336 -0
- package/dist/api/contents.js +339 -0
- package/dist/api/issues.js +165 -0
- package/dist/api/pulls.js +244 -0
- package/dist/api/releases.js +83 -0
- package/dist/api/repos.js +156 -0
- package/dist/api/write.js +518 -0
- package/dist/api.js +326 -0
- package/dist/assets.js +29 -0
- package/dist/atom.js +32 -0
- package/dist/atomic.js +171 -0
- package/dist/avatar.js +81 -0
- package/dist/browse.js +630 -0
- package/dist/build-info.json +4 -0
- package/dist/ci/actionref.js +86 -0
- package/dist/ci/api.js +829 -0
- package/dist/ci/artifacts.js +201 -0
- package/dist/ci/dispatch.js +30 -0
- package/dist/ci/engine.js +1321 -0
- package/dist/ci/expr.js +526 -0
- package/dist/ci/manual.js +199 -0
- package/dist/ci/present.js +82 -0
- package/dist/ci/protocol.js +6 -0
- package/dist/ci/runners.js +256 -0
- package/dist/ci/runs.js +208 -0
- package/dist/ci/trigger.js +28 -0
- package/dist/ci/views.js +441 -0
- package/dist/ci/wake.js +194 -0
- package/dist/ci/web.js +617 -0
- package/dist/ci/workflow.js +436 -0
- package/dist/cli/admin-cmd.js +324 -0
- package/dist/cli/api-cmd.js +128 -0
- package/dist/cli/backup-cmd.js +1500 -0
- package/dist/cli/exit.js +69 -0
- package/dist/cli/input.js +64 -0
- package/dist/cli/issue-cmd.js +243 -0
- package/dist/cli/output.js +93 -0
- package/dist/cli/parse.js +317 -0
- package/dist/cli/pr-cmd.js +289 -0
- package/dist/cli/release-cmd.js +171 -0
- package/dist/cli/repo-cmd.js +763 -0
- package/dist/cli/repo.js +101 -0
- package/dist/cli/run-cmd.js +438 -0
- package/dist/cli/target.js +54 -0
- package/dist/cli-api.js +84 -0
- package/dist/compare.js +111 -0
- package/dist/config.js +212 -0
- package/dist/credentials.js +235 -0
- package/dist/deploy-cli.js +859 -0
- package/dist/deploy-runner-cli.js +592 -0
- package/dist/diff.js +171 -0
- package/dist/discussion.js +253 -0
- package/dist/egress.js +559 -0
- package/dist/filecache.js +68 -0
- package/dist/find.js +162 -0
- package/dist/forms.js +737 -0
- package/dist/git.js +547 -0
- package/dist/githttp.js +428 -0
- package/dist/html.js +87 -0
- package/dist/icons.js +101 -0
- package/dist/import-cli.js +316 -0
- package/dist/index.js +752 -0
- package/dist/issues.js +308 -0
- package/dist/issueweb.js +447 -0
- package/dist/job-cli.js +197 -0
- package/dist/jobtoken.js +96 -0
- package/dist/languages.js +383 -0
- package/dist/layout.js +100 -0
- package/dist/lfs.js +438 -0
- package/dist/lfsstore.js +425 -0
- package/dist/limit.js +259 -0
- package/dist/logo.js +61 -0
- package/dist/markdown.js +382 -0
- package/dist/migrate.js +334 -0
- package/dist/multipart.js +90 -0
- package/dist/ops.js +869 -0
- package/dist/pagescript.js +465 -0
- package/dist/perms.js +370 -0
- package/dist/pointer.js +55 -0
- package/dist/profile.js +106 -0
- package/dist/pulls.js +320 -0
- package/dist/pullweb.js +461 -0
- package/dist/redirects.js +455 -0
- package/dist/releases.js +435 -0
- package/dist/render.js +233 -0
- package/dist/runner/actions.js +448 -0
- package/dist/runner/client.js +428 -0
- package/dist/runner/context.js +247 -0
- package/dist/runner/docker.js +197 -0
- package/dist/runner/externals.js +175 -0
- package/dist/runner/job.js +290 -0
- package/dist/runner/manual-run.js +272 -0
- package/dist/runner/overrides.js +554 -0
- package/dist/runner/steps.js +571 -0
- package/dist/runner/wake.js +84 -0
- package/dist/runner-cli.js +405 -0
- package/dist/scan.js +231 -0
- package/dist/server.js +424 -0
- package/dist/session.js +267 -0
- package/dist/site.js +259 -0
- package/dist/siteshost.js +94 -0
- package/dist/source.js +90 -0
- package/dist/style.js +1295 -0
- package/dist/themes.js +369 -0
- package/dist/vault.js +442 -0
- package/dist/version.js +88 -0
- package/dist/views.js +1007 -0
- package/dist/web.js +182 -0
- package/dist/webops.js +1402 -0
- package/package.json +71 -0
package/dist/lfsstore.js
ADDED
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.LfsConfigError = void 0;
|
|
37
|
+
exports.lfsKey = lfsKey;
|
|
38
|
+
exports.localLfsDir = localLfsDir;
|
|
39
|
+
exports.localObjectPath = localObjectPath;
|
|
40
|
+
exports.signTransfer = signTransfer;
|
|
41
|
+
exports.verifyTransfer = verifyTransfer;
|
|
42
|
+
exports.createLfsStore = createLfsStore;
|
|
43
|
+
const aws4fetch_1 = require("aws4fetch");
|
|
44
|
+
const crypto = __importStar(require("crypto"));
|
|
45
|
+
const fs = __importStar(require("fs"));
|
|
46
|
+
const path = __importStar(require("path"));
|
|
47
|
+
const ops_1 = require("./ops");
|
|
48
|
+
const layout_1 = require("./layout");
|
|
49
|
+
const scan_1 = require("./scan");
|
|
50
|
+
const session_1 = require("./session");
|
|
51
|
+
// Storage for Git LFS objects. Two backends behind one interface: "s3" issues
|
|
52
|
+
// presigned URLs against any S3-compatible bucket, so large-file bytes never
|
|
53
|
+
// pass through this process; "local" stores objects inside the vault and
|
|
54
|
+
// issues HMAC-signed URLs pointing back at mochi's own transfer routes, so
|
|
55
|
+
// LFS works with no credentials (dev, smoke test, laptop vaults). The choice
|
|
56
|
+
// is made from the environment at startup and never recorded in the vault.
|
|
57
|
+
//
|
|
58
|
+
// Objects are sharded the way git-lfs itself shards them on the client:
|
|
59
|
+
// <collection>/<repo>.lfs/<oid[0:2]>/<oid[2:4]>/<oid>
|
|
60
|
+
// That is a bucket key as written; on the volume the same shards sit under
|
|
61
|
+
// the repository, at <vault>/collections/<collection>/repos/<repo>.lfs/. The
|
|
62
|
+
// key kept its shape when the vault's layout changed, since a bucket is not
|
|
63
|
+
// the vault's directory and rewriting every key would mean moving objects
|
|
64
|
+
// nobody asked to move.
|
|
65
|
+
const EXPIRES_SECONDS = 3600;
|
|
66
|
+
const DEFAULT_MAX_SIZE = 5_000_000_000; // below the S3 single-PUT ceiling of 5 GB
|
|
67
|
+
class LfsConfigError extends Error {
|
|
68
|
+
}
|
|
69
|
+
exports.LfsConfigError = LfsConfigError;
|
|
70
|
+
// Content-Disposition filenames must stay inside printable ASCII with no
|
|
71
|
+
// quote or backslash, or the header becomes malformed.
|
|
72
|
+
function sanitizeFilename(name) {
|
|
73
|
+
return name.replace(/[^\x20-\x7e]|["\\]/g, '_');
|
|
74
|
+
}
|
|
75
|
+
// The route layer validates names with isValidName and object ids against
|
|
76
|
+
// /^[0-9a-f]{64}$/ before anything reaches this module. Re-checking here means
|
|
77
|
+
// a future caller that forgets cannot turn a name or an id into a path
|
|
78
|
+
// outside the vault, or into a key outside the repository's own prefix.
|
|
79
|
+
function checkTarget(collection, repo, oid) {
|
|
80
|
+
if (!(0, scan_1.isValidName)(collection) || !(0, scan_1.isValidName)(repo)) {
|
|
81
|
+
throw new Error(`invalid collection or repository name for LFS storage: ${collection}/${repo}`);
|
|
82
|
+
}
|
|
83
|
+
if (!/^[0-9a-f]{64}$/.test(oid)) {
|
|
84
|
+
throw new Error('invalid LFS object id');
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function checkRepoTarget(collection, repo) {
|
|
88
|
+
if (!(0, scan_1.isValidName)(collection) || !(0, scan_1.isValidName)(repo)) {
|
|
89
|
+
throw new Error(`invalid collection or repository name for LFS storage: ${collection}/${repo}`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function lfsKey(collection, repo, oid) {
|
|
93
|
+
checkTarget(collection, repo, oid);
|
|
94
|
+
return `${collection}/${repo}.lfs/${oid.slice(0, 2)}/${oid.slice(2, 4)}/${oid}`;
|
|
95
|
+
}
|
|
96
|
+
// ---- the local backend ----
|
|
97
|
+
function localLfsDir(root, collection, repo) {
|
|
98
|
+
checkRepoTarget(collection, repo);
|
|
99
|
+
return (0, layout_1.repoPath)(root, collection, `${repo}.lfs`);
|
|
100
|
+
}
|
|
101
|
+
function localObjectPath(root, collection, repo, oid) {
|
|
102
|
+
checkTarget(collection, repo, oid);
|
|
103
|
+
return path.join(localLfsDir(root, collection, repo), oid.slice(0, 2), oid.slice(2, 4), oid);
|
|
104
|
+
}
|
|
105
|
+
// HMAC over a NUL-delimited payload with an explicit domain prefix. The same
|
|
106
|
+
// secret signs session cookies; the "lfs" prefix guarantees an LFS href can
|
|
107
|
+
// never be replayed as a session token. Every value that appears in the URL
|
|
108
|
+
// is inside the signed payload, so none of them can be altered.
|
|
109
|
+
function signTransfer(root, op, collection, repo, oid, exp, disp) {
|
|
110
|
+
const payload = ['lfs', op, collection, repo, oid, String(exp), disp].join('\0');
|
|
111
|
+
return crypto.createHmac('sha256', (0, session_1.getSecret)(root)).update(payload).digest('base64url');
|
|
112
|
+
}
|
|
113
|
+
function verifyTransfer(root, op, collection, repo, oid, exp, disp, sig) {
|
|
114
|
+
const a = Buffer.from(sig);
|
|
115
|
+
const b = Buffer.from(signTransfer(root, op, collection, repo, oid, exp, disp));
|
|
116
|
+
return a.length === b.length && crypto.timingSafeEqual(a, b);
|
|
117
|
+
}
|
|
118
|
+
class LocalLfsStore {
|
|
119
|
+
root;
|
|
120
|
+
kind = 'local';
|
|
121
|
+
constructor(root) {
|
|
122
|
+
this.root = root;
|
|
123
|
+
}
|
|
124
|
+
async head(collection, repo, oid) {
|
|
125
|
+
// The path is built outside the try so a rejected name or object id
|
|
126
|
+
// propagates as an error; only a missing file becomes null.
|
|
127
|
+
const file = localObjectPath(this.root, collection, repo, oid);
|
|
128
|
+
try {
|
|
129
|
+
const st = fs.statSync(file);
|
|
130
|
+
return st.isFile() ? { size: st.size } : null;
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
// The hrefs are relative; the route layer prefixes the request's own base
|
|
137
|
+
// URL, since the store does not know what host it is being served under.
|
|
138
|
+
sign(op, collection, repo, oid, disp) {
|
|
139
|
+
const exp = Math.floor(Date.now() / 1000) + EXPIRES_SECONDS;
|
|
140
|
+
const sig = signTransfer(this.root, op, collection, repo, oid, exp, disp);
|
|
141
|
+
const q = new URLSearchParams({ exp: String(exp), sig });
|
|
142
|
+
if (disp !== '')
|
|
143
|
+
q.set('disp', disp);
|
|
144
|
+
const href = `/${encodeURIComponent(collection)}/${encodeURIComponent(repo)}/info/lfs/objects/${oid}?${q.toString()}`;
|
|
145
|
+
return { href, header: {}, expiresIn: EXPIRES_SECONDS };
|
|
146
|
+
}
|
|
147
|
+
async signDownload(collection, repo, oid, opts) {
|
|
148
|
+
const disp = opts?.filename ? sanitizeFilename(path.basename(opts.filename)) : '';
|
|
149
|
+
return this.sign('download', collection, repo, oid, disp);
|
|
150
|
+
}
|
|
151
|
+
async signUpload(collection, repo, oid, _size) {
|
|
152
|
+
return this.sign('upload', collection, repo, oid, '');
|
|
153
|
+
}
|
|
154
|
+
async deleteRepo(collection, repo) {
|
|
155
|
+
const dir = localLfsDir(this.root, collection, repo);
|
|
156
|
+
let rootReal;
|
|
157
|
+
try {
|
|
158
|
+
rootReal = fs.realpathSync(this.root);
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
// containedIn resolves the real path, so a missing directory (already
|
|
164
|
+
// deleted, or never created) is a no-op rather than an error.
|
|
165
|
+
if (!(0, ops_1.containedIn)(rootReal, dir))
|
|
166
|
+
return;
|
|
167
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
168
|
+
}
|
|
169
|
+
// The objects are a directory beside the repository, so moving them is
|
|
170
|
+
// moving that directory. A repository with no LFS objects has none.
|
|
171
|
+
async renameRepo(collection, repo, toCollection, toRepo) {
|
|
172
|
+
const from = localLfsDir(this.root, collection, repo);
|
|
173
|
+
const to = localLfsDir(this.root, toCollection, toRepo);
|
|
174
|
+
let rootReal;
|
|
175
|
+
try {
|
|
176
|
+
rootReal = fs.realpathSync(this.root);
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
if (!(0, ops_1.containedIn)(rootReal, from))
|
|
182
|
+
return;
|
|
183
|
+
if (fs.existsSync(to))
|
|
184
|
+
throw new Error(`LFS objects already exist at ${toCollection}/${toRepo}`);
|
|
185
|
+
fs.mkdirSync(path.dirname(to), { recursive: true });
|
|
186
|
+
fs.renameSync(from, to);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
// SigV4 signs a canonical query string in which a space is `%20`, but the URL
|
|
190
|
+
// aws4fetch hands back is serialized by URLSearchParams, which writes a space
|
|
191
|
+
// as `+`. The two disagree for any parameter containing a space, which for us
|
|
192
|
+
// is every `response-content-disposition` (the `; ` alone contains one), and
|
|
193
|
+
// the bucket then answers 403 SignatureDoesNotMatch. Rewriting `+` to `%20`
|
|
194
|
+
// in the query is unambiguous: URLSearchParams emits a literal plus as `%2B`,
|
|
195
|
+
// so no `+` here ever stands for itself.
|
|
196
|
+
function fixQueryEncoding(href) {
|
|
197
|
+
const i = href.indexOf('?');
|
|
198
|
+
if (i === -1)
|
|
199
|
+
return href;
|
|
200
|
+
return href.slice(0, i) + href.slice(i).replace(/\+/g, '%20');
|
|
201
|
+
}
|
|
202
|
+
function escapeXml(s) {
|
|
203
|
+
return s
|
|
204
|
+
.replace(/&/g, '&')
|
|
205
|
+
.replace(/</g, '<')
|
|
206
|
+
.replace(/>/g, '>')
|
|
207
|
+
.replace(/"/g, '"');
|
|
208
|
+
}
|
|
209
|
+
function decodeXml(s) {
|
|
210
|
+
return s
|
|
211
|
+
.replace(/</g, '<')
|
|
212
|
+
.replace(/>/g, '>')
|
|
213
|
+
.replace(/"/g, '"')
|
|
214
|
+
.replace(/'/g, "'")
|
|
215
|
+
.replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(parseInt(n, 10)))
|
|
216
|
+
.replace(/&/g, '&');
|
|
217
|
+
}
|
|
218
|
+
class S3LfsStore {
|
|
219
|
+
kind = 's3';
|
|
220
|
+
bucket;
|
|
221
|
+
endpoint;
|
|
222
|
+
aws;
|
|
223
|
+
baseUrl;
|
|
224
|
+
prefix;
|
|
225
|
+
constructor(opts) {
|
|
226
|
+
this.bucket = opts.bucket;
|
|
227
|
+
this.endpoint = opts.endpoint;
|
|
228
|
+
this.prefix = opts.prefix;
|
|
229
|
+
this.aws = new aws4fetch_1.AwsClient({
|
|
230
|
+
accessKeyId: opts.accessKeyId,
|
|
231
|
+
secretAccessKey: opts.secretAccessKey,
|
|
232
|
+
region: opts.region,
|
|
233
|
+
service: 's3',
|
|
234
|
+
});
|
|
235
|
+
let url;
|
|
236
|
+
try {
|
|
237
|
+
url = new URL(opts.endpoint);
|
|
238
|
+
}
|
|
239
|
+
catch {
|
|
240
|
+
throw new LfsConfigError(`the LFS endpoint is not a valid URL: ${opts.endpoint}`);
|
|
241
|
+
}
|
|
242
|
+
if (opts.addressing === 'vhost') {
|
|
243
|
+
url.host = `${opts.bucket}.${url.host}`;
|
|
244
|
+
this.baseUrl = url.origin;
|
|
245
|
+
}
|
|
246
|
+
else {
|
|
247
|
+
this.baseUrl = `${url.origin}/${opts.bucket}`;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
key(collection, repo, oid) {
|
|
251
|
+
return this.prefix + lfsKey(collection, repo, oid);
|
|
252
|
+
}
|
|
253
|
+
objectUrl(collection, repo, oid) {
|
|
254
|
+
return new URL(`${this.baseUrl}/${this.key(collection, repo, oid)}`);
|
|
255
|
+
}
|
|
256
|
+
async head(collection, repo, oid) {
|
|
257
|
+
const res = await this.aws.fetch(this.objectUrl(collection, repo, oid).toString(), { method: 'HEAD' });
|
|
258
|
+
if (res.status === 404)
|
|
259
|
+
return null;
|
|
260
|
+
if (!res.ok)
|
|
261
|
+
throw new Error(`LFS bucket HEAD returned ${res.status}`);
|
|
262
|
+
const len = res.headers.get('content-length');
|
|
263
|
+
return { size: len ? parseInt(len, 10) : 0 };
|
|
264
|
+
}
|
|
265
|
+
async signDownload(collection, repo, oid, opts) {
|
|
266
|
+
const url = this.objectUrl(collection, repo, oid);
|
|
267
|
+
url.searchParams.set('X-Amz-Expires', String(EXPIRES_SECONDS));
|
|
268
|
+
if (opts?.filename) {
|
|
269
|
+
url.searchParams.set('response-content-disposition', `attachment; filename="${sanitizeFilename(path.basename(opts.filename))}"`);
|
|
270
|
+
}
|
|
271
|
+
const signed = await this.aws.sign(new Request(url.toString(), { method: 'GET' }), {
|
|
272
|
+
aws: { signQuery: true },
|
|
273
|
+
});
|
|
274
|
+
return { href: fixQueryEncoding(signed.url), header: {}, expiresIn: EXPIRES_SECONDS };
|
|
275
|
+
}
|
|
276
|
+
// No Content-Type is signed: R2 requires the client to send a header
|
|
277
|
+
// exactly matching any content type in the signature, and git-lfs sends its
|
|
278
|
+
// own, so signing one is a reliable source of 403s at upload time.
|
|
279
|
+
async signUpload(collection, repo, oid, _size) {
|
|
280
|
+
const url = this.objectUrl(collection, repo, oid);
|
|
281
|
+
url.searchParams.set('X-Amz-Expires', String(EXPIRES_SECONDS));
|
|
282
|
+
const signed = await this.aws.sign(new Request(url.toString(), { method: 'PUT' }), {
|
|
283
|
+
aws: { signQuery: true },
|
|
284
|
+
});
|
|
285
|
+
return { href: fixQueryEncoding(signed.url), header: {}, expiresIn: EXPIRES_SECONDS };
|
|
286
|
+
}
|
|
287
|
+
// List the repository's key prefix and delete in batches of up to 1000,
|
|
288
|
+
// following continuation tokens. Idempotent: deleting keys that are already
|
|
289
|
+
// gone succeeds, and an empty listing does nothing.
|
|
290
|
+
async deleteRepo(collection, repo) {
|
|
291
|
+
// Validated before it becomes a delete prefix: an unchecked name here
|
|
292
|
+
// would widen the prefix and delete other repositories' objects.
|
|
293
|
+
checkRepoTarget(collection, repo);
|
|
294
|
+
for (const keys of await this.listKeys(`${this.prefix}${collection}/${repo}.lfs/`)) {
|
|
295
|
+
await this.deleteKeys(keys);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Objects carry the repository in their key, so moving a repository means
|
|
300
|
+
* copying every object to the new prefix and deleting the old ones. There
|
|
301
|
+
* is no rename in S3; a server-side copy is the closest thing, and it never
|
|
302
|
+
* moves bytes through this process.
|
|
303
|
+
*/
|
|
304
|
+
async renameRepo(collection, repo, toCollection, toRepo) {
|
|
305
|
+
checkRepoTarget(collection, repo);
|
|
306
|
+
checkRepoTarget(toCollection, toRepo);
|
|
307
|
+
const fromPrefix = `${this.prefix}${collection}/${repo}.lfs/`;
|
|
308
|
+
const toPrefix = `${this.prefix}${toCollection}/${toRepo}.lfs/`;
|
|
309
|
+
for (const keys of await this.listKeys(fromPrefix)) {
|
|
310
|
+
for (const key of keys) {
|
|
311
|
+
const target = toPrefix + key.slice(fromPrefix.length);
|
|
312
|
+
const res = await this.aws.fetch(`${this.baseUrl}/${target}`, {
|
|
313
|
+
method: 'PUT',
|
|
314
|
+
headers: { 'x-amz-copy-source': `/${this.bucket}/${key}` },
|
|
315
|
+
});
|
|
316
|
+
if (!res.ok)
|
|
317
|
+
throw new Error(`LFS bucket COPY returned ${res.status}`);
|
|
318
|
+
}
|
|
319
|
+
await this.deleteKeys(keys);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
/** Every key under a prefix, in the batches the listing returns them in. */
|
|
323
|
+
async listKeys(prefix) {
|
|
324
|
+
const batches = [];
|
|
325
|
+
let continuation;
|
|
326
|
+
do {
|
|
327
|
+
const listUrl = new URL(`${this.baseUrl}/`);
|
|
328
|
+
listUrl.searchParams.set('list-type', '2');
|
|
329
|
+
listUrl.searchParams.set('prefix', prefix);
|
|
330
|
+
if (continuation)
|
|
331
|
+
listUrl.searchParams.set('continuation-token', continuation);
|
|
332
|
+
const res = await this.aws.fetch(listUrl.toString());
|
|
333
|
+
if (!res.ok)
|
|
334
|
+
throw new Error(`LFS bucket LIST returned ${res.status}`);
|
|
335
|
+
const xml = await res.text();
|
|
336
|
+
const keys = [...xml.matchAll(/<Key>([^<]*)<\/Key>/g)].map((m) => decodeXml(m[1]));
|
|
337
|
+
if (keys.length > 0)
|
|
338
|
+
batches.push(keys);
|
|
339
|
+
continuation = undefined;
|
|
340
|
+
if (/<IsTruncated>true<\/IsTruncated>/.test(xml)) {
|
|
341
|
+
const m = xml.match(/<NextContinuationToken>([^<]*)<\/NextContinuationToken>/);
|
|
342
|
+
// Stopping quietly here would leave the rest of the repository's
|
|
343
|
+
// objects behind with nothing left to point at them, so raise it.
|
|
344
|
+
if (!m)
|
|
345
|
+
throw new Error('LFS bucket listing was truncated without a continuation token');
|
|
346
|
+
continuation = decodeXml(m[1]);
|
|
347
|
+
}
|
|
348
|
+
} while (continuation);
|
|
349
|
+
return batches;
|
|
350
|
+
}
|
|
351
|
+
async deleteKeys(keys) {
|
|
352
|
+
const body = '<Delete><Quiet>true</Quiet>' +
|
|
353
|
+
keys.map((k) => `<Object><Key>${escapeXml(k)}</Key></Object>`).join('') +
|
|
354
|
+
'</Delete>';
|
|
355
|
+
const url = new URL(`${this.baseUrl}/`);
|
|
356
|
+
url.searchParams.set('delete', '');
|
|
357
|
+
const res = await this.aws.fetch(url.toString(), {
|
|
358
|
+
method: 'POST',
|
|
359
|
+
headers: {
|
|
360
|
+
// DeleteObjects requires Content-MD5 on AWS; harmless elsewhere.
|
|
361
|
+
'Content-MD5': crypto.createHash('md5').update(body).digest('base64'),
|
|
362
|
+
'Content-Type': 'application/xml',
|
|
363
|
+
},
|
|
364
|
+
body,
|
|
365
|
+
});
|
|
366
|
+
if (!res.ok)
|
|
367
|
+
throw new Error(`LFS bucket DELETE returned ${res.status}`);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
// ---- backend selection ----
|
|
371
|
+
// Credentials come from the environment only; nothing about the choice is
|
|
372
|
+
// recorded in the vault, which stays portable between deployments. The
|
|
373
|
+
// BUCKET_NAME and AWS_* spellings are honored so a Fly deployment using
|
|
374
|
+
// Tigris works with the credentials Fly injects.
|
|
375
|
+
function createLfsStore(root, env = process.env) {
|
|
376
|
+
let maxSize = DEFAULT_MAX_SIZE;
|
|
377
|
+
if (env.MOCHI_LFS_MAX_SIZE !== undefined) {
|
|
378
|
+
maxSize = parseInt(env.MOCHI_LFS_MAX_SIZE, 10);
|
|
379
|
+
if (!Number.isSafeInteger(maxSize) || maxSize <= 0) {
|
|
380
|
+
throw new LfsConfigError(`MOCHI_LFS_MAX_SIZE must be a positive integer, got: ${env.MOCHI_LFS_MAX_SIZE}`);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
const local = () => ({
|
|
384
|
+
store: new LocalLfsStore(root),
|
|
385
|
+
maxSize,
|
|
386
|
+
label: 'local (objects stored inside the vault)',
|
|
387
|
+
offloaded: false,
|
|
388
|
+
});
|
|
389
|
+
if (env.MOCHI_LFS === 'off')
|
|
390
|
+
return local();
|
|
391
|
+
const vars = [
|
|
392
|
+
['MOCHI_LFS_BUCKET (or BUCKET_NAME)', env.MOCHI_LFS_BUCKET || env.BUCKET_NAME],
|
|
393
|
+
['MOCHI_LFS_ENDPOINT (or AWS_ENDPOINT_URL_S3)', env.MOCHI_LFS_ENDPOINT || env.AWS_ENDPOINT_URL_S3],
|
|
394
|
+
['AWS_ACCESS_KEY_ID', env.AWS_ACCESS_KEY_ID],
|
|
395
|
+
['AWS_SECRET_ACCESS_KEY', env.AWS_SECRET_ACCESS_KEY],
|
|
396
|
+
];
|
|
397
|
+
const missing = vars.filter(([, v]) => !v).map(([name]) => name);
|
|
398
|
+
if (missing.length === vars.length)
|
|
399
|
+
return local();
|
|
400
|
+
if (missing.length > 0) {
|
|
401
|
+
// A partially configured deployment must not silently fall back to
|
|
402
|
+
// storing large objects on the volume; that is the exact failure the
|
|
403
|
+
// bucket backend exists to prevent.
|
|
404
|
+
throw new LfsConfigError(`Git LFS bucket configuration is incomplete; missing: ${missing.join(', ')}. ` +
|
|
405
|
+
`Set the missing variables, or set MOCHI_LFS=off to store LFS objects inside the vault.`);
|
|
406
|
+
}
|
|
407
|
+
const [bucket, endpoint] = [vars[0][1], vars[1][1]];
|
|
408
|
+
const addressing = env.MOCHI_LFS_ADDRESSING ?? 'path';
|
|
409
|
+
if (addressing !== 'path' && addressing !== 'vhost') {
|
|
410
|
+
throw new LfsConfigError(`MOCHI_LFS_ADDRESSING must be "path" or "vhost", got: ${addressing}`);
|
|
411
|
+
}
|
|
412
|
+
let prefix = env.MOCHI_LFS_PREFIX ?? '';
|
|
413
|
+
if (prefix !== '')
|
|
414
|
+
prefix = prefix.replace(/^\/+|\/+$/g, '') + '/';
|
|
415
|
+
const store = new S3LfsStore({
|
|
416
|
+
bucket,
|
|
417
|
+
endpoint,
|
|
418
|
+
accessKeyId: env.AWS_ACCESS_KEY_ID,
|
|
419
|
+
secretAccessKey: env.AWS_SECRET_ACCESS_KEY,
|
|
420
|
+
region: env.AWS_REGION || 'auto',
|
|
421
|
+
prefix,
|
|
422
|
+
addressing,
|
|
423
|
+
});
|
|
424
|
+
return { store, maxSize, label: `s3 (endpoint ${endpoint}, bucket ${bucket})`, offloaded: true };
|
|
425
|
+
}
|
package/dist/limit.js
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BUSY_RETRY_SECONDS = void 0;
|
|
4
|
+
exports.clientKey = clientKey;
|
|
5
|
+
exports.createLimiter = createLimiter;
|
|
6
|
+
exports.createGate = createGate;
|
|
7
|
+
exports.createGates = createGates;
|
|
8
|
+
exports.createAuthLimiter = createAuthLimiter;
|
|
9
|
+
// Rate limiting and abuse controls, in two primitives and one key function.
|
|
10
|
+
//
|
|
11
|
+
// Both primitives return a decision rather than writing a response, the way
|
|
12
|
+
// checkPushAuth does, because the callers refuse in four content types: HTML for
|
|
13
|
+
// the web, JSON for the API, plain text for git, and LFS's own JSON shape.
|
|
14
|
+
//
|
|
15
|
+
// Nothing here survives a restart or is shared between processes. Rate-limit
|
|
16
|
+
// state is high-frequency and worthless once stale, and it does not belong in a
|
|
17
|
+
// vault directory whose whole design is plain files written durably. Two servers
|
|
18
|
+
// pointed at one vault therefore count separately, and a restart forgives every
|
|
19
|
+
// offender. Both are documented limitations rather than bugs to engineer around.
|
|
20
|
+
/**
|
|
21
|
+
* The address a limit is charged to. Express's req.ip reads X-Forwarded-For when
|
|
22
|
+
* trust proxy is set, which is client-supplied, so it is used only when the
|
|
23
|
+
* operator has said a proxy is in front. Otherwise the socket's peer address is
|
|
24
|
+
* the only thing an attacker cannot choose.
|
|
25
|
+
*
|
|
26
|
+
* Every limiter keys on this. No route may key on req.ip directly, or a vault
|
|
27
|
+
* exposed without a proxy would have every per-address limit defeated by one
|
|
28
|
+
* fabricated header, and the limiter's own key space would be unbounded.
|
|
29
|
+
*/
|
|
30
|
+
function clientKey(req) {
|
|
31
|
+
const trusted = Boolean(req.app?.get('trust proxy'));
|
|
32
|
+
const raw = trusted ? req.ip ?? req.socket.remoteAddress : req.socket.remoteAddress;
|
|
33
|
+
return normalizeAddress(raw ?? 'unknown');
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* One address, one key. Without this an attacker gets a fresh bucket per
|
|
37
|
+
* spelling of the same address.
|
|
38
|
+
*
|
|
39
|
+
* IPv6 addresses are deliberately not aggregated into a /64. That would be the
|
|
40
|
+
* more effective choice against a single host with a routed prefix and the more
|
|
41
|
+
* damaging one behind a CGNAT or a university, and guessing wrong is worse than
|
|
42
|
+
* the coarse behaviour.
|
|
43
|
+
*/
|
|
44
|
+
function normalizeAddress(address) {
|
|
45
|
+
const lower = address.trim().toLowerCase();
|
|
46
|
+
return lower.startsWith('::ffff:') ? lower.slice('::ffff:'.length) : lower;
|
|
47
|
+
}
|
|
48
|
+
const OK = { ok: true, retryAfter: 0 };
|
|
49
|
+
/** How many expired entries one call may clear, so that a request cannot pay for a very large map. */
|
|
50
|
+
const SWEEP_BUDGET = 64;
|
|
51
|
+
/**
|
|
52
|
+
* A fixed-window counter. A fixed window lets twice the limit through across a
|
|
53
|
+
* window boundary; a token bucket would not, and buys a smoothness none of these
|
|
54
|
+
* limits need.
|
|
55
|
+
*
|
|
56
|
+
* maxKeys is a hard ceiling, because an unbounded map keyed on anything an
|
|
57
|
+
* attacker influences is itself a memory-exhaustion vector. When the map is full
|
|
58
|
+
* and a sweep frees nothing, a hit on an unseen key is allowed. That is failing
|
|
59
|
+
* open, and it is the right way round: a limiter that refuses everyone the moment
|
|
60
|
+
* it is full has become the outage it exists to prevent. clientKey is what makes
|
|
61
|
+
* this rare, by bounding the key space to real peers.
|
|
62
|
+
*/
|
|
63
|
+
function createLimiter(opts) {
|
|
64
|
+
const { limit, windowMs, maxKeys } = opts;
|
|
65
|
+
const counts = new Map();
|
|
66
|
+
let fullWarnedAt = 0;
|
|
67
|
+
const retryAfter = (resetAt) => Math.max(1, Math.ceil((resetAt - Date.now()) / 1000));
|
|
68
|
+
// Amortised over a bounded number of entries, and only on a write, so one
|
|
69
|
+
// request cannot pay for a very large map.
|
|
70
|
+
const sweep = (now) => {
|
|
71
|
+
let looked = 0;
|
|
72
|
+
for (const [key, entry] of counts) {
|
|
73
|
+
if (looked++ >= SWEEP_BUDGET)
|
|
74
|
+
break;
|
|
75
|
+
if (entry.resetAt <= now)
|
|
76
|
+
counts.delete(key);
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
return {
|
|
80
|
+
// check and hit agree on the boundary: after exactly `limit` events in a
|
|
81
|
+
// window, check says not ok and the next hit would too.
|
|
82
|
+
check(key) {
|
|
83
|
+
if (limit <= 0)
|
|
84
|
+
return OK;
|
|
85
|
+
const entry = counts.get(key);
|
|
86
|
+
const now = Date.now();
|
|
87
|
+
if (!entry || entry.resetAt <= now)
|
|
88
|
+
return OK;
|
|
89
|
+
if (entry.count < limit)
|
|
90
|
+
return OK;
|
|
91
|
+
return { ok: false, retryAfter: retryAfter(entry.resetAt) };
|
|
92
|
+
},
|
|
93
|
+
hit(key) {
|
|
94
|
+
if (limit <= 0)
|
|
95
|
+
return OK;
|
|
96
|
+
const now = Date.now();
|
|
97
|
+
let entry = counts.get(key);
|
|
98
|
+
if (!entry || entry.resetAt <= now) {
|
|
99
|
+
if (!entry) {
|
|
100
|
+
sweep(now);
|
|
101
|
+
if (counts.size >= maxKeys) {
|
|
102
|
+
// Once per window at most: a full map under attack would otherwise
|
|
103
|
+
// fill the log faster than it fills memory.
|
|
104
|
+
if (now - fullWarnedAt > windowMs) {
|
|
105
|
+
fullWarnedAt = now;
|
|
106
|
+
console.warn(`rate limiter at its ${maxKeys}-key ceiling; new addresses are not being counted until entries expire`);
|
|
107
|
+
}
|
|
108
|
+
return OK;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
entry = { count: 0, resetAt: now + windowMs };
|
|
112
|
+
counts.set(key, entry);
|
|
113
|
+
}
|
|
114
|
+
entry.count++;
|
|
115
|
+
if (entry.count > limit)
|
|
116
|
+
return { ok: false, retryAfter: retryAfter(entry.resetAt) };
|
|
117
|
+
return OK;
|
|
118
|
+
},
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* A concurrency gate: a count of slots in use and a FIFO of waiters.
|
|
123
|
+
*
|
|
124
|
+
* This is the primitive that bounds git. Counting requests per minute does not
|
|
125
|
+
* help there, because the requests are slow rather than frequent: a clone holds a
|
|
126
|
+
* subprocess and a socket for as long as the client cares to read, and a handful
|
|
127
|
+
* of concurrent ls-tree calls on a large repository is already a memory problem
|
|
128
|
+
* given execGit's 256 MB maxBuffer. What bounds it is a limit on how many may run
|
|
129
|
+
* at once.
|
|
130
|
+
*
|
|
131
|
+
* The release function is idempotent, and every caller must invoke it from
|
|
132
|
+
* somewhere that runs on every path out of the request, a client disconnect
|
|
133
|
+
* included. An aborted clone is ordinary traffic, not an error, and a gate that
|
|
134
|
+
* leaks a slot per abort stops answering after `concurrency` of them.
|
|
135
|
+
*/
|
|
136
|
+
function createGate(opts) {
|
|
137
|
+
const { concurrency, queue, timeoutMs } = opts;
|
|
138
|
+
let busy = 0;
|
|
139
|
+
const waiters = [];
|
|
140
|
+
const releaser = () => {
|
|
141
|
+
let released = false;
|
|
142
|
+
return () => {
|
|
143
|
+
if (released)
|
|
144
|
+
return;
|
|
145
|
+
released = true;
|
|
146
|
+
const next = waiters.shift();
|
|
147
|
+
if (next) {
|
|
148
|
+
clearTimeout(next.timer);
|
|
149
|
+
// The slot passes straight to the waiter, so busy stays as it is.
|
|
150
|
+
next.resolve(releaser());
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
busy--;
|
|
154
|
+
};
|
|
155
|
+
};
|
|
156
|
+
return {
|
|
157
|
+
get busy() {
|
|
158
|
+
return busy;
|
|
159
|
+
},
|
|
160
|
+
get queued() {
|
|
161
|
+
return waiters.length;
|
|
162
|
+
},
|
|
163
|
+
enter() {
|
|
164
|
+
if (busy < concurrency) {
|
|
165
|
+
busy++;
|
|
166
|
+
return Promise.resolve(releaser());
|
|
167
|
+
}
|
|
168
|
+
if (waiters.length >= queue)
|
|
169
|
+
return Promise.resolve(null);
|
|
170
|
+
return new Promise((resolve) => {
|
|
171
|
+
// A waiter that has been queued longer than this is dropped, so a
|
|
172
|
+
// request never waits indefinitely for work a client has probably
|
|
173
|
+
// given up on.
|
|
174
|
+
const timer = setTimeout(() => {
|
|
175
|
+
const i = waiters.findIndex((w) => w.timer === timer);
|
|
176
|
+
if (i !== -1)
|
|
177
|
+
waiters.splice(i, 1);
|
|
178
|
+
resolve(null);
|
|
179
|
+
}, timeoutMs);
|
|
180
|
+
// Nothing should keep the process alive for a queued waiter.
|
|
181
|
+
timer.unref?.();
|
|
182
|
+
waiters.push({ resolve, timer });
|
|
183
|
+
});
|
|
184
|
+
},
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
// ---- the gates a vault holds ----
|
|
188
|
+
/**
|
|
189
|
+
* What to put in Retry-After when a gate refuses. A gate refusal is about server
|
|
190
|
+
* capacity right now rather than about a client's quota over a window, so there
|
|
191
|
+
* is no window to compute from; a few seconds is long enough for the queue to
|
|
192
|
+
* drain and short enough that a client retry is not a second outage.
|
|
193
|
+
*/
|
|
194
|
+
exports.BUSY_RETRY_SECONDS = 5;
|
|
195
|
+
// Queue depths and timeouts are constants rather than configuration: nobody
|
|
196
|
+
// tunes these without reading the code, and they can be promoted later if anyone
|
|
197
|
+
// asks. The concurrencies are configurable, because those are the numbers that
|
|
198
|
+
// depend on the machine.
|
|
199
|
+
const QUEUES = {
|
|
200
|
+
clone: { queue: 16, timeoutMs: 10000 },
|
|
201
|
+
push: { queue: 16, timeoutMs: 30000 },
|
|
202
|
+
search: { queue: 8, timeoutMs: 5000 },
|
|
203
|
+
tree: { queue: 16, timeoutMs: 10000 },
|
|
204
|
+
};
|
|
205
|
+
/**
|
|
206
|
+
* Separate gates and not one, so that a flood of anonymous clones cannot stop an
|
|
207
|
+
* authorized push, which is the operation whose failure costs a person their
|
|
208
|
+
* work.
|
|
209
|
+
*/
|
|
210
|
+
function createGates(limits) {
|
|
211
|
+
return {
|
|
212
|
+
clone: createGate({ concurrency: limits.clone, ...QUEUES.clone }),
|
|
213
|
+
push: createGate({ concurrency: limits.push, ...QUEUES.push }),
|
|
214
|
+
search: createGate({ concurrency: limits.search, ...QUEUES.search }),
|
|
215
|
+
tree: createGate({ concurrency: limits.tree, ...QUEUES.tree }),
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
const AUTH_WINDOW_MS = 15 * 60 * 1000;
|
|
219
|
+
const AUTH_MAX_KEYS = 20000;
|
|
220
|
+
/** Bearer and runner tokens carry no username, so they share one bucket in the fine-grained window. */
|
|
221
|
+
const NO_USERNAME = 'token';
|
|
222
|
+
/**
|
|
223
|
+
* A limiter charged only on failure, so a working credential is never throttled
|
|
224
|
+
* however often it is used. That matters for /api/runner/*, which a runner calls
|
|
225
|
+
* continuously with a valid token.
|
|
226
|
+
*
|
|
227
|
+
* Two windows. The fine-grained one is per address and username; the coarse one
|
|
228
|
+
* is per address alone and deliberately the more generous, so that a shared
|
|
229
|
+
* address behind NAT is not cut off by one person mistyping a token, while an
|
|
230
|
+
* attacker spreading attempts over many usernames is still caught.
|
|
231
|
+
*
|
|
232
|
+
* The fine-grained map is the one an attacker can grow, by varying the username.
|
|
233
|
+
* When it is full its hits stop counting, so `allow` falls back to the coarse
|
|
234
|
+
* decision alone rather than failing open altogether; the coarse map's key space
|
|
235
|
+
* is bounded by real peers. That is the reason there are two windows and not one.
|
|
236
|
+
*/
|
|
237
|
+
function createAuthLimiter(authFailures) {
|
|
238
|
+
const fine = createLimiter({ limit: authFailures, windowMs: AUTH_WINDOW_MS, maxKeys: AUTH_MAX_KEYS });
|
|
239
|
+
// Derived rather than configured, so there is one number to think about.
|
|
240
|
+
const coarse = createLimiter({ limit: authFailures * 5, windowMs: AUTH_WINDOW_MS, maxKeys: AUTH_MAX_KEYS });
|
|
241
|
+
const keys = (req, username) => {
|
|
242
|
+
const address = clientKey(req);
|
|
243
|
+
return { address, fine: `${address}${username ?? NO_USERNAME}` };
|
|
244
|
+
};
|
|
245
|
+
return {
|
|
246
|
+
allow(req, username) {
|
|
247
|
+
const k = keys(req, username);
|
|
248
|
+
const a = fine.check(k.fine);
|
|
249
|
+
if (!a.ok)
|
|
250
|
+
return a;
|
|
251
|
+
return coarse.check(k.address);
|
|
252
|
+
},
|
|
253
|
+
fail(req, username) {
|
|
254
|
+
const k = keys(req, username);
|
|
255
|
+
fine.hit(k.fine);
|
|
256
|
+
coarse.hit(k.address);
|
|
257
|
+
},
|
|
258
|
+
};
|
|
259
|
+
}
|