@magland/mochi 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +108 -0
- package/dist/ansi.js +174 -0
- package/dist/api/admin.js +416 -0
- package/dist/api/auth.js +166 -0
- package/dist/api/backup.js +598 -0
- package/dist/api/ci.js +336 -0
- package/dist/api/contents.js +339 -0
- package/dist/api/issues.js +165 -0
- package/dist/api/pulls.js +244 -0
- package/dist/api/releases.js +83 -0
- package/dist/api/repos.js +156 -0
- package/dist/api/write.js +518 -0
- package/dist/api.js +326 -0
- package/dist/assets.js +29 -0
- package/dist/atom.js +32 -0
- package/dist/atomic.js +171 -0
- package/dist/avatar.js +81 -0
- package/dist/browse.js +630 -0
- package/dist/build-info.json +4 -0
- package/dist/ci/actionref.js +86 -0
- package/dist/ci/api.js +829 -0
- package/dist/ci/artifacts.js +201 -0
- package/dist/ci/dispatch.js +30 -0
- package/dist/ci/engine.js +1321 -0
- package/dist/ci/expr.js +526 -0
- package/dist/ci/manual.js +199 -0
- package/dist/ci/present.js +82 -0
- package/dist/ci/protocol.js +6 -0
- package/dist/ci/runners.js +256 -0
- package/dist/ci/runs.js +208 -0
- package/dist/ci/trigger.js +28 -0
- package/dist/ci/views.js +441 -0
- package/dist/ci/wake.js +194 -0
- package/dist/ci/web.js +617 -0
- package/dist/ci/workflow.js +436 -0
- package/dist/cli/admin-cmd.js +324 -0
- package/dist/cli/api-cmd.js +128 -0
- package/dist/cli/backup-cmd.js +1500 -0
- package/dist/cli/exit.js +69 -0
- package/dist/cli/input.js +64 -0
- package/dist/cli/issue-cmd.js +243 -0
- package/dist/cli/output.js +93 -0
- package/dist/cli/parse.js +317 -0
- package/dist/cli/pr-cmd.js +289 -0
- package/dist/cli/release-cmd.js +171 -0
- package/dist/cli/repo-cmd.js +763 -0
- package/dist/cli/repo.js +101 -0
- package/dist/cli/run-cmd.js +438 -0
- package/dist/cli/target.js +54 -0
- package/dist/cli-api.js +84 -0
- package/dist/compare.js +111 -0
- package/dist/config.js +212 -0
- package/dist/credentials.js +235 -0
- package/dist/deploy-cli.js +859 -0
- package/dist/deploy-runner-cli.js +592 -0
- package/dist/diff.js +171 -0
- package/dist/discussion.js +253 -0
- package/dist/egress.js +559 -0
- package/dist/filecache.js +68 -0
- package/dist/find.js +162 -0
- package/dist/forms.js +737 -0
- package/dist/git.js +547 -0
- package/dist/githttp.js +428 -0
- package/dist/html.js +87 -0
- package/dist/icons.js +101 -0
- package/dist/import-cli.js +316 -0
- package/dist/index.js +752 -0
- package/dist/issues.js +308 -0
- package/dist/issueweb.js +447 -0
- package/dist/job-cli.js +197 -0
- package/dist/jobtoken.js +96 -0
- package/dist/languages.js +383 -0
- package/dist/layout.js +100 -0
- package/dist/lfs.js +438 -0
- package/dist/lfsstore.js +425 -0
- package/dist/limit.js +259 -0
- package/dist/logo.js +61 -0
- package/dist/markdown.js +382 -0
- package/dist/migrate.js +334 -0
- package/dist/multipart.js +90 -0
- package/dist/ops.js +869 -0
- package/dist/pagescript.js +465 -0
- package/dist/perms.js +370 -0
- package/dist/pointer.js +55 -0
- package/dist/profile.js +106 -0
- package/dist/pulls.js +320 -0
- package/dist/pullweb.js +461 -0
- package/dist/redirects.js +455 -0
- package/dist/releases.js +435 -0
- package/dist/render.js +233 -0
- package/dist/runner/actions.js +448 -0
- package/dist/runner/client.js +428 -0
- package/dist/runner/context.js +247 -0
- package/dist/runner/docker.js +197 -0
- package/dist/runner/externals.js +175 -0
- package/dist/runner/job.js +290 -0
- package/dist/runner/manual-run.js +272 -0
- package/dist/runner/overrides.js +554 -0
- package/dist/runner/steps.js +571 -0
- package/dist/runner/wake.js +84 -0
- package/dist/runner-cli.js +405 -0
- package/dist/scan.js +231 -0
- package/dist/server.js +424 -0
- package/dist/session.js +267 -0
- package/dist/site.js +259 -0
- package/dist/siteshost.js +94 -0
- package/dist/source.js +90 -0
- package/dist/style.js +1295 -0
- package/dist/themes.js +369 -0
- package/dist/vault.js +442 -0
- package/dist/version.js +88 -0
- package/dist/views.js +1007 -0
- package/dist/web.js +182 -0
- package/dist/webops.js +1402 -0
- package/package.json +71 -0
package/dist/egress.js
ADDED
|
@@ -0,0 +1,559 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.EGRESS_FILE = void 0;
|
|
37
|
+
exports.egressMessage = egressMessage;
|
|
38
|
+
exports.egressFilePath = egressFilePath;
|
|
39
|
+
exports.createEgress = createEgress;
|
|
40
|
+
const fs = __importStar(require("fs"));
|
|
41
|
+
const path = __importStar(require("path"));
|
|
42
|
+
const atomic_1 = require("./atomic");
|
|
43
|
+
const config_1 = require("./config");
|
|
44
|
+
const redirects_1 = require("./redirects");
|
|
45
|
+
const scan_1 = require("./scan");
|
|
46
|
+
const siteshost_1 = require("./siteshost");
|
|
47
|
+
// Outgoing bytes, counted and capped.
|
|
48
|
+
//
|
|
49
|
+
// A host that bills for egress and does not cap it -- Fly is the one this was
|
|
50
|
+
// written for -- turns a popular repository, a crawler, or a badly written CI
|
|
51
|
+
// loop into a bill nobody chose. Nothing else in mochi bounds it: the rate
|
|
52
|
+
// limiter in src/limit.ts counts requests, and a request for a 2 GB release
|
|
53
|
+
// asset costs the same one request as a request for the front page.
|
|
54
|
+
//
|
|
55
|
+
// So this counts bytes instead, keeps a running total per day, and refuses to
|
|
56
|
+
// send more once the day's budget is spent. Two deliberate asymmetries with the
|
|
57
|
+
// rate limiter, which keeps nothing on disk:
|
|
58
|
+
//
|
|
59
|
+
// - The counts are persisted, because a budget that a restart forgives is not
|
|
60
|
+
// a budget. A crash loop would otherwise send 20 GB per restart.
|
|
61
|
+
// - The cap is read per request rather than once at startup, unlike every
|
|
62
|
+
// other setting in the limits block. The moment an operator wants to change
|
|
63
|
+
// it is the moment the vault has stopped answering, and telling them to
|
|
64
|
+
// restart it then is telling them to reach the volume by hand.
|
|
65
|
+
//
|
|
66
|
+
// Two limitations, both worth stating plainly rather than engineering around.
|
|
67
|
+
//
|
|
68
|
+
// LFS objects served from a configured S3 bucket leave through presigned URLs
|
|
69
|
+
// that never touch this process, so those bytes are invisible here and the admin
|
|
70
|
+
// page says so. The cap is not defeated by that, only measured around it: the
|
|
71
|
+
// batch endpoint that mints those URLs is an ordinary route, so it is refused
|
|
72
|
+
// with everything else once the budget is spent, and what still works afterwards
|
|
73
|
+
// is the URLs signed in the previous hour. The bytes are uncounted; the
|
|
74
|
+
// permission is not.
|
|
75
|
+
//
|
|
76
|
+
// And the numbers are bytes written to sockets by this process, which is close to
|
|
77
|
+
// what a host meters but not identical: it excludes the TCP/TLS framing a proxy
|
|
78
|
+
// adds, and includes bytes queued for a client that hung up before reading them.
|
|
79
|
+
exports.EGRESS_FILE = 'egress.json';
|
|
80
|
+
/** Days of history kept in the file, today included. */
|
|
81
|
+
const HISTORY_DAYS = 30;
|
|
82
|
+
/**
|
|
83
|
+
* The most repositories one day may be broken down by. Attribution only ever
|
|
84
|
+
* names repositories that exist, so the key space is bounded by the vault's own
|
|
85
|
+
* contents rather than by whoever is making requests; the ceiling remains as a
|
|
86
|
+
* backstop for a vault that genuinely holds more repositories than this.
|
|
87
|
+
* Beyond it, bytes are still counted, under one overflow row.
|
|
88
|
+
*/
|
|
89
|
+
const MAX_KEYS = 2000;
|
|
90
|
+
/** The row bytes land in once the ceiling is reached. */
|
|
91
|
+
const OVERFLOW_KEY = '(other)';
|
|
92
|
+
/** The row for everything that belongs to no repository: the front page, the admin pages, assets, the API. */
|
|
93
|
+
const VAULT_KEY = '(vault)';
|
|
94
|
+
/**
|
|
95
|
+
* The row for a repository-shaped path that resolves to no repository: a 404,
|
|
96
|
+
* a crawler probing names, a link to something since deleted. One row rather
|
|
97
|
+
* than one per path, so an anonymous visitor cannot grow the admin page by
|
|
98
|
+
* asking for made-up names, and so the rows that do name a repository can be
|
|
99
|
+
* trusted to be one.
|
|
100
|
+
*/
|
|
101
|
+
const UNMATCHED_KEY = '(unmatched)';
|
|
102
|
+
/** How a site's row is spelled, so the parser and the page agree on one thing. */
|
|
103
|
+
const SITE_SUFFIX = ':site';
|
|
104
|
+
/**
|
|
105
|
+
* How often the file may be written while counting is happening. Every request
|
|
106
|
+
* touches the in-memory total; a write per request would put a synchronous fsync
|
|
107
|
+
* on the hot path for a number nobody reads more than once a minute. Half a
|
|
108
|
+
* minute of counting is what a hard kill costs.
|
|
109
|
+
*/
|
|
110
|
+
const FLUSH_MS = 30000;
|
|
111
|
+
const GB = 1024 * 1024 * 1024;
|
|
112
|
+
/**
|
|
113
|
+
* How much the exempt paths -- /admin, /login, the stylesheet they need -- may
|
|
114
|
+
* send after the budget is spent. Without an allowance they are unreachable, and
|
|
115
|
+
* an operator locked out of the page that would raise the cap is worse off than
|
|
116
|
+
* one whose vault sent a few megabytes too many. Without a bound on the
|
|
117
|
+
* allowance, a crawler on /assets/style.css would send those bytes forever.
|
|
118
|
+
*/
|
|
119
|
+
const EXEMPT_GRACE = 64 * 1024 * 1024;
|
|
120
|
+
/** How long until the window resets, as a person would say it. */
|
|
121
|
+
function untilReset(seconds) {
|
|
122
|
+
const h = Math.floor(seconds / 3600);
|
|
123
|
+
const m = Math.round((seconds % 3600) / 60);
|
|
124
|
+
if (h > 0)
|
|
125
|
+
return m > 0 ? `${h}h ${m}m` : `${h}h`;
|
|
126
|
+
return `${Math.max(1, m)}m`;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* What a refused visitor is told. One sentence, and it names the setting and
|
|
130
|
+
* when the vault comes back: someone reading this wants to know whether to wait
|
|
131
|
+
* or to go and raise a number.
|
|
132
|
+
*/
|
|
133
|
+
function egressMessage(decision) {
|
|
134
|
+
const cap = decision.capBytes / GB;
|
|
135
|
+
const gb = cap >= 10 ? cap.toFixed(0) : cap.toFixed(1);
|
|
136
|
+
return (`This vault has sent its daily limit of ${gb} GB and is not sending more. ` +
|
|
137
|
+
`The limit resets at 00:00 UTC, in ${untilReset(decision.retryAfter)}.`);
|
|
138
|
+
}
|
|
139
|
+
function egressFilePath(root) {
|
|
140
|
+
return path.join(root, exports.EGRESS_FILE);
|
|
141
|
+
}
|
|
142
|
+
function utcDay(ms) {
|
|
143
|
+
return new Date(ms).toISOString().slice(0, 10);
|
|
144
|
+
}
|
|
145
|
+
/** The next UTC midnight after `ms`, which is when the window rolls over. */
|
|
146
|
+
function nextRollover(ms) {
|
|
147
|
+
const d = new Date(ms);
|
|
148
|
+
return Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate() + 1);
|
|
149
|
+
}
|
|
150
|
+
function sum(keys) {
|
|
151
|
+
let total = 0;
|
|
152
|
+
for (const v of Object.values(keys))
|
|
153
|
+
if (Number.isFinite(v) && v > 0)
|
|
154
|
+
total += v;
|
|
155
|
+
return total;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Read the file, keeping only what is well formed. A file that has been
|
|
159
|
+
* hand-edited into nonsense loses its history rather than the day's cap: the
|
|
160
|
+
* counts are an accounting record, and the thing that must not happen is a
|
|
161
|
+
* parse error taking the vault down.
|
|
162
|
+
*/
|
|
163
|
+
function readDays(file) {
|
|
164
|
+
let parsed;
|
|
165
|
+
try {
|
|
166
|
+
parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
return [];
|
|
170
|
+
}
|
|
171
|
+
const days = parsed?.days;
|
|
172
|
+
if (!Array.isArray(days))
|
|
173
|
+
return [];
|
|
174
|
+
const out = [];
|
|
175
|
+
for (const entry of days) {
|
|
176
|
+
if (typeof entry !== 'object' || entry === null)
|
|
177
|
+
continue;
|
|
178
|
+
const e = entry;
|
|
179
|
+
if (typeof e.day !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(e.day))
|
|
180
|
+
continue;
|
|
181
|
+
const keys = {};
|
|
182
|
+
if (typeof e.keys === 'object' && e.keys !== null) {
|
|
183
|
+
for (const [k, v] of Object.entries(e.keys)) {
|
|
184
|
+
if (typeof v === 'number' && Number.isFinite(v) && v > 0)
|
|
185
|
+
keys[k] = Math.floor(v);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
out.push({ day: e.day, total: sum(keys), keys });
|
|
189
|
+
}
|
|
190
|
+
out.sort((a, b) => (a.day < b.day ? 1 : a.day > b.day ? -1 : 0));
|
|
191
|
+
return out.slice(0, HISTORY_DAYS);
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Paths that keep working after the budget is spent, within EXEMPT_GRACE.
|
|
195
|
+
*
|
|
196
|
+
* /admin and /login because that is the path to raising the cap; the API's
|
|
197
|
+
* config and egress routes because they are the same two operations for a
|
|
198
|
+
* program, and reading the number is what an operator does before changing it;
|
|
199
|
+
* the stylesheet and the icon because the admin page is unreadable without them.
|
|
200
|
+
*
|
|
201
|
+
* None of it applies on a sites hostname, where these paths belong to the site
|
|
202
|
+
* being served and are ordinary traffic, exactly as isRateExempt has it.
|
|
203
|
+
*/
|
|
204
|
+
function isEgressExempt(root, req) {
|
|
205
|
+
if ((0, siteshost_1.isUnderSitesHost)((0, config_1.loadConfig)(root).sites.host, req.hostname))
|
|
206
|
+
return false;
|
|
207
|
+
const p = req.path;
|
|
208
|
+
if (p === '/login' || p === '/logout' || p === '/api/config' || p === '/api/egress')
|
|
209
|
+
return true;
|
|
210
|
+
if (p === '/admin' || p.startsWith('/admin/'))
|
|
211
|
+
return true;
|
|
212
|
+
return p.startsWith('/assets/') || p === '/favicon.svg' || p === '/favicon.ico';
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* How long a resolved name is believed before the filesystem is asked again.
|
|
216
|
+
* The cache is what keeps resolution off the hot path: attribution runs once
|
|
217
|
+
* per response, and paying a handful of stats per distinct name per half
|
|
218
|
+
* minute is nothing, where a stat per response for every 404 a crawler sends
|
|
219
|
+
* would not be. Thirty seconds matches the flush interval, and bounds how long
|
|
220
|
+
* a rename or a deletion is attributed under the old answer.
|
|
221
|
+
*/
|
|
222
|
+
const RESOLVE_TTL_MS = 30000;
|
|
223
|
+
const resolveCache = new Map();
|
|
224
|
+
/**
|
|
225
|
+
* The `collection/repo` a name actually reaches, following rename redirects,
|
|
226
|
+
* or null when it reaches no repository at all. This is what keys the rows:
|
|
227
|
+
* bytes are attributed to the repository a request resolved to, never to the
|
|
228
|
+
* raw path string, so a made-up path cannot mint a row and a renamed
|
|
229
|
+
* repository keeps one row rather than one per name it has had.
|
|
230
|
+
*/
|
|
231
|
+
function resolveRepoKey(root, collection, repo) {
|
|
232
|
+
if (!(0, scan_1.isValidName)(collection) || !(0, scan_1.isValidName)(repo))
|
|
233
|
+
return null;
|
|
234
|
+
const cacheKey = `${root}\0${collection}/${repo}`;
|
|
235
|
+
const hit = resolveCache.get(cacheKey);
|
|
236
|
+
const now = Date.now();
|
|
237
|
+
if (hit && now - hit.at < RESOLVE_TTL_MS)
|
|
238
|
+
return hit.to;
|
|
239
|
+
let to = null;
|
|
240
|
+
if ((0, scan_1.findRepo)(root, collection, repo)) {
|
|
241
|
+
to = `${collection}/${repo}`;
|
|
242
|
+
}
|
|
243
|
+
else {
|
|
244
|
+
const moved = (0, redirects_1.resolveRepoRedirect)(root, collection, repo);
|
|
245
|
+
if (moved)
|
|
246
|
+
to = `${moved.collection}/${moved.repo}`;
|
|
247
|
+
}
|
|
248
|
+
// Dropped wholesale rather than evicted by age, so the map cannot grow
|
|
249
|
+
// without bound under a crawler inventing names.
|
|
250
|
+
if (resolveCache.size > 4096)
|
|
251
|
+
resolveCache.clear();
|
|
252
|
+
resolveCache.set(cacheKey, { at: now, to });
|
|
253
|
+
return to;
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Re-bucket one day's keys through resolution: a key naming a former name of a
|
|
257
|
+
* repository joins that repository's row, and a key naming nothing joins the
|
|
258
|
+
* unmatched row. No bytes are gained or lost, only moved, so the day's total
|
|
259
|
+
* is untouched. Returns null when every key already stands.
|
|
260
|
+
*/
|
|
261
|
+
function normalizeKeys(root, keys) {
|
|
262
|
+
let changed = false;
|
|
263
|
+
const out = {};
|
|
264
|
+
for (const [key, bytes] of Object.entries(keys)) {
|
|
265
|
+
let to = key;
|
|
266
|
+
if (!key.startsWith('(')) {
|
|
267
|
+
const site = key.endsWith(SITE_SUFFIX);
|
|
268
|
+
const base = site ? key.slice(0, -SITE_SUFFIX.length) : key;
|
|
269
|
+
const slash = base.indexOf('/');
|
|
270
|
+
const resolved = slash > 0 ? resolveRepoKey(root, base.slice(0, slash), base.slice(slash + 1)) : null;
|
|
271
|
+
to = resolved === null ? UNMATCHED_KEY : `${resolved}${site ? SITE_SUFFIX : ''}`;
|
|
272
|
+
if (to !== key)
|
|
273
|
+
changed = true;
|
|
274
|
+
}
|
|
275
|
+
out[to] = (out[to] ?? 0) + bytes;
|
|
276
|
+
}
|
|
277
|
+
return changed ? out : null;
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Which row a request's bytes belong to.
|
|
281
|
+
*
|
|
282
|
+
* A site on its own hostname is recognised from the hostname; on the forge host
|
|
283
|
+
* every repository surface lives under /:collection/:repo, so one regex covers
|
|
284
|
+
* browsing, the git wire protocol, LFS, releases, issues, and CI alike, and
|
|
285
|
+
* /:collection/:repo/site is the site. What the path names is then resolved to
|
|
286
|
+
* a repository that exists, through rename redirects, and a name that resolves
|
|
287
|
+
* to none is one unmatched row rather than a row of its own.
|
|
288
|
+
*/
|
|
289
|
+
function keyFor(root, req) {
|
|
290
|
+
const site = (0, siteshost_1.parseSiteHost)((0, config_1.loadConfig)(root).sites.host, req.hostname);
|
|
291
|
+
if (site) {
|
|
292
|
+
const resolved = resolveRepoKey(root, site.collection, site.repo);
|
|
293
|
+
return resolved === null ? UNMATCHED_KEY : `${resolved}${SITE_SUFFIX}`;
|
|
294
|
+
}
|
|
295
|
+
const m = /^\/([^/]+)\/([^/]+)(?:\/(.*))?$/.exec(req.path);
|
|
296
|
+
if (!m)
|
|
297
|
+
return VAULT_KEY;
|
|
298
|
+
let collection;
|
|
299
|
+
let repo;
|
|
300
|
+
try {
|
|
301
|
+
collection = decodeURIComponent(m[1]);
|
|
302
|
+
repo = decodeURIComponent(m[2]);
|
|
303
|
+
}
|
|
304
|
+
catch {
|
|
305
|
+
return VAULT_KEY;
|
|
306
|
+
}
|
|
307
|
+
if (!(0, scan_1.isValidName)(collection) || !(0, scan_1.isValidName)(repo))
|
|
308
|
+
return VAULT_KEY;
|
|
309
|
+
const resolved = resolveRepoKey(root, collection, repo);
|
|
310
|
+
if (resolved === null)
|
|
311
|
+
return UNMATCHED_KEY;
|
|
312
|
+
const rest = m[3] ?? '';
|
|
313
|
+
const isSite = rest === 'site' || rest.startsWith('site/');
|
|
314
|
+
return `${resolved}${isSite ? SITE_SUFFIX : ''}`;
|
|
315
|
+
}
|
|
316
|
+
function splitKey(key) {
|
|
317
|
+
return key.endsWith(SITE_SUFFIX)
|
|
318
|
+
? { repo: key.slice(0, -SITE_SUFFIX.length), site: true, bytes: 0 }
|
|
319
|
+
: { repo: key, site: false, bytes: 0 };
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* The counter one server holds.
|
|
323
|
+
*
|
|
324
|
+
* `capGb` is a thunk rather than a number, so the caller decides where the
|
|
325
|
+
* setting comes from and a change to it is in force on the next request.
|
|
326
|
+
*
|
|
327
|
+
* The day's total is kept in two halves: `base`, as last seen on disk, and
|
|
328
|
+
* `pending`, counted here since then. Every flush adds the pending half to
|
|
329
|
+
* whatever the file says and adopts the result as the new base, so two servers
|
|
330
|
+
* pointed at one vault add up instead of overwriting each other, and so does a
|
|
331
|
+
* server sharing a vault with a hand-edited file. Neither sees the other's bytes
|
|
332
|
+
* until a flush, which means each may send up to FLUSH_MS worth past the cap.
|
|
333
|
+
*/
|
|
334
|
+
function createEgress(root, capGb) {
|
|
335
|
+
const file = egressFilePath(root);
|
|
336
|
+
const lock = `${file}.lock`;
|
|
337
|
+
let day = utcDay(Date.now());
|
|
338
|
+
let rollAt = nextRollover(Date.now());
|
|
339
|
+
let base = new Map();
|
|
340
|
+
let baseTotal = 0;
|
|
341
|
+
let pending = new Map();
|
|
342
|
+
let pendingTotal = 0;
|
|
343
|
+
let past = [];
|
|
344
|
+
let timer = null;
|
|
345
|
+
let warnedAt = 0;
|
|
346
|
+
// Load once at startup. A vault whose file says 19 GB have gone out today
|
|
347
|
+
// comes back still knowing that, which is the whole reason the file exists.
|
|
348
|
+
//
|
|
349
|
+
// The keys are re-bucketed through resolution on the way in: a file written
|
|
350
|
+
// before attribution resolved names, or before a rename, holds rows for
|
|
351
|
+
// former names and for paths that never named a repository, and this is
|
|
352
|
+
// where those join the rows they belong to. Totals are untouched, and a file
|
|
353
|
+
// whose keys all stand is not rewritten.
|
|
354
|
+
{
|
|
355
|
+
const days = readDays(file);
|
|
356
|
+
let migrated = false;
|
|
357
|
+
for (const d of days) {
|
|
358
|
+
const fixed = normalizeKeys(root, d.keys);
|
|
359
|
+
if (fixed) {
|
|
360
|
+
d.keys = fixed;
|
|
361
|
+
migrated = true;
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
if (migrated) {
|
|
365
|
+
try {
|
|
366
|
+
(0, atomic_1.withFileLock)(lock, () => {
|
|
367
|
+
(0, atomic_1.writeFileAtomic)(file, JSON.stringify({ version: 1, days }, null, 2) + '\n');
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
catch (e) {
|
|
371
|
+
// The flush path merges into whatever is on disk, so nothing is lost by
|
|
372
|
+
// failing here; the migration simply happens again next start.
|
|
373
|
+
console.warn(`could not migrate ${exports.EGRESS_FILE}: ${e instanceof Error ? e.message : String(e)}`);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
const today = days.find((d) => d.day === day);
|
|
377
|
+
if (today) {
|
|
378
|
+
base = new Map(Object.entries(today.keys));
|
|
379
|
+
baseTotal = today.total;
|
|
380
|
+
}
|
|
381
|
+
past = days.filter((d) => d.day !== day);
|
|
382
|
+
}
|
|
383
|
+
const capBytes = () => {
|
|
384
|
+
const gb = capGb();
|
|
385
|
+
return Number.isFinite(gb) && gb > 0 ? Math.floor(gb * GB) : 0;
|
|
386
|
+
};
|
|
387
|
+
const used = () => baseTotal + pendingTotal;
|
|
388
|
+
function writeOut() {
|
|
389
|
+
if (pending.size === 0)
|
|
390
|
+
return;
|
|
391
|
+
// Taken out of the way first, so that bytes counted while the file is being
|
|
392
|
+
// written are not lost to the reset below.
|
|
393
|
+
const mine = pending;
|
|
394
|
+
const myDay = day;
|
|
395
|
+
pending = new Map();
|
|
396
|
+
pendingTotal = 0;
|
|
397
|
+
try {
|
|
398
|
+
(0, atomic_1.withFileLock)(lock, () => {
|
|
399
|
+
const days = readDays(file);
|
|
400
|
+
let rec = days.find((d) => d.day === myDay);
|
|
401
|
+
if (!rec) {
|
|
402
|
+
rec = { day: myDay, total: 0, keys: {} };
|
|
403
|
+
days.push(rec);
|
|
404
|
+
}
|
|
405
|
+
for (const [k, v] of mine)
|
|
406
|
+
rec.keys[k] = (rec.keys[k] ?? 0) + v;
|
|
407
|
+
// A rename mid-day leaves the file's rows under the old name; they are
|
|
408
|
+
// folded into the new one here, so every row on disk names something a
|
|
409
|
+
// link can reach. Cached resolution makes this cost nothing when
|
|
410
|
+
// nothing has moved.
|
|
411
|
+
rec.keys = normalizeKeys(root, rec.keys) ?? rec.keys;
|
|
412
|
+
rec.total = sum(rec.keys);
|
|
413
|
+
days.sort((a, b) => (a.day < b.day ? 1 : a.day > b.day ? -1 : 0));
|
|
414
|
+
const kept = days.slice(0, HISTORY_DAYS);
|
|
415
|
+
(0, atomic_1.writeFileAtomic)(file, JSON.stringify({ version: 1, days: kept }, null, 2) + '\n');
|
|
416
|
+
if (myDay === day) {
|
|
417
|
+
base = new Map(Object.entries(rec.keys));
|
|
418
|
+
baseTotal = rec.total;
|
|
419
|
+
}
|
|
420
|
+
past = kept.filter((d) => d.day !== day);
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
catch (e) {
|
|
424
|
+
// Put the counts back rather than dropping them: a full disk or a lock
|
|
425
|
+
// held by a dead process is a reason to try again in half a minute, not a
|
|
426
|
+
// reason to forgive the traffic. Reported at most once per flush interval,
|
|
427
|
+
// so a persistent failure does not fill the log.
|
|
428
|
+
for (const [k, v] of mine)
|
|
429
|
+
pending.set(k, (pending.get(k) ?? 0) + v);
|
|
430
|
+
for (const v of mine.values())
|
|
431
|
+
pendingTotal += v;
|
|
432
|
+
const now = Date.now();
|
|
433
|
+
if (now - warnedAt > FLUSH_MS) {
|
|
434
|
+
warnedAt = now;
|
|
435
|
+
console.warn(`could not write ${exports.EGRESS_FILE}: ${e instanceof Error ? e.message : String(e)}`);
|
|
436
|
+
}
|
|
437
|
+
schedule();
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
function schedule() {
|
|
441
|
+
if (timer)
|
|
442
|
+
return;
|
|
443
|
+
timer = setTimeout(() => {
|
|
444
|
+
timer = null;
|
|
445
|
+
writeOut();
|
|
446
|
+
}, FLUSH_MS);
|
|
447
|
+
// A pending flush is not a reason to keep the process alive; the exit
|
|
448
|
+
// handler below is what makes sure the counts are written on the way out.
|
|
449
|
+
timer.unref?.();
|
|
450
|
+
}
|
|
451
|
+
/** Roll the window if the day has turned. Called from the two paths that read the total. */
|
|
452
|
+
function rollIfNeeded() {
|
|
453
|
+
const now = Date.now();
|
|
454
|
+
if (now < rollAt)
|
|
455
|
+
return;
|
|
456
|
+
// The pending half belongs to the day it was counted in, so it is written
|
|
457
|
+
// under the old date before anything is reset.
|
|
458
|
+
writeOut();
|
|
459
|
+
if (baseTotal > 0) {
|
|
460
|
+
past = [{ day, total: baseTotal, keys: Object.fromEntries(base) }, ...past].slice(0, HISTORY_DAYS);
|
|
461
|
+
}
|
|
462
|
+
day = utcDay(now);
|
|
463
|
+
rollAt = nextRollover(now);
|
|
464
|
+
base = new Map();
|
|
465
|
+
baseTotal = 0;
|
|
466
|
+
}
|
|
467
|
+
function add(key, bytes) {
|
|
468
|
+
rollIfNeeded();
|
|
469
|
+
let k = key;
|
|
470
|
+
if (!base.has(k) && !pending.has(k) && base.size + pending.size >= MAX_KEYS)
|
|
471
|
+
k = OVERFLOW_KEY;
|
|
472
|
+
pending.set(k, (pending.get(k) ?? 0) + bytes);
|
|
473
|
+
pendingTotal += bytes;
|
|
474
|
+
schedule();
|
|
475
|
+
}
|
|
476
|
+
// Written on the way out, so an ordinary restart or a Fly deploy loses
|
|
477
|
+
// nothing. `exit` covers a clean end; the two signals do not reach it on
|
|
478
|
+
// their own, because a handler for either replaces the default of
|
|
479
|
+
// terminating, so each ends the process itself once the counts are down.
|
|
480
|
+
process.on('exit', () => writeOut());
|
|
481
|
+
for (const sig of ['SIGTERM', 'SIGINT']) {
|
|
482
|
+
process.on(sig, () => {
|
|
483
|
+
writeOut();
|
|
484
|
+
process.exit(0);
|
|
485
|
+
});
|
|
486
|
+
}
|
|
487
|
+
return {
|
|
488
|
+
watch(req, res) {
|
|
489
|
+
// The socket is captured now rather than read at the end: a response that
|
|
490
|
+
// ends because the client hung up has no res.socket left, and the object
|
|
491
|
+
// still remembers what was written to it. bytesWritten is cumulative over
|
|
492
|
+
// a keep-alive connection, so the delta over one response is this
|
|
493
|
+
// response's share -- HTTP/1.1 answers one request at a time per
|
|
494
|
+
// connection, which is what makes the delta attributable at all.
|
|
495
|
+
const sock = res.socket;
|
|
496
|
+
if (!sock)
|
|
497
|
+
return;
|
|
498
|
+
const start = sock.bytesWritten;
|
|
499
|
+
let done = false;
|
|
500
|
+
const finish = () => {
|
|
501
|
+
if (done)
|
|
502
|
+
return;
|
|
503
|
+
done = true;
|
|
504
|
+
const bytes = sock.bytesWritten - start;
|
|
505
|
+
// Headers alone are never zero, so a zero here means the response never
|
|
506
|
+
// reached the socket. Negative cannot happen, and is guarded because a
|
|
507
|
+
// negative would silently buy back budget.
|
|
508
|
+
if (bytes > 0)
|
|
509
|
+
add(keyFor(root, req), bytes);
|
|
510
|
+
};
|
|
511
|
+
// Both, because only one of them fires: 'finish' when the response was
|
|
512
|
+
// written, 'close' when it was abandoned. An aborted clone is ordinary
|
|
513
|
+
// traffic and its bytes were still sent.
|
|
514
|
+
res.on('finish', finish);
|
|
515
|
+
res.on('close', finish);
|
|
516
|
+
},
|
|
517
|
+
allow(req) {
|
|
518
|
+
const cap = capBytes();
|
|
519
|
+
if (cap <= 0)
|
|
520
|
+
return null;
|
|
521
|
+
rollIfNeeded();
|
|
522
|
+
const total = used();
|
|
523
|
+
if (total < cap)
|
|
524
|
+
return null;
|
|
525
|
+
if (total < cap + EXEMPT_GRACE && isEgressExempt(root, req))
|
|
526
|
+
return null;
|
|
527
|
+
return { retryAfter: Math.max(1, Math.ceil((rollAt - Date.now()) / 1000)), capBytes: cap };
|
|
528
|
+
},
|
|
529
|
+
snapshot() {
|
|
530
|
+
rollIfNeeded();
|
|
531
|
+
let mergedKeys = {};
|
|
532
|
+
for (const [k, v] of base)
|
|
533
|
+
mergedKeys[k] = (mergedKeys[k] ?? 0) + v;
|
|
534
|
+
for (const [k, v] of pending)
|
|
535
|
+
mergedKeys[k] = (mergedKeys[k] ?? 0) + v;
|
|
536
|
+
// Resolved again for display, so a rename made minutes ago shows one row
|
|
537
|
+
// under the new name rather than two under both.
|
|
538
|
+
mergedKeys = normalizeKeys(root, mergedKeys) ?? mergedKeys;
|
|
539
|
+
const merged = new Map(Object.entries(mergedKeys));
|
|
540
|
+
const rows = [...merged.entries()]
|
|
541
|
+
.map(([k, bytes]) => ({ ...splitKey(k), bytes }))
|
|
542
|
+
.filter((r) => r.bytes > 0)
|
|
543
|
+
.sort((a, b) => b.bytes - a.bytes);
|
|
544
|
+
const cap = capBytes();
|
|
545
|
+
const total = used();
|
|
546
|
+
return {
|
|
547
|
+
day,
|
|
548
|
+
rows,
|
|
549
|
+
total,
|
|
550
|
+
capBytes: cap,
|
|
551
|
+
capGb: capGb(),
|
|
552
|
+
overBudget: cap > 0 && total >= cap,
|
|
553
|
+
resetsIn: Math.max(0, Math.ceil((rollAt - Date.now()) / 1000)),
|
|
554
|
+
history: past.map((d) => ({ day: d.day, total: d.total })),
|
|
555
|
+
};
|
|
556
|
+
},
|
|
557
|
+
flush: writeOut,
|
|
558
|
+
};
|
|
559
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.fileCache = fileCache;
|
|
37
|
+
const fs = __importStar(require("fs"));
|
|
38
|
+
/**
|
|
39
|
+
* `read` parses the file and is only called when the stat is fresh; what it
|
|
40
|
+
* returns is cached, so it should catch its own parse errors and return the
|
|
41
|
+
* value the caller should see, or the error state it wants remembered.
|
|
42
|
+
* `missing` answers for a file that does not exist, and is not cached: a file
|
|
43
|
+
* that appears later is noticed on the next get.
|
|
44
|
+
*/
|
|
45
|
+
function fileCache(opts) {
|
|
46
|
+
const slots = new Map();
|
|
47
|
+
return {
|
|
48
|
+
get(file) {
|
|
49
|
+
let st;
|
|
50
|
+
try {
|
|
51
|
+
st = fs.statSync(file);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
slots.delete(file);
|
|
55
|
+
return opts.missing(file);
|
|
56
|
+
}
|
|
57
|
+
const hit = slots.get(file);
|
|
58
|
+
if (hit && hit.mtimeMs === st.mtimeMs && hit.size === st.size)
|
|
59
|
+
return hit.value;
|
|
60
|
+
const value = opts.read(file);
|
|
61
|
+
slots.set(file, { mtimeMs: st.mtimeMs, size: st.size, value });
|
|
62
|
+
return value;
|
|
63
|
+
},
|
|
64
|
+
invalidate(file) {
|
|
65
|
+
slots.delete(file);
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
}
|