@mostra/runtime 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +784 -432
- package/dist/bin.js.map +1 -1
- package/dist/index.js +1 -1
- package/dist/{instance-CBUlOmzG.js → instance-D5gh-sMM.js} +17 -4
- package/dist/instance-D5gh-sMM.js.map +1 -0
- package/dist/reference/diagnostics.json +2 -2
- package/dist/reference/errors/MOSTRA_REMOTE_QUEUE_FULL.md +2 -2
- package/dist/reference/self-hosting.md +1 -1
- package/dist/skill/SKILL.md +2 -2
- package/dist/skill/references/mcp.md +2 -2
- package/package.json +6 -6
- package/src/actions/index.ts +5 -2
- package/src/gateway/actions.ts +6 -1
- package/src/gateway/capabilities.ts +5 -1
- package/src/gateway/index.ts +3 -0
- package/src/ledger/migrations.ts +9 -0
- package/src/server/api.ts +1 -1
- package/src/server/auth.ts +83 -16
- package/src/server/authorize.ts +27 -4
- package/src/server/config.ts +53 -0
- package/src/server/directory.ts +51 -0
- package/src/server/hosted-pages.ts +19 -6
- package/src/server/hosted.ts +54 -27
- package/src/server/index.ts +62 -15
- package/src/server/metering.ts +117 -12
- package/src/server/providers.ts +61 -9
- package/src/server/relay.ts +258 -107
- package/dist/instance-CBUlOmzG.js.map +0 -1
|
@@ -152,19 +152,32 @@ export function hostedPages(
|
|
|
152
152
|
});
|
|
153
153
|
};
|
|
154
154
|
|
|
155
|
-
/**
|
|
156
|
-
|
|
155
|
+
/**
|
|
156
|
+
* A link past the operator's longest lifetime or idle limit: revoked now, 410 (D165 item 9).
|
|
157
|
+
* "Idle" is as `LinkPolicy.idleExpiry` says, without a view: the link's last use before this
|
|
158
|
+
* visit is the later of its last view (a frame ticket) and the activity of its other sessions
|
|
159
|
+
* (`last_seen_at`, which a page's every request keeps, so a page open and in use for longer
|
|
160
|
+
* than the idle time keeps its link). The session asking for this ticket is left out, since its
|
|
161
|
+
* own exchange, or any request of a returning browser, has just written it: counted, an idle
|
|
162
|
+
* link would open for every new visit (MT-03, and its Opus review). A session's activity is
|
|
163
|
+
* written at most every `touchMs` (an hour), so a link lapses at most that much early.
|
|
164
|
+
*/
|
|
165
|
+
const expired = (tokenId: string, asking: string, policy: LinkPolicy): Refusal | undefined => {
|
|
157
166
|
if (policy.maxTtl === undefined && policy.idleExpiry === undefined) return undefined;
|
|
158
167
|
const used = ledger.read((db) =>
|
|
159
|
-
db.get<{ created_at: string;
|
|
160
|
-
`SELECT t.created_at,
|
|
168
|
+
db.get<{ created_at: string; viewed: string | null; active: string | null }>(
|
|
169
|
+
`SELECT t.created_at,
|
|
170
|
+
(SELECT MAX(v.opened_at) FROM view v WHERE v.token_id = t.id) AS viewed,
|
|
171
|
+
(SELECT MAX(s.last_seen_at) FROM session s WHERE s.token_id = t.id AND s.id != ?) AS active
|
|
161
172
|
FROM token t WHERE t.id = ?`,
|
|
173
|
+
asking,
|
|
162
174
|
tokenId,
|
|
163
175
|
),
|
|
164
176
|
);
|
|
165
177
|
if (!used) return undefined;
|
|
166
178
|
const minted = Date.parse(used.created_at);
|
|
167
|
-
const
|
|
179
|
+
const at = (value: string | null): number => (value === null ? 0 : Date.parse(value));
|
|
180
|
+
const lastUse = Math.max(minted, at(used.viewed), at(used.active));
|
|
168
181
|
const tooOld = policy.maxTtl !== undefined && now() - minted > durationMs(policy.maxTtl);
|
|
169
182
|
const idle = policy.idleExpiry !== undefined && now() - lastUse > durationMs(policy.idleExpiry);
|
|
170
183
|
if (!tooOld && !idle) return undefined;
|
|
@@ -411,7 +424,7 @@ export function hostedPages(
|
|
|
411
424
|
async admitView(input) {
|
|
412
425
|
if (quarantineOf(input.artifactId, input.versionId) !== undefined) return underReview();
|
|
413
426
|
const policy = await policyFor(input.artifactId, input.tokenId);
|
|
414
|
-
const gone = expired(input.tokenId, policy);
|
|
427
|
+
const gone = expired(input.tokenId, input.sessionId, policy);
|
|
415
428
|
if (gone) return gone;
|
|
416
429
|
const route = options.routeOf(input.artifactId);
|
|
417
430
|
// `view` proceeds when the operator doesn't answer: nothing a person sees is held up by it.
|
package/src/server/hosted.ts
CHANGED
|
@@ -45,7 +45,7 @@ import {
|
|
|
45
45
|
} from "../gateway/index.ts";
|
|
46
46
|
import { platformName } from "../home/index.ts";
|
|
47
47
|
import { createEphemeralBus, createInteraction, type Interaction } from "../interaction/index.ts";
|
|
48
|
-
import type { Ledger } from "../ledger/index.ts";
|
|
48
|
+
import type { Ledger, LedgerDb } from "../ledger/index.ts";
|
|
49
49
|
import { createLive, liveMethods } from "../live/index.ts";
|
|
50
50
|
import type { Log } from "../log.ts";
|
|
51
51
|
import { removeTree } from "../platform/index.ts";
|
|
@@ -114,7 +114,7 @@ export interface HostedHome {
|
|
|
114
114
|
*/
|
|
115
115
|
wanted(sha256: string): number | undefined;
|
|
116
116
|
/** Records a file this home uploaded (`PUT /files/<sha256>`, §18.2). */
|
|
117
|
-
uploaded(sha256: string): void;
|
|
117
|
+
uploaded(sha256: string, size: number): void;
|
|
118
118
|
/** Pauses or resumes an artifact as its owner or the operator (D165 item 14). */
|
|
119
119
|
pause(
|
|
120
120
|
artifactId: string,
|
|
@@ -183,6 +183,8 @@ export interface HostedHomeOptions {
|
|
|
183
183
|
/** The relay's server settings (`relay` in the configuration, D165 item 4). */
|
|
184
184
|
/** The test fault points (`MOSTRA_TEST_FAULTS`); none by default. */
|
|
185
185
|
readonly faults?: Faults;
|
|
186
|
+
/** How often a session's activity is written at most (tests shorten it; an hour by default). */
|
|
187
|
+
readonly sessionTouchMs?: number | undefined;
|
|
186
188
|
readonly relay?: {
|
|
187
189
|
readonly waitingRunsPerArtifact?: number | undefined;
|
|
188
190
|
readonly drainPerSecond?: number | undefined;
|
|
@@ -380,6 +382,7 @@ export async function openHostedHome(options: HostedHomeOptions): Promise<Hosted
|
|
|
380
382
|
now: () => Date.now(),
|
|
381
383
|
shellOrigin: () => originOf(options.domains.active),
|
|
382
384
|
servesShell,
|
|
385
|
+
...(options.sessionTouchMs !== undefined ? { touchIntervalMs: options.sessionTouchMs } : {}),
|
|
383
386
|
// Every hosted link is indexed in the directory, so the operator's `revokeLink` finds its home.
|
|
384
387
|
onMinted: ({ tokenId, artifactId }) => {
|
|
385
388
|
const route = routeOf(artifactId);
|
|
@@ -456,6 +459,23 @@ export async function openHostedHome(options: HostedHomeOptions): Promise<Hosted
|
|
|
456
459
|
...(options.relay?.sweepMs !== undefined ? { sweepMs: options.relay.sweepMs } : {}),
|
|
457
460
|
...(options.relay?.liveRenewMs !== undefined ? { liveRenewMs: options.relay.liveRenewMs } : {}),
|
|
458
461
|
...(options.relay?.perLink !== undefined ? { perLink: options.relay.perLink } : {}),
|
|
462
|
+
// The operator's `limits` for a page's relayed actions (D165 item 9), each with quantity 1.
|
|
463
|
+
...(options.services
|
|
464
|
+
? {
|
|
465
|
+
limit: (artifactId: string, operation: "run" | "run.wait" | "call") => {
|
|
466
|
+
const services = options.services as HostedServices;
|
|
467
|
+
const route = routeOf(artifactId);
|
|
468
|
+
return services.operator.limit({
|
|
469
|
+
account: services.owner(id).account,
|
|
470
|
+
home: id,
|
|
471
|
+
artifact: artifactId,
|
|
472
|
+
...(route ? { routeId: route.id } : {}),
|
|
473
|
+
operation,
|
|
474
|
+
quantity: 1,
|
|
475
|
+
});
|
|
476
|
+
},
|
|
477
|
+
}
|
|
478
|
+
: {}),
|
|
459
479
|
latestVersion: (artifactId) =>
|
|
460
480
|
ledger.read(
|
|
461
481
|
(db) =>
|
|
@@ -633,6 +653,33 @@ export async function openHostedHome(options: HostedHomeOptions): Promise<Hosted
|
|
|
633
653
|
if (!gateway.dispatch(key, request, response)) notServed(response);
|
|
634
654
|
};
|
|
635
655
|
|
|
656
|
+
/**
|
|
657
|
+
* Records that this home holds a version file (DATA_MODEL §7.1's `home_file`), counted once in
|
|
658
|
+
* `file_ref`, with its size in bytes (the size its push gave, held to the bytes), which metering's
|
|
659
|
+
* `bytes.held` sums (MT-03). A row recorded before sizes were kept takes it now.
|
|
660
|
+
*/
|
|
661
|
+
const holdFile = (db: LedgerDb, sha: string, size: number | undefined): void => {
|
|
662
|
+
const added = db.run(
|
|
663
|
+
"INSERT OR IGNORE INTO home_file (home_id, sha256, size) VALUES (?, ?, ?)",
|
|
664
|
+
id,
|
|
665
|
+
sha,
|
|
666
|
+
size ?? null,
|
|
667
|
+
);
|
|
668
|
+
if (added.changes > 0) {
|
|
669
|
+
db.run(
|
|
670
|
+
"INSERT INTO file_ref (sha256, homes) VALUES (?, 1) ON CONFLICT (sha256) DO UPDATE SET homes = homes + 1, unreferenced_at = NULL",
|
|
671
|
+
sha,
|
|
672
|
+
);
|
|
673
|
+
} else if (size !== undefined) {
|
|
674
|
+
db.run(
|
|
675
|
+
"UPDATE home_file SET size = ? WHERE home_id = ? AND sha256 = ? AND size IS NULL",
|
|
676
|
+
size,
|
|
677
|
+
id,
|
|
678
|
+
sha,
|
|
679
|
+
);
|
|
680
|
+
}
|
|
681
|
+
};
|
|
682
|
+
|
|
636
683
|
/** The client files of a listing this home doesn't hold yet. */
|
|
637
684
|
const missingOf = (files: readonly string[]): string[] =>
|
|
638
685
|
directory.read((db) =>
|
|
@@ -868,6 +915,8 @@ export async function openHostedHome(options: HostedHomeOptions): Promise<Hosted
|
|
|
868
915
|
home: id,
|
|
869
916
|
artifact: params.artifactId,
|
|
870
917
|
routeId: route.routeId,
|
|
918
|
+
// The version itself, so a push that takes several rounds is one version (MT-03).
|
|
919
|
+
versionId: version.versionId,
|
|
871
920
|
operation,
|
|
872
921
|
quantity,
|
|
873
922
|
});
|
|
@@ -939,17 +988,7 @@ export async function openHostedHome(options: HostedHomeOptions): Promise<Hosted
|
|
|
939
988
|
});
|
|
940
989
|
directory.write((db) => {
|
|
941
990
|
for (const sha of new Set(clientFiles.map(([, , file]) => file))) {
|
|
942
|
-
|
|
943
|
-
"INSERT OR IGNORE INTO home_file (home_id, sha256) VALUES (?, ?)",
|
|
944
|
-
id,
|
|
945
|
-
sha,
|
|
946
|
-
);
|
|
947
|
-
if (added.changes > 0) {
|
|
948
|
-
db.run(
|
|
949
|
-
"INSERT INTO file_ref (sha256, homes) VALUES (?, 1) ON CONFLICT (sha256) DO UPDATE SET homes = homes + 1, unreferenced_at = NULL",
|
|
950
|
-
sha,
|
|
951
|
-
);
|
|
952
|
-
}
|
|
991
|
+
holdFile(db, sha, sizes.get(sha));
|
|
953
992
|
}
|
|
954
993
|
});
|
|
955
994
|
log.info("server.version_pushed", {
|
|
@@ -1165,20 +1204,8 @@ export async function openHostedHome(options: HostedHomeOptions): Promise<Hosted
|
|
|
1165
1204
|
}
|
|
1166
1205
|
return undefined;
|
|
1167
1206
|
},
|
|
1168
|
-
uploaded(sha) {
|
|
1169
|
-
directory.write((db) =>
|
|
1170
|
-
const added = db.run(
|
|
1171
|
-
"INSERT OR IGNORE INTO home_file (home_id, sha256) VALUES (?, ?)",
|
|
1172
|
-
id,
|
|
1173
|
-
sha,
|
|
1174
|
-
);
|
|
1175
|
-
if (added.changes > 0) {
|
|
1176
|
-
db.run(
|
|
1177
|
-
"INSERT INTO file_ref (sha256, homes) VALUES (?, 1) ON CONFLICT (sha256) DO UPDATE SET homes = homes + 1, unreferenced_at = NULL",
|
|
1178
|
-
sha,
|
|
1179
|
-
);
|
|
1180
|
-
}
|
|
1181
|
-
});
|
|
1207
|
+
uploaded(sha, size) {
|
|
1208
|
+
directory.write((db) => holdFile(db, sha, size));
|
|
1182
1209
|
for (const files of pending.values()) files.delete(sha);
|
|
1183
1210
|
},
|
|
1184
1211
|
pause: (artifactId, change) => interaction.pause(artifactId, change),
|
package/src/server/index.ts
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* ```
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
import { readFileSync } from "node:fs";
|
|
15
|
+
import { readFileSync, statSync } from "node:fs";
|
|
16
16
|
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
|
17
17
|
import type { AddressInfo } from "node:net";
|
|
18
18
|
import { join } from "node:path";
|
|
@@ -35,7 +35,7 @@ import {
|
|
|
35
35
|
shellAssetResponse,
|
|
36
36
|
} from "../gateway/index.ts";
|
|
37
37
|
import { appendEvent } from "../interaction/index.ts";
|
|
38
|
-
import type { LedgerOwner } from "../ledger/index.ts";
|
|
38
|
+
import type { Ledger, LedgerOwner } from "../ledger/index.ts";
|
|
39
39
|
import type { Environment, Log } from "../log.ts";
|
|
40
40
|
import { type Api, createApi } from "./api.ts";
|
|
41
41
|
import { type Authorization, createAuthorization } from "./auth.ts";
|
|
@@ -47,7 +47,7 @@ import { createHandle } from "./handle.ts";
|
|
|
47
47
|
import { type HostedHome, type HostedServices, openHostedHome } from "./hosted.ts";
|
|
48
48
|
import { type HomeOwner, homeUnavailable } from "./hosted-pages.ts";
|
|
49
49
|
import { createRemoteMcp, serverDocs } from "./mcp.ts";
|
|
50
|
-
import { createMeter, type Meter } from "./metering.ts";
|
|
50
|
+
import { createMeter, heldLevels, type Meter } from "./metering.ts";
|
|
51
51
|
import { createClients } from "./oauth-clients.ts";
|
|
52
52
|
import { loadOperator, type Operator } from "./operator.ts";
|
|
53
53
|
import { type BuiltInProvider, builtInProviders } from "./providers.ts";
|
|
@@ -136,27 +136,28 @@ export async function startServer(options: ServerOptions): Promise<RemoteServer>
|
|
|
136
136
|
* health check is never counted. Without the setting there is no limit.
|
|
137
137
|
*/
|
|
138
138
|
const perIpRequests = rateOf(config.rateLimits?.perIp?.requests);
|
|
139
|
-
/**
|
|
139
|
+
/**
|
|
140
|
+
* Who owns a home, as its pages' sender line names them (D161): the account's name, and the
|
|
141
|
+
* verifier its first identity recorded when its provider named the person (MT-03), never the
|
|
142
|
+
* configuration now, so a provider renamed or retired since changes no sender line. A name the
|
|
143
|
+
* development provider took on trust, or one no provider verified, carries none.
|
|
144
|
+
*/
|
|
140
145
|
const ownerOf = (homeId: string): HomeOwner => {
|
|
141
146
|
const row = data.directory.read((db) =>
|
|
142
|
-
db.get<{ account: string; name: string | null;
|
|
143
|
-
`SELECT a.id AS account, a.name,
|
|
144
|
-
|
|
145
|
-
|
|
147
|
+
db.get<{ account: string; name: string | null; verified_by: string | null }>(
|
|
148
|
+
`SELECT a.id AS account, a.name, i.verified_by
|
|
149
|
+
FROM home h JOIN account a ON a.id = h.owner_account_id
|
|
150
|
+
LEFT JOIN identity i ON i.rowid = (SELECT rowid FROM identity WHERE account_id = a.id
|
|
151
|
+
ORDER BY linked_at, rowid LIMIT 1)
|
|
152
|
+
WHERE h.id = ?`,
|
|
146
153
|
homeId,
|
|
147
154
|
),
|
|
148
155
|
);
|
|
149
156
|
if (!row) return { account: "" };
|
|
150
|
-
const provider = config.identityProviders?.find((candidate) => candidate.id === row.provider);
|
|
151
|
-
// A name the development provider took on trust is unverified (D161).
|
|
152
|
-
const verifiedBy =
|
|
153
|
-
provider && provider.kind !== "development"
|
|
154
|
-
? (provider.label ?? (provider.kind === "github" ? "GitHub" : provider.id))
|
|
155
|
-
: undefined;
|
|
156
157
|
return {
|
|
157
158
|
account: row.account,
|
|
158
159
|
...(row.name !== null ? { name: row.name } : {}),
|
|
159
|
-
...(
|
|
160
|
+
...(row.verified_by !== null ? { verifiedBy: row.verified_by } : {}),
|
|
160
161
|
};
|
|
161
162
|
};
|
|
162
163
|
const services = (): HostedServices => ({
|
|
@@ -194,6 +195,9 @@ export async function startServer(options: ServerOptions): Promise<RemoteServer>
|
|
|
194
195
|
domains: { active: config.domains.active, spares: config.domains.spares ?? [] },
|
|
195
196
|
shellDir,
|
|
196
197
|
services: services(),
|
|
198
|
+
...(Number(options.env["MOSTRA_TEST_SESSION_TOUCH_MS"]) > 0
|
|
199
|
+
? { sessionTouchMs: Number(options.env["MOSTRA_TEST_SESSION_TOUCH_MS"]) }
|
|
200
|
+
: {}),
|
|
197
201
|
relay: {
|
|
198
202
|
waitingRunsPerArtifact: config.relay?.waitingRunsPerArtifact,
|
|
199
203
|
drainPerSecond: config.relay?.drainPerSecond,
|
|
@@ -263,6 +267,8 @@ export async function startServer(options: ServerOptions): Promise<RemoteServer>
|
|
|
263
267
|
await data.close();
|
|
264
268
|
throw error;
|
|
265
269
|
}
|
|
270
|
+
// A held file recorded before sizes were kept (directory migration 6) takes its stored size once.
|
|
271
|
+
fillHeldSizes(data.directory, join(options.data, "files"), log);
|
|
266
272
|
routing = createRouting({
|
|
267
273
|
apiHost: new URL(config.apiOrigin).host,
|
|
268
274
|
domains: () => [config.domains.active, ...(config.domains.spares ?? [])],
|
|
@@ -279,6 +285,11 @@ export async function startServer(options: ServerOptions): Promise<RemoteServer>
|
|
|
279
285
|
? { windowMs: Number(options.env["MOSTRA_TEST_METER_WINDOW_MS"]) }
|
|
280
286
|
: {}),
|
|
281
287
|
activeHomes: () => data.open().map((home) => ({ home, account: ownerOf(home).account })),
|
|
288
|
+
// `bytes.held` (MT-03): each home's running total, sent when it changed, and all in a snapshot.
|
|
289
|
+
held: heldLevels(data.directory),
|
|
290
|
+
...(Number(options.env["MOSTRA_TEST_HELD_SNAPSHOT_MS"]) > 0
|
|
291
|
+
? { snapshotMs: Number(options.env["MOSTRA_TEST_HELD_SNAPSHOT_MS"]) }
|
|
292
|
+
: {}),
|
|
282
293
|
});
|
|
283
294
|
auth = createAuthorization({
|
|
284
295
|
directory: data.directory,
|
|
@@ -658,3 +669,39 @@ export async function startServer(options: ServerOptions): Promise<RemoteServer>
|
|
|
658
669
|
},
|
|
659
670
|
};
|
|
660
671
|
}
|
|
672
|
+
|
|
673
|
+
/**
|
|
674
|
+
* Gives each `home_file` row recorded before sizes were kept (directory migration 6) the size of
|
|
675
|
+
* its stored file, so `bytes.held` counts it (MT-03); the triggers add it to the home's total. A
|
|
676
|
+
* file that is gone holds nothing on disk, so its row takes 0 and is never read again.
|
|
677
|
+
*/
|
|
678
|
+
function fillHeldSizes(directory: Ledger, filesDir: string, log: Log): void {
|
|
679
|
+
const rows = directory.read((db) =>
|
|
680
|
+
db.all<{ home_id: string; sha256: string }>(
|
|
681
|
+
"SELECT home_id, sha256 FROM home_file WHERE size IS NULL",
|
|
682
|
+
),
|
|
683
|
+
);
|
|
684
|
+
if (rows.length === 0) return;
|
|
685
|
+
const sizes = new Map<string, number>();
|
|
686
|
+
let missing = 0;
|
|
687
|
+
for (const row of rows) {
|
|
688
|
+
if (sizes.has(row.sha256)) continue;
|
|
689
|
+
try {
|
|
690
|
+
sizes.set(row.sha256, statSync(join(filesDir, row.sha256)).size);
|
|
691
|
+
} catch {
|
|
692
|
+
sizes.set(row.sha256, 0);
|
|
693
|
+
missing += 1;
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
directory.write((db) => {
|
|
697
|
+
for (const row of rows) {
|
|
698
|
+
db.run(
|
|
699
|
+
"UPDATE home_file SET size = ? WHERE home_id = ? AND sha256 = ? AND size IS NULL",
|
|
700
|
+
sizes.get(row.sha256) ?? 0,
|
|
701
|
+
row.home_id,
|
|
702
|
+
row.sha256,
|
|
703
|
+
);
|
|
704
|
+
}
|
|
705
|
+
});
|
|
706
|
+
log.info("server.held_sizes_filled", { files: sizes.size, missing });
|
|
707
|
+
}
|
package/src/server/metering.ts
CHANGED
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
* Metering for an operator (INTERFACES §18.6, D165 item 9): counters kept in memory for a window,
|
|
3
3
|
* then handed to the operator's `metering.record` as one batch with a stable `batchId`, through
|
|
4
4
|
* the durable outbound queue, so a batch is delivered at least once across restarts. With no
|
|
5
|
-
* metering port nothing is counted.
|
|
5
|
+
* metering port nothing is counted. Besides the sums, windows carry one level, `bytes.held`: what
|
|
6
|
+
* a home holds now, sent when it changes (and 0 once when it holds nothing any more), and for every
|
|
7
|
+
* home that holds anything at the first window after a start and once a day after (MT-03). A
|
|
8
|
+
* window's counters go out in batches of at most {@link MAX_COUNTERS}.
|
|
6
9
|
*
|
|
7
10
|
* Views carry `viewerKey`, a monthly pseudonymous key: an HMAC of the viewer's subject under a key
|
|
8
11
|
* the server makes for each month (`<data>/keys/metering-<YYYY-MM>`), so an operator can count
|
|
@@ -14,11 +17,23 @@ import { createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
|
14
17
|
import { readFileSync, writeFileSync } from "node:fs";
|
|
15
18
|
import { join } from "node:path";
|
|
16
19
|
import type { MeterCounter, MeterName } from "@mostra/protocol/operator";
|
|
20
|
+
import type { Ledger } from "../ledger/index.ts";
|
|
17
21
|
import type { Log } from "../log.ts";
|
|
18
22
|
import type { Operator } from "./operator.ts";
|
|
19
23
|
|
|
20
24
|
/** How long one batch's window lasts. */
|
|
21
25
|
const WINDOW_MS = 60_000;
|
|
26
|
+
/** How often every home that holds anything reports its level again, changed or not. */
|
|
27
|
+
const SNAPSHOT_MS = 24 * 60 * 60 * 1000;
|
|
28
|
+
/** The most counters one batch carries; a window with more sends several batches. */
|
|
29
|
+
export const MAX_COUNTERS = 1000;
|
|
30
|
+
|
|
31
|
+
/** One home's held level (`bytes.held`, MT-03). */
|
|
32
|
+
export interface HeldLevel {
|
|
33
|
+
readonly home: string;
|
|
34
|
+
readonly account: string;
|
|
35
|
+
readonly bytes: number;
|
|
36
|
+
}
|
|
22
37
|
|
|
23
38
|
/** What one counter is counted under, besides its name. */
|
|
24
39
|
export interface MeterScope {
|
|
@@ -53,6 +68,21 @@ export interface MeterOptions {
|
|
|
53
68
|
readonly windowMs?: number;
|
|
54
69
|
/** Seconds each open home was active, added at every window. */
|
|
55
70
|
readonly activeHomes?: () => readonly { readonly home: string; readonly account: string }[];
|
|
71
|
+
/**
|
|
72
|
+
* The homes' held levels for `bytes.held` (MT-03), kept durably by the caller so a change made
|
|
73
|
+
* just before a stop, or while nothing was reported, is still sent after a restart.
|
|
74
|
+
*/
|
|
75
|
+
readonly held?: {
|
|
76
|
+
/**
|
|
77
|
+
* The levels to report: each home whose level changed since it was last reported, or with
|
|
78
|
+
* `full`, also every home that holds anything (or last reported more than nothing).
|
|
79
|
+
*/
|
|
80
|
+
pending(full: boolean): readonly HeldLevel[];
|
|
81
|
+
/** The levels were queued: each is reported unless its home's level changed meanwhile. */
|
|
82
|
+
reported(levels: readonly HeldLevel[]): void;
|
|
83
|
+
};
|
|
84
|
+
/** How often every held level is sent again; a day unless a test shortens it. */
|
|
85
|
+
readonly snapshotMs?: number;
|
|
56
86
|
}
|
|
57
87
|
|
|
58
88
|
/** Creates the meter. */
|
|
@@ -63,6 +93,9 @@ export function createMeter(options: MeterOptions): Meter {
|
|
|
63
93
|
const counters = new Map<string, { counter: MeterCounter; value: number }>();
|
|
64
94
|
let from = now();
|
|
65
95
|
const monthKeys = new Map<string, Buffer>();
|
|
96
|
+
const snapshotMs = options.snapshotMs ?? SNAPSHOT_MS;
|
|
97
|
+
/** When every level was last sent; never, so the first window after a start sends them all. */
|
|
98
|
+
let snapshotAt = Number.NEGATIVE_INFINITY;
|
|
66
99
|
|
|
67
100
|
const monthKey = (): Buffer => {
|
|
68
101
|
const month = new Date(now()).toISOString().slice(0, 7);
|
|
@@ -94,19 +127,39 @@ export function createMeter(options: MeterOptions): Meter {
|
|
|
94
127
|
add("home.activeSeconds", { account: active.account, home: active.home }, seconds);
|
|
95
128
|
}
|
|
96
129
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
130
|
+
// `bytes.held` is a level, not a sum: a home whose level changed reports it, and every home
|
|
131
|
+
// that holds anything reports it again at the first window after a start and once a day.
|
|
132
|
+
const full = to - snapshotAt >= snapshotMs;
|
|
133
|
+
const levels = options.held?.pending(full) ?? [];
|
|
134
|
+
if (full) snapshotAt = to;
|
|
135
|
+
const held: MeterCounter[] = levels
|
|
136
|
+
.filter((level) => level.account !== "" && level.bytes >= 0)
|
|
137
|
+
.map((level) => ({
|
|
138
|
+
name: "bytes.held",
|
|
139
|
+
account: level.account,
|
|
140
|
+
home: level.home,
|
|
141
|
+
value: level.bytes,
|
|
142
|
+
}));
|
|
143
|
+
const all = [
|
|
144
|
+
...[...counters.values()].map(({ counter, value }) => ({ ...counter, value })),
|
|
145
|
+
...held,
|
|
146
|
+
];
|
|
107
147
|
counters.clear();
|
|
148
|
+
if (all.length > 0) {
|
|
149
|
+
const window = { from: new Date(from).toISOString(), to: new Date(to).toISOString() };
|
|
150
|
+
for (let start = 0; start < all.length; start += MAX_COUNTERS) {
|
|
151
|
+
options.operator.enqueue({
|
|
152
|
+
port: "metering",
|
|
153
|
+
batch: {
|
|
154
|
+
batchId: randomUUID(),
|
|
155
|
+
...window,
|
|
156
|
+
counters: all.slice(start, start + MAX_COUNTERS),
|
|
157
|
+
},
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (levels.length > 0) options.held?.reported(levels);
|
|
108
162
|
from = to;
|
|
109
|
-
options.operator.enqueue({ port: "metering", batch });
|
|
110
163
|
};
|
|
111
164
|
|
|
112
165
|
const add = (name: MeterName, scope: MeterScope, value = 1): void => {
|
|
@@ -154,3 +207,55 @@ export function createMeter(options: MeterOptions): Meter {
|
|
|
154
207
|
},
|
|
155
208
|
};
|
|
156
209
|
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* The held levels of a server's homes, from its directory (MT-03): each home's running total in
|
|
213
|
+
* `home.held_bytes`, which `home_file`'s triggers keep in the transaction of every change, and
|
|
214
|
+
* `held_dirty`, set by them until the level is queued, so a change made just before a stop is
|
|
215
|
+
* still reported after the restart. A level is marked reported only while it is still the home's.
|
|
216
|
+
*/
|
|
217
|
+
export function heldLevels(directory: Ledger): NonNullable<MeterOptions["held"]> {
|
|
218
|
+
return {
|
|
219
|
+
pending: (full) =>
|
|
220
|
+
directory.write((db) => {
|
|
221
|
+
const rows = db.all<{
|
|
222
|
+
home: string;
|
|
223
|
+
account: string;
|
|
224
|
+
bytes: number;
|
|
225
|
+
reported: number | null;
|
|
226
|
+
}>(
|
|
227
|
+
`SELECT id AS home, owner_account_id AS account, held_bytes AS bytes,
|
|
228
|
+
held_reported AS reported
|
|
229
|
+
FROM home
|
|
230
|
+
WHERE held_dirty = 1${full ? " OR held_bytes > 0 OR COALESCE(held_reported, 0) > 0" : ""}`,
|
|
231
|
+
);
|
|
232
|
+
const levels: HeldLevel[] = [];
|
|
233
|
+
for (const row of rows) {
|
|
234
|
+
// Outside a snapshot, a change that came back to the level last sent, or a home that
|
|
235
|
+
// never held anything, has nothing new to say.
|
|
236
|
+
if (!full && row.bytes === (row.reported ?? 0)) {
|
|
237
|
+
db.run(
|
|
238
|
+
"UPDATE home SET held_dirty = 0 WHERE id = ? AND held_bytes = ?",
|
|
239
|
+
row.home,
|
|
240
|
+
row.bytes,
|
|
241
|
+
);
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
if (row.reported === null && row.bytes === 0) continue;
|
|
245
|
+
levels.push({ home: row.home, account: row.account, bytes: row.bytes });
|
|
246
|
+
}
|
|
247
|
+
return levels;
|
|
248
|
+
}),
|
|
249
|
+
reported: (levels) =>
|
|
250
|
+
directory.write((db) => {
|
|
251
|
+
for (const level of levels) {
|
|
252
|
+
db.run(
|
|
253
|
+
"UPDATE home SET held_reported = ?, held_dirty = 0 WHERE id = ? AND held_bytes = ?",
|
|
254
|
+
level.bytes,
|
|
255
|
+
level.home,
|
|
256
|
+
level.bytes,
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
}),
|
|
260
|
+
};
|
|
261
|
+
}
|
package/src/server/providers.ts
CHANGED
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
*
|
|
6
6
|
* - `oidc`: any OpenID Connect provider, by its issuer's discovery document; the authorization
|
|
7
7
|
* code flow with PKCE (S256) and a nonce, and the ID token checked (signature, issuer, audience,
|
|
8
|
-
* expiry, nonce) by `openid-client`.
|
|
8
|
+
* expiry, nonce) by `openid-client`; the claims its entry names (`claims`) go into `facts`.
|
|
9
9
|
* - `github`: GitHub's OAuth web flow with PKCE, then `GET /user` for the account's stable ID, its
|
|
10
|
-
* login and name, and its age (`facts.createdAt`)
|
|
10
|
+
* login and name, and for an operator's policy its age (`facts.createdAt`) and public counts
|
|
11
|
+
* (`facts.publicRepos`, `publicGists`, `followers`).
|
|
11
12
|
* - `development`: a page that asks for a name and trusts it, for tests and a self-hoster's first
|
|
12
13
|
* run; its names are never shown as verified.
|
|
13
14
|
*
|
|
@@ -24,7 +25,7 @@ import type {
|
|
|
24
25
|
} from "@mostra/protocol/operator";
|
|
25
26
|
import * as oidc from "openid-client";
|
|
26
27
|
import type { Environment, Log } from "../log.ts";
|
|
27
|
-
import type
|
|
28
|
+
import { configuredVerifier, type IdentityProviderConfig } from "./config.ts";
|
|
28
29
|
|
|
29
30
|
/** How long a provider's discovery, token exchange or user lookup may take. */
|
|
30
31
|
const PROVIDER_TIMEOUT_MS = 8_000;
|
|
@@ -36,11 +37,17 @@ const GITHUB = {
|
|
|
36
37
|
user: "https://api.github.com/user",
|
|
37
38
|
} as const;
|
|
38
39
|
|
|
39
|
-
/** A built-in provider: the port's shape, and
|
|
40
|
+
/** A built-in provider: the port's shape, its kind, and its verifier (all but `development`). */
|
|
40
41
|
export interface BuiltInProvider extends IdentityProvider {
|
|
41
42
|
readonly kind: IdentityProviderConfig["kind"];
|
|
42
|
-
|
|
43
|
-
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** The `verifies` of a configured provider: the label its verified names carry, if any. */
|
|
46
|
+
function verifiesOf(config: IdentityProviderConfig): {
|
|
47
|
+
readonly verifies?: { readonly label: string };
|
|
48
|
+
} {
|
|
49
|
+
const label = configuredVerifier(config);
|
|
50
|
+
return label === undefined ? {} : { verifies: { label } };
|
|
44
51
|
}
|
|
45
52
|
|
|
46
53
|
function invalid(message: string): RpcError {
|
|
@@ -69,6 +76,36 @@ async function timedFetch(url: string, init: RequestInit): Promise<Response> {
|
|
|
69
76
|
return fetch(url, { ...init, signal: AbortSignal.timeout(PROVIDER_TIMEOUT_MS) });
|
|
70
77
|
}
|
|
71
78
|
|
|
79
|
+
/** The longest a chosen claim's value may be, as JSON: a policy reads names, not documents. */
|
|
80
|
+
const CLAIM_MAX = 4_096;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The ID token's claims a configuration chose (`claims`), as `facts` under their own names: each
|
|
84
|
+
* one present whose value is JSON of at most 4 KiB (MT-03). A claim the token doesn't carry is left
|
|
85
|
+
* out; with none chosen, nothing is added.
|
|
86
|
+
*/
|
|
87
|
+
function chosen(
|
|
88
|
+
claims: Readonly<Record<string, unknown>>,
|
|
89
|
+
names: readonly string[] | undefined,
|
|
90
|
+
): { readonly facts: Record<string, unknown> } | undefined {
|
|
91
|
+
if (names === undefined) return undefined;
|
|
92
|
+
const facts: Record<string, unknown> = {};
|
|
93
|
+
for (const name of names) {
|
|
94
|
+
if (!Object.hasOwn(claims, name)) continue;
|
|
95
|
+
const value = claims[name];
|
|
96
|
+
const text = JSON.stringify(value);
|
|
97
|
+
if (text === undefined || text.length > CLAIM_MAX) continue;
|
|
98
|
+
// A key of its own, whatever the name (`__proto__` would set the prototype by assignment).
|
|
99
|
+
Object.defineProperty(facts, name, {
|
|
100
|
+
value: JSON.parse(text) as unknown,
|
|
101
|
+
enumerable: true,
|
|
102
|
+
writable: true,
|
|
103
|
+
configurable: true,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
return Object.keys(facts).length > 0 ? { facts } : undefined;
|
|
107
|
+
}
|
|
108
|
+
|
|
72
109
|
/** The OpenID Connect adapter. Discovery happens at the first sign-in and is kept. */
|
|
73
110
|
function oidcProvider(config: IdentityProviderConfig, env: Environment): BuiltInProvider {
|
|
74
111
|
if (config.issuer === undefined || config.clientId === undefined) {
|
|
@@ -106,7 +143,7 @@ function oidcProvider(config: IdentityProviderConfig, env: Environment): BuiltIn
|
|
|
106
143
|
id: config.id,
|
|
107
144
|
label: config.label ?? config.id,
|
|
108
145
|
kind: "oidc",
|
|
109
|
-
|
|
146
|
+
...verifiesOf(config),
|
|
110
147
|
async authorize(state: string, pkce: SignInPkce, context: SignInContext) {
|
|
111
148
|
const url = oidc.buildAuthorizationUrl(await configuration(), {
|
|
112
149
|
redirect_uri: context.redirectUri,
|
|
@@ -143,12 +180,18 @@ function oidcProvider(config: IdentityProviderConfig, env: Environment): BuiltIn
|
|
|
143
180
|
? { emailVerified: claims["email_verified"] }
|
|
144
181
|
: {}),
|
|
145
182
|
...(name !== undefined ? { name } : {}),
|
|
183
|
+
...(chosen(claims, config.claims) ?? {}),
|
|
146
184
|
};
|
|
147
185
|
return identity;
|
|
148
186
|
},
|
|
149
187
|
};
|
|
150
188
|
}
|
|
151
189
|
|
|
190
|
+
/** A count GitHub gave, under its name in `facts`: a non-negative integer, or nothing. */
|
|
191
|
+
function count(name: string, value: unknown): Record<string, number> {
|
|
192
|
+
return Number.isSafeInteger(value) && (value as number) >= 0 ? { [name]: value as number } : {};
|
|
193
|
+
}
|
|
194
|
+
|
|
152
195
|
/** The GitHub adapter: an OAuth app's web flow with PKCE (VERIFIED_FACTS, WP-30). */
|
|
153
196
|
function githubProvider(config: IdentityProviderConfig, env: Environment): BuiltInProvider {
|
|
154
197
|
if (config.clientId === undefined) {
|
|
@@ -169,7 +212,7 @@ function githubProvider(config: IdentityProviderConfig, env: Environment): Built
|
|
|
169
212
|
id: config.id,
|
|
170
213
|
label: config.label ?? "GitHub",
|
|
171
214
|
kind: "github",
|
|
172
|
-
|
|
215
|
+
...verifiesOf(config),
|
|
173
216
|
authorize(state, pkce, context) {
|
|
174
217
|
const url = new URL(endpoints.authorize);
|
|
175
218
|
url.searchParams.set("client_id", config.clientId as string);
|
|
@@ -221,6 +264,9 @@ function githubProvider(config: IdentityProviderConfig, env: Environment): Built
|
|
|
221
264
|
name?: string | null;
|
|
222
265
|
email?: string | null;
|
|
223
266
|
created_at?: string;
|
|
267
|
+
public_repos?: number;
|
|
268
|
+
public_gists?: number;
|
|
269
|
+
followers?: number;
|
|
224
270
|
};
|
|
225
271
|
if (!user.ok || typeof found.id !== "number") {
|
|
226
272
|
throw new Error(`GitHub's user lookup failed (${user.status}).`);
|
|
@@ -231,9 +277,16 @@ function githubProvider(config: IdentityProviderConfig, env: Environment): Built
|
|
|
231
277
|
subject: String(found.id),
|
|
232
278
|
...(typeof found.email === "string" ? { email: found.email } : {}),
|
|
233
279
|
name: found.name ?? found.login ?? String(found.id),
|
|
280
|
+
// What the same answer tells an operator's policy (an account's age and its public
|
|
281
|
+
// counts, so a fresh, empty account is told from an established one with no other
|
|
282
|
+
// request inside `accounts.resolve`'s time limit, MT-03); a count GitHub didn't give is
|
|
283
|
+
// left out rather than guessed.
|
|
234
284
|
facts: {
|
|
235
285
|
login: found.login ?? null,
|
|
236
286
|
...(found.created_at !== undefined ? { createdAt: found.created_at } : {}),
|
|
287
|
+
...count("publicRepos", found.public_repos),
|
|
288
|
+
...count("publicGists", found.public_gists),
|
|
289
|
+
...count("followers", found.followers),
|
|
237
290
|
},
|
|
238
291
|
};
|
|
239
292
|
},
|
|
@@ -249,7 +302,6 @@ function developmentProvider(config: IdentityProviderConfig, apiOrigin: string):
|
|
|
249
302
|
id: config.id,
|
|
250
303
|
label: config.label ?? "Development sign-in",
|
|
251
304
|
kind: "development",
|
|
252
|
-
verifies: false,
|
|
253
305
|
authorize(state) {
|
|
254
306
|
const url = new URL(`${apiOrigin}/signin/${config.id}/name`);
|
|
255
307
|
url.searchParams.set("state", state);
|