@typetorch/framework 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -16
- package/out/analytics/SCHEMA.md +11 -2
- package/out/analytics/schema.d.ts +32 -0
- package/out/analytics/schema.luau +10 -0
- package/out/analytics/server.d.ts +7 -0
- package/out/analytics/server.luau +112 -20
- package/out/analytics/settings.d.ts +10 -0
- package/out/analytics/settings.luau +85 -0
- package/out/analytics/sinks.d.ts +11 -4
- package/out/analytics/sinks.luau +35 -3
- package/out/devtools/ab.d.ts +6 -6
- package/out/devtools/ab.luau +5 -5
- package/out/devtools/admin-server.d.ts +14 -12
- package/out/devtools/admin-server.luau +55 -66
- package/out/devtools/admin-ui.d.ts +2 -2
- package/out/devtools/admin-ui.luau +782 -298
- package/out/devtools/build-actions.d.ts +102 -0
- package/out/devtools/build-actions.luau +272 -0
- package/out/devtools/claude.luau +111 -8
- package/out/devtools/client.luau +451 -138
- package/out/devtools/health.d.ts +5 -1
- package/out/devtools/health.luau +29 -9
- package/out/devtools/roll-call.d.ts +2 -2
- package/out/devtools/roll-call.luau +4 -4
- package/out/devtools/server.luau +85 -14
- package/out/devtools/state-inspect.d.ts +87 -0
- package/out/devtools/state-inspect.luau +1030 -0
- package/out/devtools/state-ui.d.ts +39 -0
- package/out/devtools/state-ui.luau +691 -0
- package/out/devtools/state.d.ts +6 -0
- package/out/devtools/state.luau +62 -3
- package/out/devtools/widgets.d.ts +26 -0
- package/out/devtools/widgets.luau +136 -0
- package/out/kernel.d.ts +50 -10
- package/out/kernel.luau +21 -3
- package/out/typetorch.d.ts +6 -4
- package/out/typetorch.luau +9 -6
- package/out/version.d.ts +5 -0
- package/out/version.luau +11 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -115,13 +115,31 @@ export class CoinService extends Module implements OnStart {
|
|
|
115
115
|
work the same way; see the transformer's README.
|
|
116
116
|
- **UI:** `observeElement(trove, tag, (instance, elementTrove) => ...)`, `isRealFrame`, `popIn` / `popOut` / `bump`
|
|
117
117
|
(UIScale, never Size tweens) and `PopupQueue` (one modal at a time).
|
|
118
|
-
- **Dev menu:** devs (Studio, project members, dev badge) get a DEV button, `Ctrl+Shift+D` and `/tt dev`: artifact
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
118
|
+
- **Dev menu:** devs (Studio, project members, dev badge) get a DEV button, `Ctrl+Shift+D` and `/tt dev`: artifact
|
|
119
|
+
(with how long the server and client generations have run, the server's uptime and the build's age), server status,
|
|
120
|
+
logs (server, own client, other players' clients), client and server dex, network stats, module state, a branch and
|
|
121
|
+
build picker (Server > Branch) and a Claude prompt. Prod-channel servers are read-only. The window can be dragged by
|
|
122
|
+
its header and resized from its corner (double-tap the header to reset).
|
|
123
|
+
- **State explorer (Modules > State, Server | Client):** every live module of the running generation (services on the
|
|
124
|
+
server, controllers on the client) and the persist store. Tap a row to open it: the module's own fields (not its
|
|
125
|
+
methods), then tables, arrays, Maps and Sets, 100 entries per page (Prev / Next). Each value has a short preview:
|
|
126
|
+
strings (cut), numbers, booleans, nil, datatypes through `tostring`, an Instance as its full path, a Player as its
|
|
127
|
+
name, `function`, `thread`, buffer sizes. Nothing is ever called (raw reads only: no metamethods, no getters); a
|
|
128
|
+
value that is its own ancestor shows as a cycle. A key filter (open nodes stay listed), Refresh and Auto (every 2 s).
|
|
129
|
+
Server state comes from the op `state.inspect` (rate-limited, size-capped): devs on dev-channel servers, owners only
|
|
130
|
+
on prod-effective servers, since server state can hold player data.
|
|
131
|
+
- **Logs > Upload:** sends the logs shown (Server: this server's log ring; Client: your own client's; Others: the
|
|
132
|
+
picked player's client) to the dev PC paired in the Claude tab, which saves them under `<repo>/.typetorch/logs/`.
|
|
133
|
+
No Claude run, no prompt quota. Not paired: "Pair in the Claude tab first".
|
|
134
|
+
- **Switch a live server (kernel 0.3.4):** owners switch any server and load builds on it, public and prod ones too:
|
|
135
|
+
Server > Branch, "Switch" on a branch or "Load here" on a build, then the green Confirm. Everyone stays. These are
|
|
136
|
+
ordinary switches and pins: Reload and Rollback keep working, and going back to prod is Switch on the prod row (its
|
|
137
|
+
verified head). A public switch lasts for that server's lifetime and is never stored, so new servers still boot the
|
|
138
|
+
signed prod head. Others see "Join", which moves only them to a reserved server. Manage > Servers "Load a build..."
|
|
139
|
+
does the same for this server, the ticked servers or a share of one branch (A/B pins), with one status line under
|
|
140
|
+
the list.
|
|
141
|
+
- **Owners and devs:** two roles (no admins: an older kernel's or typetorch.json's "admin" counts as a dev). The Manage
|
|
142
|
+
group (Players, Servers, Bans) is for owners only.
|
|
125
143
|
- **Remote Claude** (dev-channel servers only): while `typetorch remote-claude` (`@typetorch/dev-server`) runs on a dev's machine, allowlisted
|
|
126
144
|
devs prompt Claude Code from the Claude tab. Each dev pairs once by pasting the pairing code printed by
|
|
127
145
|
typetorch-dev-server; the game server keeps the session URL and the tokens in memory and never sends them to clients.
|
|
@@ -155,13 +173,13 @@ this.trove.add(TypeTorch.onUpdatePending((update) => (hint.Visible = !update.can
|
|
|
155
173
|
const scores = TypeTorch.persist("scores.v1", () => new Map<number, number>());
|
|
156
174
|
|
|
157
175
|
// Dev and roles (server: the kernel decides; client: only the local player, cosmetic)
|
|
158
|
-
if (TypeTorch.
|
|
159
|
-
TypeTorch.onPlayerDevChanged((player, info) =>
|
|
176
|
+
if (TypeTorch.isOwner(player)) showOwnerPanel(player);
|
|
177
|
+
TypeTorch.onPlayerDevChanged((player, info) => setDevTools(player, info.dev));
|
|
160
178
|
|
|
161
179
|
// Server only
|
|
162
180
|
TypeTorch.status(); // uptime, players, memory, history: cheap
|
|
163
181
|
TypeTorch.branches(); TypeTorch.artifacts(); // registry reads, cached ~30 s: may yield, don't call per frame
|
|
164
|
-
TypeTorch.requestReload(player); //
|
|
182
|
+
TypeTorch.requestReload(player); // owners only, checked by the kernel
|
|
165
183
|
|
|
166
184
|
// Logs (the kernel's ring buffer)
|
|
167
185
|
TypeTorch.logs(0, 50);
|
|
@@ -284,8 +302,9 @@ const variant = analytics.experiment("onboarding", ["short", "long"]); // same a
|
|
|
284
302
|
country, friends in the server, first-ever vs returning, days since the last visit), `tech` (FPS, ping, memory, load
|
|
285
303
|
time, client and server errors, swaps and rollbacks, leaves within 60 s of a swap), `zones` (parts or models tagged
|
|
286
304
|
`TTZone`, named by a `Name` attribute or the instance name), `screens` (ScreenGuis in PlayerGui, GuiObjects tagged
|
|
287
|
-
`TTScreen`), `recording` (the first-ever session in detail), `fleet` (kernel 0.3.2 heartbeats and deploy reports)
|
|
288
|
-
`settings` (server only) replaces the
|
|
305
|
+
`TTScreen`), `recording` (the first-ever session in detail), `fleet` (kernel 0.3.2 heartbeats and deploy reports),
|
|
306
|
+
`identity` (one `{ pid, uid }` row per session to the dev's own server, below). `settings` (server only) replaces the
|
|
307
|
+
ConfigService key, for tests.
|
|
289
308
|
- **Every row** carries the time (server clock), a random player id (never the UserId), the session, the server
|
|
290
309
|
(JobId, type, place), the artifact (id, seq, branch, channel), the device, new vs returning, the player's state
|
|
291
310
|
(`zone:Lobby|screen:Shop|activity:round`) and experiment variants. The exact rows: `src/analytics/SCHEMA.md`.
|
|
@@ -297,7 +316,12 @@ const variant = analytics.experiment("onboarding", ["short", "long"]); // same a
|
|
|
297
316
|
about 10 times a second, every input (never while a TextBox or the chat has focus; text boxes only say which box was
|
|
298
317
|
used), buttons pressed and hovered, screens, prompts and deaths, from the join until 60 s after the first input.
|
|
299
318
|
Packed into small binary chunks (10-20 KB a player) and sent through the server.
|
|
300
|
-
- **Never collected:** chat or anything typed, usernames and display names (error texts have them replaced)
|
|
319
|
+
- **Never collected:** chat or anything typed, usernames and display names (error texts have them replaced). No event
|
|
320
|
+
row carries a UserId.
|
|
321
|
+
- **Identities** (option `identity`, default on): once the pid is known, one row `{ pid, uid, t }` (the UserId, nothing
|
|
322
|
+
else) per session goes to the dev's own server only, never into the events: duckdb games in the batch body, Basin
|
|
323
|
+
games to the fleet API (`TypeTorchFleet` url + `/v1/identity`, or the `identity` / `identityToken` settings). The
|
|
324
|
+
analytics server keeps pid -> UserId in a deletable table (support lookups, Right to Erasure).
|
|
301
325
|
- **Player ids:** DataStore `TypeTorchAnalytics`, key `p/<UserId>` -> `{ pid, first, last }`: one read per join, a
|
|
302
326
|
write on the first join and at leave. Deleting the key (Right to Erasure) leaves that player's rows anonymous.
|
|
303
327
|
- **Sending:** one queue on the server (10,000 events; over that the oldest are dropped and counted), a flush every
|
|
@@ -305,8 +329,9 @@ const variant = analytics.experiment("onboarding", ["short", "long"]); // same a
|
|
|
305
329
|
flush on shutdown (kernel 0.3.2). Delivery is at least once (a swap mid-request sends that batch again).
|
|
306
330
|
`analytics.stats()` (server) has the counters; `flush()` sends soon.
|
|
307
331
|
|
|
308
|
-
**Settings** (server only, never sent to clients): the ConfigService key `TypeTorchAnalytics`, written
|
|
309
|
-
Creator Hub (Configs)
|
|
332
|
+
**Settings** (server only, never sent to clients): the ConfigService key `TypeTorchAnalytics`, written with
|
|
333
|
+
`writeSettings()` from `@typetorch/analytics` (Open Cloud, universe:write) or in Creator Hub (Configs); no CLI command
|
|
334
|
+
writes it. Re-read live every few minutes:
|
|
310
335
|
|
|
311
336
|
```json
|
|
312
337
|
{ "backend": "basin", "events": "https://<stream-id>.ingest.cloudflare.com",
|
|
@@ -335,9 +360,13 @@ bun run build # rbxtsc --type package -> out/
|
|
|
335
360
|
kernel's and the template's `rokit.toml`). `scripts/test-generations.luau` takes a game's built payload:
|
|
336
361
|
`cd ../template && bun run payload && lune run ../framework/scripts/test-generations.luau build/payload.rbxm` boots
|
|
337
362
|
two generations in one VM and checks fresh registries, DI and generated guards, then runs the real `startServer`
|
|
338
|
-
with stub kernels (onStart failures reported to kernel 0.3.2, raised on older ones; onClose)
|
|
363
|
+
with stub kernels (onStart failures reported to kernel 0.3.2, raised on older ones; onClose), the health lines and
|
|
364
|
+
the Branch tab / "Load a build" rules (`devtools/build-actions.ts`).
|
|
339
365
|
To test framework changes before the template takes them, build the payload from a copy of the template whose
|
|
340
366
|
`node_modules/@typetorch/framework/out` is this repo's `out/`.
|
|
367
|
+
`scripts/test-state-inspect.luau` checks the state explorer's core (previews, raw reads only, cycles, paging,
|
|
368
|
+
Maps/Sets with Instance keys, the depth limit, filters, request validation) and that `src/version.ts` matches
|
|
369
|
+
package.json.
|
|
341
370
|
`scripts/test-analytics.luau` checks the analytics engine's pure parts (experiment assignment, settings, the queue
|
|
342
371
|
and HTTP budget, tt-rec-1, the sink request bodies); `test-analytics-server.luau` and `test-analytics-client.luau`
|
|
343
372
|
run the compiled server and client cores against mocked services (sessions, intake, retries, a swap with a request
|
package/out/analytics/SCHEMA.md
CHANGED
|
@@ -150,6 +150,8 @@ ConfigService key `TypeTorchAnalytics` (server-only), a JSON object (or a JSON s
|
|
|
150
150
|
| `recordShare` | 1 | 0..1, share of new players recorded (stable per pid) |
|
|
151
151
|
| `techEvery` | 60 | 15..3600 seconds between tech samples |
|
|
152
152
|
| `experiments` | none | Per experiment: `active: false` (everyone gets the first variant, not stamped), `weights` (per variant, in the game's order), `variant` (force one) |
|
|
153
|
+
| `identity` | `TypeTorchFleet` url + `/v1/identity` | basin: where identity rows go (below); none and no readable `TypeTorchFleet`: not sent. duckdb: unused |
|
|
154
|
+
| `identityToken` | the `TypeTorchFleet` token | basin: the token for `identity`. Never logged |
|
|
153
155
|
|
|
154
156
|
Unknown fields are ignored. Settings errors are warned once (never with the token).
|
|
155
157
|
|
|
@@ -158,7 +160,13 @@ Unknown fields are ignored. Settings errors are warned once (never with the toke
|
|
|
158
160
|
- **basin:** `POST <events>` with a JSON array of event rows and `POST <recordings>` with a JSON array of recording
|
|
159
161
|
rows. `Content-Type: application/json`, `Authorization: Bearer <token>` when set, no compression.
|
|
160
162
|
- **duckdb:** `POST <events>` with `{"events":[...],"recordings":[...]}`, gzip (`Content-Encoding: gzip`, HttpService
|
|
161
|
-
`Compress`), `Content-Type: application/json`, `Authorization: Bearer <token>`.
|
|
163
|
+
`Compress`), `Content-Type: application/json`, `Authorization: Bearer <token>`. With identity rows waiting, the body
|
|
164
|
+
also has `"identities":[...]`.
|
|
165
|
+
- **Identities** (engine option `identity`, default on): once a player's pid is known, one row `{ "pid", "uid", "t" }`
|
|
166
|
+
(the UserId as a number, unix ms; nothing else about the player) per session, so the dev's own server can map pids
|
|
167
|
+
and UserIds (support, Right to Erasure) and delete the link. Never part of the events table. duckdb: in the batch
|
|
168
|
+
body; basin: `POST <identity>` with `{"identities":[...]}` and `Authorization: Bearer <identityToken>` (Basin rows
|
|
169
|
+
can't be deleted, so they never go to Basin). At most 200 per request; 500 wait at most.
|
|
162
170
|
- **Batches:** every `flushSeconds`, or at 500 queued events or 20 recordings. At most 500 event rows (fleet rows
|
|
163
171
|
first) and 50 recording rows per request, about 900 KB each. At most ~10 requests a minute per server (burst 3).
|
|
164
172
|
- **Answers:** 2xx sent; 413 the batch is halved and resent (a single row is dropped); 401/403/404 rows are kept and
|
|
@@ -174,7 +182,8 @@ Unknown fields are ignored. Settings errors are warned once (never with the toke
|
|
|
174
182
|
|
|
175
183
|
DataStore `TypeTorchAnalytics`, key `p/<UserId>` -> `{ "pid": "<32 hex>", "first": <unix s>, "last": <unix s> }`.
|
|
176
184
|
One read per join (with retries), a write on the first join (UpdateAsync, keeps a pid another server wrote first) and
|
|
177
|
-
one at leave (`last`). A Right to Erasure request deletes the key, which leaves the player's rows anonymous.
|
|
185
|
+
one at leave (`last`). A Right to Erasure request deletes the key, which leaves the player's rows anonymous. The
|
|
186
|
+
analytics server also keeps pid -> UserId from identity rows (above), which its erasure webhook uses and deletes.
|
|
178
187
|
|
|
179
188
|
## Suggested Basin stream schemas
|
|
180
189
|
|
|
@@ -7,6 +7,8 @@
|
|
|
7
7
|
export declare const SCHEMA_VERSION = 1;
|
|
8
8
|
/** The ConfigService key (server-only) that holds the sink settings. */
|
|
9
9
|
export declare const SETTINGS_KEY = "TypeTorchAnalytics";
|
|
10
|
+
/** The kernel's fleet API settings `{ url, token }` (server-only ConfigService key); identities can go there. */
|
|
11
|
+
export declare const FLEET_SETTINGS_KEY = "TypeTorchFleet";
|
|
10
12
|
/** The first-session recording codec (column `codec`). */
|
|
11
13
|
export declare const RECORDING_CODEC = "tt-rec-1";
|
|
12
14
|
/** Most bytes of the `props` JSON text. Larger props become `{"_trunc":<bytes>}`. */
|
|
@@ -68,6 +70,18 @@ export interface RecordingRow {
|
|
|
68
70
|
/** Position samples in the chunk. */
|
|
69
71
|
n: number;
|
|
70
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* Which UserId a pid belongs to, sent once per session when the pid is known. Only to the dev's own server (the
|
|
75
|
+
* analytics server, or the fleet API for Basin games), never into the events table: it lets the dev map pids and
|
|
76
|
+
* UserIds (support, Right to Erasure) and delete the link.
|
|
77
|
+
*/
|
|
78
|
+
export interface IdentityRow {
|
|
79
|
+
pid: string;
|
|
80
|
+
/** The player's UserId. */
|
|
81
|
+
uid: number;
|
|
82
|
+
/** Unix ms. */
|
|
83
|
+
t: number;
|
|
84
|
+
}
|
|
71
85
|
/** Live overrides of one per-player experiment (settings key `experiments`). */
|
|
72
86
|
export interface ExperimentOverride {
|
|
73
87
|
/** false: everyone gets the first variant (control) and the experiment isn't stamped on events. */
|
|
@@ -94,6 +108,14 @@ export interface AnalyticsSettings {
|
|
|
94
108
|
techEvery?: number;
|
|
95
109
|
/** Live experiment overrides by experiment name. */
|
|
96
110
|
experiments?: Record<string, ExperimentOverride>;
|
|
111
|
+
/**
|
|
112
|
+
* basin: where identity rows go (the fleet API's `POST /v1/identity` URL). Default: the `TypeTorchFleet` key's url
|
|
113
|
+
* + /v1/identity when the server can read it; without either, identities aren't sent. duckdb: unused (they go in
|
|
114
|
+
* the batch body).
|
|
115
|
+
*/
|
|
116
|
+
identity?: string;
|
|
117
|
+
/** basin: the token for `identity` (default: the `TypeTorchFleet` token). Never logged. */
|
|
118
|
+
identityToken?: string;
|
|
97
119
|
}
|
|
98
120
|
/** AnalyticsSettings with every default filled in. */
|
|
99
121
|
export interface ResolvedSettings {
|
|
@@ -105,6 +127,8 @@ export interface ResolvedSettings {
|
|
|
105
127
|
recordShare: number;
|
|
106
128
|
techEvery: number;
|
|
107
129
|
experiments: Map<string, ExperimentOverride>;
|
|
130
|
+
identity?: string;
|
|
131
|
+
identityToken?: string;
|
|
108
132
|
}
|
|
109
133
|
export type AnalyticsValue = string | number | boolean;
|
|
110
134
|
/** Event properties: a flat JSON object (nested tables work but are harder to query). */
|
|
@@ -140,6 +164,12 @@ export interface AnalyticsOptions {
|
|
|
140
164
|
* one row per deploy report, kind "fleet". They go before analytics events. Default true.
|
|
141
165
|
*/
|
|
142
166
|
fleet?: boolean;
|
|
167
|
+
/**
|
|
168
|
+
* Server: one identity row `{ pid, uid, t }` per session once the pid is known (UserId and pid, nothing else), so
|
|
169
|
+
* the dev's own server can map them. duckdb: in the batch body (`identities`); basin: to the fleet API (Basin rows
|
|
170
|
+
* can't be deleted). Never part of the events table. Default true.
|
|
171
|
+
*/
|
|
172
|
+
identity?: boolean;
|
|
143
173
|
}
|
|
144
174
|
/** Kernel 0.3.2+ `fleetStatus()`: the server's heartbeat (sent as the props of a "fleet" / "heartbeat" row). */
|
|
145
175
|
export type FleetStatus = {
|
|
@@ -159,6 +189,8 @@ export interface AnalyticsStats {
|
|
|
159
189
|
queuedRecordings: number;
|
|
160
190
|
/** Fleet rows waiting (they go first). */
|
|
161
191
|
queuedFleet: number;
|
|
192
|
+
/** Identity rows waiting (pid -> UserId, for the dev's own server). */
|
|
193
|
+
queuedIdentities: number;
|
|
162
194
|
/** Rows sent since the server started (across swaps). */
|
|
163
195
|
sent: number;
|
|
164
196
|
/** Rows dropped because the queue was full. */
|
|
@@ -10,12 +10,21 @@
|
|
|
10
10
|
local SCHEMA_VERSION = 1
|
|
11
11
|
--* The ConfigService key (server-only) that holds the sink settings.
|
|
12
12
|
local SETTINGS_KEY = "TypeTorchAnalytics"
|
|
13
|
+
--* The kernel's fleet API settings `{ url, token }` (server-only ConfigService key); identities can go there.
|
|
14
|
+
local FLEET_SETTINGS_KEY = "TypeTorchFleet"
|
|
13
15
|
--* The first-session recording codec (column `codec`).
|
|
14
16
|
local RECORDING_CODEC = "tt-rec-1"
|
|
15
17
|
--* Most bytes of the `props` JSON text. Larger props become `{"_trunc":<bytes>}`.
|
|
16
18
|
local PROPS_MAX_BYTES = 4096
|
|
17
19
|
--* One row of the events table. Flat; every column is always present.
|
|
18
20
|
--* One row of the recordings table: one packed chunk of a player's first-ever session.
|
|
21
|
+
--[[
|
|
22
|
+
*
|
|
23
|
+
* Which UserId a pid belongs to, sent once per session when the pid is known. Only to the dev's own server (the
|
|
24
|
+
* analytics server, or the fleet API for Basin games), never into the events table: it lets the dev map pids and
|
|
25
|
+
* UserIds (support, Right to Erasure) and delete the link.
|
|
26
|
+
|
|
27
|
+
]]
|
|
19
28
|
--* Live overrides of one per-player experiment (settings key `experiments`).
|
|
20
29
|
--* The sink settings: ConfigService key `TypeTorchAnalytics` (server-only), or `new AnalyticsEngine({ settings })`.
|
|
21
30
|
--* AnalyticsSettings with every default filled in.
|
|
@@ -27,6 +36,7 @@ local PROPS_MAX_BYTES = 4096
|
|
|
27
36
|
return {
|
|
28
37
|
SCHEMA_VERSION = SCHEMA_VERSION,
|
|
29
38
|
SETTINGS_KEY = SETTINGS_KEY,
|
|
39
|
+
FLEET_SETTINGS_KEY = FLEET_SETTINGS_KEY,
|
|
30
40
|
RECORDING_CODEC = RECORDING_CODEC,
|
|
31
41
|
PROPS_MAX_BYTES = PROPS_MAX_BYTES,
|
|
32
42
|
}
|
|
@@ -63,6 +63,8 @@ export declare class ServerAnalytics {
|
|
|
63
63
|
private settingsErrors;
|
|
64
64
|
/** The first settings read finished (found or not), so experiment overrides are as known as they get. */
|
|
65
65
|
private settingsTried;
|
|
66
|
+
/** The `TypeTorchFleet` value (Basin games send identities to the fleet API). */
|
|
67
|
+
private fleetSettings?;
|
|
66
68
|
private nextFlushAt;
|
|
67
69
|
private backoffUntil;
|
|
68
70
|
private batchRows;
|
|
@@ -128,6 +130,11 @@ export declare class ServerAnalytics {
|
|
|
128
130
|
private post;
|
|
129
131
|
/** Sends one batch per table. `final`: shutdown, ignore the budget. */
|
|
130
132
|
private flushOnce;
|
|
133
|
+
private identityList;
|
|
134
|
+
private queueIdentity;
|
|
135
|
+
private takeIdentities;
|
|
136
|
+
/** Puts identity rows back in front (a failed or unsent request), keeping the cap. */
|
|
137
|
+
private requeueIdentities;
|
|
131
138
|
private startFlushing;
|
|
132
139
|
/** game code's flush(): send now (within the budget), on its own thread. */
|
|
133
140
|
flushSoon(): void;
|
|
@@ -39,10 +39,12 @@ local newQueueState = _queue.newQueueState
|
|
|
39
39
|
local RowQueue = _queue.RowQueue
|
|
40
40
|
local takeToken = _queue.takeToken
|
|
41
41
|
local _schema = TS.import(script, script.Parent, "schema")
|
|
42
|
+
local FLEET_SETTINGS_KEY = _schema.FLEET_SETTINGS_KEY
|
|
42
43
|
local RECORDING_CODEC = _schema.RECORDING_CODEC
|
|
43
44
|
local SCHEMA_VERSION = _schema.SCHEMA_VERSION
|
|
44
45
|
local SETTINGS_KEY = _schema.SETTINGS_KEY
|
|
45
46
|
local _settings = TS.import(script, script.Parent, "settings")
|
|
47
|
+
local identityTarget = _settings.identityTarget
|
|
46
48
|
local parseSettings = _settings.parseSettings
|
|
47
49
|
local recordsSupported = _settings.recordsSupported
|
|
48
50
|
local DEFAULT_TECH_EVERY = _settings.DEFAULT_TECH_EVERY
|
|
@@ -77,6 +79,9 @@ local BATCH_BYTES = 900000
|
|
|
77
79
|
--* HTTP budget: a burst of 3, refilled at 10 a minute (Roblox: 500 a minute per server for everything).
|
|
78
80
|
local HTTP_BURST = 3
|
|
79
81
|
local HTTP_PER_MINUTE = 10
|
|
82
|
+
--* Identity rows (pid -> UserId) waiting; past this the oldest go.
|
|
83
|
+
local IDENTITIES_CAP = 500
|
|
84
|
+
local BATCH_IDENTITIES = 200
|
|
80
85
|
--* Rows kept per player while their pid loads.
|
|
81
86
|
local PENDING_MAX = 200
|
|
82
87
|
--* Client events: burst and refill per minute, and the most per session.
|
|
@@ -165,6 +170,8 @@ local function newStore()
|
|
|
165
170
|
fleetSeen = {},
|
|
166
171
|
inflightEvents = {},
|
|
167
172
|
inflightRecordings = {},
|
|
173
|
+
identities = {},
|
|
174
|
+
inflightIdentities = {},
|
|
168
175
|
budget = {
|
|
169
176
|
tokens = HTTP_BURST,
|
|
170
177
|
last = os.clock(),
|
|
@@ -235,6 +242,15 @@ do
|
|
|
235
242
|
if #self.store.inflightRecordings > 0 then
|
|
236
243
|
self.recordings:requeue(self.store.inflightRecordings)
|
|
237
244
|
end
|
|
245
|
+
-- Stores from before identities existed (a hot swap from an older framework) have neither list.
|
|
246
|
+
if self.store.identities == nil then
|
|
247
|
+
self.store.identities = {}
|
|
248
|
+
end
|
|
249
|
+
local handedOver = self.store.inflightIdentities or {}
|
|
250
|
+
if #handedOver > 0 then
|
|
251
|
+
self:requeueIdentities(handedOver)
|
|
252
|
+
end
|
|
253
|
+
self.store.inflightIdentities = {}
|
|
238
254
|
self.store.inflightEvents = {}
|
|
239
255
|
self.store.inflightRecordings = {}
|
|
240
256
|
trove:add(function()
|
|
@@ -440,7 +456,7 @@ do
|
|
|
440
456
|
end
|
|
441
457
|
function ServerAnalytics:track(player, kind, name, props)
|
|
442
458
|
if not isEventName(name) then
|
|
443
|
-
warn("[src/analytics/server.ts:
|
|
459
|
+
warn("[src/analytics/server.ts:435]", `[analytics] {kind} event name must be 1-64 characters: {string.sub(tostring(name), 1, 80)}`)
|
|
444
460
|
return nil
|
|
445
461
|
end
|
|
446
462
|
local session = if player then self:sessionOf(player) else nil
|
|
@@ -548,7 +564,7 @@ do
|
|
|
548
564
|
end
|
|
549
565
|
local control = _condition
|
|
550
566
|
if not isExperimentName(name) or not isVariantList(variants) then
|
|
551
|
-
warn("[src/analytics/server.ts:
|
|
567
|
+
warn("[src/analytics/server.ts:486]", `[analytics] experiment("{string.sub(tostring(name), 1, 64)}"): bad name or variants (1-16 distinct names)`)
|
|
552
568
|
return control
|
|
553
569
|
end
|
|
554
570
|
local session = self:sessionOf(player)
|
|
@@ -597,7 +613,7 @@ do
|
|
|
597
613
|
function ServerAnalytics:applySettings(parsed)
|
|
598
614
|
local errorsText = table.concat(parsed.errors, "; ")
|
|
599
615
|
if errorsText ~= table.concat(self.settingsErrors, "; ") and errorsText ~= "" then
|
|
600
|
-
warn("[src/analytics/server.ts:
|
|
616
|
+
warn("[src/analytics/server.ts:517]", `[analytics] {SETTINGS_KEY}: {errorsText}`)
|
|
601
617
|
end
|
|
602
618
|
self.settingsErrors = parsed.errors
|
|
603
619
|
local hadSettings = self.settings ~= nil
|
|
@@ -629,6 +645,12 @@ do
|
|
|
629
645
|
if not snapshot or (missing and not force) then
|
|
630
646
|
return nil
|
|
631
647
|
end
|
|
648
|
+
local fleetOk, fleetValue = pcall(function()
|
|
649
|
+
return snapshot:GetValue(FLEET_SETTINGS_KEY)
|
|
650
|
+
end)
|
|
651
|
+
if fleetOk then
|
|
652
|
+
self.fleetSettings = fleetValue
|
|
653
|
+
end
|
|
632
654
|
local ok, value = pcall(function()
|
|
633
655
|
return snapshot:GetValue(SETTINGS_KEY)
|
|
634
656
|
end)
|
|
@@ -668,7 +690,7 @@ do
|
|
|
668
690
|
snapshot = result
|
|
669
691
|
else
|
|
670
692
|
if attempt == 1 then
|
|
671
|
-
warn("[src/analytics/server.ts:
|
|
693
|
+
warn("[src/analytics/server.ts:563]", `[analytics] ConfigService unavailable: {string.sub(tostring(result), 1, 120)}`)
|
|
672
694
|
end
|
|
673
695
|
task.wait(delay)
|
|
674
696
|
delay = math.min(delay * 2, 30)
|
|
@@ -856,6 +878,9 @@ do
|
|
|
856
878
|
session.first = first
|
|
857
879
|
session.ret = ret
|
|
858
880
|
session.ready = true
|
|
881
|
+
if self.options.identity ~= false then
|
|
882
|
+
self:queueIdentity(player.UserId, pid)
|
|
883
|
+
end
|
|
859
884
|
for _, row in session.pending do
|
|
860
885
|
row.pid = session.pid
|
|
861
886
|
row.newp = session.newp
|
|
@@ -1355,7 +1380,7 @@ do
|
|
|
1355
1380
|
end
|
|
1356
1381
|
end)
|
|
1357
1382
|
if not ok then
|
|
1358
|
-
warn("[src/analytics/server.ts:
|
|
1383
|
+
warn("[src/analytics/server.ts:1008]", `[analytics] client message failed: {string.sub(tostring(err), 1, 200)}`)
|
|
1359
1384
|
end
|
|
1360
1385
|
end)
|
|
1361
1386
|
self.trove:add(function()
|
|
@@ -1635,7 +1660,7 @@ do
|
|
|
1635
1660
|
return nil
|
|
1636
1661
|
end
|
|
1637
1662
|
self.lastWarnAt = now
|
|
1638
|
-
warn("[src/analytics/server.ts:
|
|
1663
|
+
warn("[src/analytics/server.ts:1197]", `[analytics] {message}`)
|
|
1639
1664
|
end
|
|
1640
1665
|
function ServerAnalytics:post(request)
|
|
1641
1666
|
local ok, response = pcall(function()
|
|
@@ -1672,7 +1697,12 @@ do
|
|
|
1672
1697
|
end
|
|
1673
1698
|
end
|
|
1674
1699
|
local recordings = if recordsSupported(settings) then self.recordings:take(BATCH_RECORDINGS, BATCH_BYTES, estimateRecordingBytes) else {}
|
|
1675
|
-
|
|
1700
|
+
local waiting = self.store.identities or {}
|
|
1701
|
+
local identities = if #waiting > 0 then self:takeIdentities() else {}
|
|
1702
|
+
-- Basin without a fleet API to send them to: identities are dropped (they never go into Basin).
|
|
1703
|
+
local target = if settings.backend == "basin" and #identities > 0 then identityTarget(settings, self.fleetSettings, decode) else nil
|
|
1704
|
+
local sendIdentities = if settings.backend == "duckdb" or target ~= nil then identities else {}
|
|
1705
|
+
if #events == 0 and #recordings == 0 and #sendIdentities == 0 then
|
|
1676
1706
|
self.flushing = false
|
|
1677
1707
|
return nil
|
|
1678
1708
|
end
|
|
@@ -1680,7 +1710,8 @@ do
|
|
|
1680
1710
|
-- to the next generation (sent again: at least once).
|
|
1681
1711
|
self.store.inflightEvents = events
|
|
1682
1712
|
self.store.inflightRecordings = recordings
|
|
1683
|
-
|
|
1713
|
+
self.store.inflightIdentities = sendIdentities
|
|
1714
|
+
local requests = buildRequests(settings, events, recordings, quote, sendIdentities, target)
|
|
1684
1715
|
local settle = function(request, giveBack)
|
|
1685
1716
|
if request.events > 0 then
|
|
1686
1717
|
if giveBack then
|
|
@@ -1694,6 +1725,12 @@ do
|
|
|
1694
1725
|
end
|
|
1695
1726
|
self.store.inflightRecordings = {}
|
|
1696
1727
|
end
|
|
1728
|
+
if request.identities > 0 then
|
|
1729
|
+
if giveBack then
|
|
1730
|
+
self:requeueIdentities(sendIdentities)
|
|
1731
|
+
end
|
|
1732
|
+
self.store.inflightIdentities = {}
|
|
1733
|
+
end
|
|
1697
1734
|
end
|
|
1698
1735
|
local failed = false
|
|
1699
1736
|
for _, request in requests do
|
|
@@ -1703,9 +1740,10 @@ do
|
|
|
1703
1740
|
end
|
|
1704
1741
|
local status = self:post(request)
|
|
1705
1742
|
local outcome = classifyStatus(status)
|
|
1706
|
-
local rows = request.events + request.recordings
|
|
1743
|
+
local rows = request.events + request.recordings + request.identities
|
|
1707
1744
|
if outcome == "sent" then
|
|
1708
|
-
|
|
1745
|
+
-- Identity rows aren't analytics rows: "sent" counts events and recordings.
|
|
1746
|
+
self.store.sent += request.events + request.recordings
|
|
1709
1747
|
self.store.failures = 0
|
|
1710
1748
|
self.batchRows = BATCH_ROWS
|
|
1711
1749
|
settle(request, false)
|
|
@@ -1715,33 +1753,77 @@ do
|
|
|
1715
1753
|
elseif outcome == "drop" or outcome == "split" then
|
|
1716
1754
|
self.store.rejected += rows
|
|
1717
1755
|
settle(request, false)
|
|
1718
|
-
self:warn(`
|
|
1756
|
+
self:warn(`analytics upload refused {rows} rows (HTTP {status}); they were dropped`)
|
|
1719
1757
|
else
|
|
1720
1758
|
settle(request, true)
|
|
1721
1759
|
failed = true
|
|
1722
1760
|
self.store.failures += 1
|
|
1723
1761
|
local wait = if outcome == "config" then 300 else backoffSeconds(self.store.failures, math.random())
|
|
1724
1762
|
self.backoffUntil = os.clock() + wait
|
|
1725
|
-
|
|
1763
|
+
-- "both" = one batch with events and recordings (the DuckDB sink); say "analytics upload" instead.
|
|
1764
|
+
local what = if request.table == "both" then "analytics upload" else `analytics upload ({request.table})`
|
|
1726
1765
|
local _result
|
|
1727
|
-
if
|
|
1728
|
-
_result = `
|
|
1766
|
+
if status > 0 then
|
|
1767
|
+
_result = `HTTP {status}`
|
|
1729
1768
|
else
|
|
1730
|
-
local _exp = request.table
|
|
1731
1769
|
local _condition = self.store.lastError
|
|
1732
1770
|
if _condition == nil then
|
|
1733
|
-
_condition =
|
|
1771
|
+
_condition = "no response"
|
|
1734
1772
|
end
|
|
1735
|
-
_result =
|
|
1773
|
+
_result = _condition
|
|
1736
1774
|
end
|
|
1737
|
-
|
|
1775
|
+
local reason = _result
|
|
1776
|
+
local fix = if status == 530 then ": the tunnel is down, run bun run local on the dev PC" else ""
|
|
1777
|
+
self:warn(if outcome == "config" then `{what} got HTTP {status}: check the URL and token in {SETTINGS_KEY}; retrying in {math.floor(wait)} s` else `{what} failed ({reason}{fix}); retrying in {math.floor(wait)} s`)
|
|
1738
1778
|
end
|
|
1739
1779
|
end
|
|
1740
1780
|
-- Recordings without a request (basin without a recordings URL) were never taken; nothing else is left.
|
|
1741
1781
|
self.store.inflightEvents = {}
|
|
1742
1782
|
self.store.inflightRecordings = {}
|
|
1783
|
+
self.store.inflightIdentities = {}
|
|
1743
1784
|
self.flushing = false
|
|
1744
1785
|
end
|
|
1786
|
+
function ServerAnalytics:identityList()
|
|
1787
|
+
if self.store.identities == nil then
|
|
1788
|
+
self.store.identities = {}
|
|
1789
|
+
end
|
|
1790
|
+
return self.store.identities
|
|
1791
|
+
end
|
|
1792
|
+
function ServerAnalytics:queueIdentity(userId, pid)
|
|
1793
|
+
local list = self:identityList()
|
|
1794
|
+
local _arg0 = {
|
|
1795
|
+
pid = pid,
|
|
1796
|
+
uid = userId,
|
|
1797
|
+
t = nowMs(),
|
|
1798
|
+
}
|
|
1799
|
+
table.insert(list, _arg0)
|
|
1800
|
+
while #list > IDENTITIES_CAP do
|
|
1801
|
+
table.remove(list, 1)
|
|
1802
|
+
end
|
|
1803
|
+
end
|
|
1804
|
+
function ServerAnalytics:takeIdentities()
|
|
1805
|
+
local list = self:identityList()
|
|
1806
|
+
local taken = {}
|
|
1807
|
+
while #list > 0 and #taken < BATCH_IDENTITIES do
|
|
1808
|
+
local _arg0 = table.remove(list, 1)
|
|
1809
|
+
table.insert(taken, _arg0)
|
|
1810
|
+
end
|
|
1811
|
+
return taken
|
|
1812
|
+
end
|
|
1813
|
+
function ServerAnalytics:requeueIdentities(rows)
|
|
1814
|
+
local list = self:identityList()
|
|
1815
|
+
local _array = {}
|
|
1816
|
+
local _length = #_array
|
|
1817
|
+
table.move(rows, 1, #rows, _length + 1, _array)
|
|
1818
|
+
local merged = _array
|
|
1819
|
+
for _, row in list do
|
|
1820
|
+
table.insert(merged, row)
|
|
1821
|
+
end
|
|
1822
|
+
while #merged > IDENTITIES_CAP do
|
|
1823
|
+
table.remove(merged, 1)
|
|
1824
|
+
end
|
|
1825
|
+
self.store.identities = merged
|
|
1826
|
+
end
|
|
1745
1827
|
function ServerAnalytics:startFlushing()
|
|
1746
1828
|
self.nextFlushAt = os.clock() + 5
|
|
1747
1829
|
self.trove:add(task.spawn(function()
|
|
@@ -1843,17 +1925,27 @@ do
|
|
|
1843
1925
|
_object.queued = self.events:size()
|
|
1844
1926
|
_object.queuedRecordings = self.recordings:size()
|
|
1845
1927
|
_object.queuedFleet = self.fleet:size()
|
|
1928
|
+
local _left_1 = "queuedIdentities"
|
|
1929
|
+
local _result_1 = self.store.identities
|
|
1930
|
+
if _result_1 ~= nil then
|
|
1931
|
+
_result_1 = #_result_1
|
|
1932
|
+
end
|
|
1933
|
+
local _condition = _result_1
|
|
1934
|
+
if _condition == nil then
|
|
1935
|
+
_condition = 0
|
|
1936
|
+
end
|
|
1937
|
+
_object[_left_1] = _condition
|
|
1846
1938
|
_object.sent = self.store.sent
|
|
1847
1939
|
_object.dropped = self.store.events.dropped + self.store.recordings.dropped + self.store.fleet.dropped
|
|
1848
1940
|
_object.rejected = self.store.rejected
|
|
1849
1941
|
_object.failures = self.store.failures
|
|
1850
1942
|
_object.lastError = self.store.lastError
|
|
1851
|
-
local
|
|
1943
|
+
local _left_2 = "settingsErrors"
|
|
1852
1944
|
local _array = {}
|
|
1853
1945
|
local _length = #_array
|
|
1854
1946
|
local _array_1 = self.settingsErrors
|
|
1855
1947
|
table.move(_array_1, 1, #_array_1, _length + 1, _array)
|
|
1856
|
-
_object[
|
|
1948
|
+
_object[_left_2] = _array
|
|
1857
1949
|
return _object
|
|
1858
1950
|
end
|
|
1859
1951
|
end
|
|
@@ -18,5 +18,15 @@ export interface ParsedSettings {
|
|
|
18
18
|
* no settings and no errors. Unknown fields are ignored.
|
|
19
19
|
*/
|
|
20
20
|
export declare function parseSettings(raw: unknown, decode: (text: string) => unknown): ParsedSettings;
|
|
21
|
+
/** Where identity rows go on a Basin game: the fleet API (`TypeTorchFleet` = { url, token }) at /v1/identity. */
|
|
22
|
+
export interface IdentityTarget {
|
|
23
|
+
url: string;
|
|
24
|
+
token?: string;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The identity target for these settings: duckdb none (identities ride in the batch body); basin the `identity` URL,
|
|
28
|
+
* else the fleet API's /v1/identity from the `TypeTorchFleet` value (a table or JSON `{ url, token }`), else none.
|
|
29
|
+
*/
|
|
30
|
+
export declare function identityTarget(settings: ResolvedSettings, fleetRaw: unknown, decode: (text: string) => unknown): IdentityTarget | undefined;
|
|
21
31
|
/** Whether recordings can be sent at all: duckdb always (one body), basin only with a recordings stream URL. */
|
|
22
32
|
export declare function recordsSupported(settings: ResolvedSettings): boolean;
|