@uniquedivine/tg-sai 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,471 @@
1
+ import { Database, type Statement } from "bun:sqlite"
2
+ import { createHash } from "node:crypto"
3
+ import { chmodSync, mkdirSync } from "node:fs"
4
+ import { hostname } from "node:os"
5
+ import { dirname, join } from "node:path"
6
+
7
+ import { configDir, type ProfileStoreEnv } from "../profiles"
8
+ // Embedded at bundle/run time so the published package carries the schema.
9
+ import eventMigrationSql from "./migrations/0001-event.sql" with {
10
+ type: "text",
11
+ }
12
+ import eventJobIndexSql from "./migrations/0002-event-job-index.sql" with {
13
+ type: "text",
14
+ }
15
+ import eventLogSql from "./migrations/0003-event-log.sql" with {
16
+ type: "text",
17
+ }
18
+
19
+ export const TELEMETRY_DB_FILE = "telemetry.db"
20
+ export const TELEMETRY_ENV = "TG_SAI_TELEMETRY"
21
+
22
+ /** Values of the `event` column. */
23
+ export const CALL_EVENT = "tg.rpc.call"
24
+ export const ATTEMPT_EVENT = "tg.rpc.attempt"
25
+ export const FLOOD_WAIT_EVENT = "tg.flood.wait"
26
+ export const RUN_START_EVENT = "run.start"
27
+ export const RUN_END_EVENT = "run.end"
28
+ export const WORKER_TICK_EVENT = "worker.tick"
29
+ /** Default prefix for app log rows: `log.<subsystem>`. */
30
+ export const LOG_EVENT_PREFIX = "log."
31
+
32
+ /** Values of the `level` column; only app log rows set it. */
33
+ export const LOG_LEVELS = ["debug", "info", "warn", "error"] as const
34
+ export type LogLevel = (typeof LOG_LEVELS)[number]
35
+
36
+ export type TelemetryOutcome =
37
+ | "started"
38
+ | "ok"
39
+ | "error"
40
+ | "timeout"
41
+ | "aborted"
42
+
43
+ export type TelemetrySubsystem = "worker" | "health" | "cli"
44
+
45
+ /**
46
+ * One telemetry row. `req` and `res` must already be summaries (see
47
+ * `summarize.ts`); the sink only serializes them.
48
+ */
49
+ export interface TelemetryEvent {
50
+ eventId?: string
51
+ requestId?: string | null
52
+ runId: string
53
+ jobId?: string | null
54
+ tsMs?: number
55
+ durationMs?: number | null
56
+ event: string
57
+ subsystem: TelemetrySubsystem | string
58
+ command?: string | null
59
+ accountId?: number | null
60
+ profile?: string | null
61
+ method?: string | null
62
+ attempt?: number | null
63
+ outcome: TelemetryOutcome
64
+ errorCode?: number | null
65
+ err?: string | null
66
+ waitS?: number | null
67
+ req?: unknown
68
+ res?: unknown
69
+ host?: string | null
70
+ pid?: number | null
71
+ version?: string | null
72
+ /** Set only on app log rows (see `run.ts` `log`). */
73
+ level?: LogLevel | null
74
+ /** Short constant text written by tg-sai code; never user data. */
75
+ msg?: string | null
76
+ /** Log attributes; must already be a summary, like `req`/`res`. */
77
+ attrs?: unknown
78
+ }
79
+
80
+ /** Row shape as stored in the `event` table. */
81
+ export interface EventRow {
82
+ event_id: string
83
+ request_id: string | null
84
+ run_id: string
85
+ job_id: string | null
86
+ ts_ms: number
87
+ duration_ms: number | null
88
+ event: string
89
+ subsystem: string
90
+ command: string | null
91
+ account_id: number | null
92
+ profile: string | null
93
+ method: string | null
94
+ attempt: number | null
95
+ outcome: TelemetryOutcome
96
+ error_code: number | null
97
+ err: string | null
98
+ wait_s: number | null
99
+ req: string | null
100
+ res: string | null
101
+ host: string | null
102
+ pid: number | null
103
+ version: string | null
104
+ level: LogLevel | null
105
+ msg: string | null
106
+ attrs: string | null
107
+ }
108
+
109
+ export interface Migration {
110
+ filename: string
111
+ sql: string
112
+ }
113
+
114
+ export interface MigrationResult {
115
+ applied: string[]
116
+ pending: string[]
117
+ }
118
+
119
+ export const MIGRATIONS: readonly Migration[] = [
120
+ { filename: "0001-event.sql", sql: eventMigrationSql },
121
+ { filename: "0002-event-job-index.sql", sql: eventJobIndexSql },
122
+ { filename: "0003-event-log.sql", sql: eventLogSql },
123
+ ]
124
+
125
+ const HISTORY_SQL = `CREATE TABLE IF NOT EXISTS schema_migrations (
126
+ filename TEXT PRIMARY KEY,
127
+ checksum TEXT NOT NULL,
128
+ applied_at INTEGER NOT NULL
129
+ )`
130
+
131
+ export function migrationChecksum(sql: string): string {
132
+ return createHash("sha256").update(sql).digest("hex")
133
+ }
134
+
135
+ export function telemetryDbPath(env: ProfileStoreEnv = process.env): string {
136
+ return join(configDir(env), TELEMETRY_DB_FILE)
137
+ }
138
+
139
+ export function telemetryDisabled(
140
+ env: ProfileStoreEnv = process.env,
141
+ ): boolean {
142
+ return env[TELEMETRY_ENV]?.trim().toLowerCase() === "off"
143
+ }
144
+
145
+ /**
146
+ * Returns a time-ordered UUID (RFC 9562 v7). Uses `Bun.randomUUIDv7` when
147
+ * the runtime provides it.
148
+ */
149
+ export function uuidv7(nowMs: number = Date.now()): string {
150
+ const bun = (globalThis as { Bun?: { randomUUIDv7?: unknown } }).Bun
151
+ if (typeof bun?.randomUUIDv7 === "function") {
152
+ return (bun.randomUUIDv7 as (encoding: "hex", ts: number) => string)(
153
+ "hex",
154
+ nowMs,
155
+ )
156
+ }
157
+ const bytes = new Uint8Array(16)
158
+ crypto.getRandomValues(bytes)
159
+ let ts = Math.floor(nowMs)
160
+ for (let i = 5; i >= 0; i -= 1) {
161
+ bytes[i] = ts % 256
162
+ ts = Math.floor(ts / 256)
163
+ }
164
+ bytes[6] = ((bytes[6] ?? 0) & 0x0f) | 0x70
165
+ bytes[8] = ((bytes[8] ?? 0) & 0x3f) | 0x80
166
+ const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("")
167
+ return [
168
+ hex.slice(0, 8),
169
+ hex.slice(8, 12),
170
+ hex.slice(12, 16),
171
+ hex.slice(16, 20),
172
+ hex.slice(20),
173
+ ].join("-")
174
+ }
175
+
176
+ /**
177
+ * Applies pending migrations under `BEGIN IMMEDIATE`. Refuses an applied
178
+ * migration whose checksum changed, an unknown applied migration, or a
179
+ * history that is not a prefix of `migrations`.
180
+ */
181
+ export function applyMigrations(
182
+ db: Database,
183
+ migrations: readonly Migration[] = MIGRATIONS,
184
+ nowMs: number = Date.now(),
185
+ ): MigrationResult {
186
+ db.run("BEGIN IMMEDIATE")
187
+ try {
188
+ db.run(HISTORY_SQL)
189
+ const rows = db
190
+ .query<{ filename: string; checksum: string }, []>(
191
+ "SELECT filename, checksum FROM schema_migrations ORDER BY filename",
192
+ )
193
+ .all()
194
+ rows.forEach((row, i) => {
195
+ const expected = migrations[i]
196
+ if (!expected || expected.filename !== row.filename) {
197
+ throw new Error(`Unknown or out-of-order migration: ${row.filename}`)
198
+ }
199
+ if (migrationChecksum(expected.sql) !== row.checksum) {
200
+ throw new Error(`Migration checksum mismatch: ${row.filename}`)
201
+ }
202
+ })
203
+ const applied = rows.map((row) => row.filename)
204
+ const pending = migrations.slice(rows.length)
205
+ const insert = db.prepare(
206
+ "INSERT INTO schema_migrations (filename, checksum, applied_at) " +
207
+ "VALUES (?, ?, ?)",
208
+ )
209
+ for (const migration of pending) {
210
+ db.run(migration.sql)
211
+ insert.run(migration.filename, migrationChecksum(migration.sql), nowMs)
212
+ }
213
+ db.run("COMMIT")
214
+ return { applied, pending: pending.map((m) => m.filename) }
215
+ } catch (error) {
216
+ try {
217
+ db.run("ROLLBACK")
218
+ } catch {
219
+ // The failed statement may already have ended the transaction.
220
+ }
221
+ throw error
222
+ }
223
+ }
224
+
225
+ export const BUSY_TIMEOUT_MS = 5000
226
+
227
+ function isBusyError(error: unknown): boolean {
228
+ const code = (error as { code?: unknown } | null)?.code
229
+ return typeof code === "string" && code.startsWith("SQLITE_BUSY")
230
+ }
231
+
232
+ function journalMode(db: Database, sql: string): string {
233
+ const row = db.query<{ journal_mode: string }, []>(sql).get()
234
+ return row?.journal_mode?.toLowerCase() ?? ""
235
+ }
236
+
237
+ /**
238
+ * Switches the database to WAL, retrying on SQLITE_BUSY until `timeoutMs`.
239
+ *
240
+ * Changing the journal mode needs an exclusive lock, and SQLite returns
241
+ * SQLITE_BUSY for it without consulting the busy handler. Two processes
242
+ * opening a fresh database together (e.g. the worker and its health check)
243
+ * otherwise race here. Once one has switched, the mode is persistent, so the
244
+ * cheap read-only check short-circuits every later open.
245
+ */
246
+ export function enableWal(
247
+ db: Database,
248
+ timeoutMs: number = BUSY_TIMEOUT_MS,
249
+ ): void {
250
+ const deadline = Date.now() + timeoutMs
251
+ for (let attempt = 0; ; attempt += 1) {
252
+ try {
253
+ const current = journalMode(db, "PRAGMA journal_mode")
254
+ // In-memory databases cannot use WAL; they have no other process.
255
+ if (current === "wal" || current === "memory") return
256
+ const mode = journalMode(db, "PRAGMA journal_mode = WAL")
257
+ if (mode === "wal") return
258
+ throw new Error(`journal_mode stayed ${mode || "unknown"}`)
259
+ } catch (error) {
260
+ if (!isBusyError(error) || Date.now() >= deadline) throw error
261
+ Bun.sleepSync(Math.min(5 * 2 ** attempt, 100) + Math.random() * 10)
262
+ }
263
+ }
264
+ }
265
+
266
+ export function applyPragmas(db: Database): void {
267
+ db.run(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`)
268
+ enableWal(db)
269
+ db.run("PRAGMA foreign_keys = ON")
270
+ db.run("PRAGMA synchronous = NORMAL")
271
+ }
272
+
273
+ /**
274
+ * Opens (creating if needed) the telemetry database at `path`, sets pragmas,
275
+ * and applies migrations. Throws on failure; callers that must not fail
276
+ * should use `openTelemetrySink`.
277
+ */
278
+ export function openTelemetryDb(
279
+ path: string,
280
+ migrations: readonly Migration[] = MIGRATIONS,
281
+ ): Database {
282
+ if (path !== ":memory:") {
283
+ mkdirSync(dirname(path), { recursive: true, mode: 0o700 })
284
+ }
285
+ const db = new Database(path, { create: true, strict: true })
286
+ try {
287
+ applyPragmas(db)
288
+ applyMigrations(db, migrations)
289
+ if (path !== ":memory:") restrictToOwner(path)
290
+ return db
291
+ } catch (error) {
292
+ db.close()
293
+ throw error
294
+ }
295
+ }
296
+
297
+ /** Makes the DB and its WAL files owner-only; best effort. */
298
+ function restrictToOwner(path: string): void {
299
+ for (const file of [path, `${path}-wal`, `${path}-shm`]) {
300
+ try {
301
+ chmodSync(file, 0o600)
302
+ } catch {
303
+ // Missing WAL files or a read-only filesystem are fine.
304
+ }
305
+ }
306
+ }
307
+
308
+ export interface TelemetrySink {
309
+ /** Records one event. Never throws. */
310
+ record(event: TelemetryEvent): void
311
+ /** Number of events that failed to record. */
312
+ dropped(): number
313
+ close(): void
314
+ }
315
+
316
+ export interface SinkDefaults {
317
+ host?: string
318
+ pid?: number
319
+ version?: string
320
+ }
321
+
322
+ export type StderrWriter = (line: string) => void
323
+
324
+ const defaultStderr: StderrWriter = (line) => {
325
+ try {
326
+ process.stderr.write(`${line}\n`)
327
+ } catch {
328
+ // Nothing left to report to.
329
+ }
330
+ }
331
+
332
+ function jsonOrNull(value: unknown): string | null {
333
+ if (value === undefined || value === null) return null
334
+ return JSON.stringify(value)
335
+ }
336
+
337
+ /** Maps an event plus sink defaults to the positional insert parameters. */
338
+ export function eventToParams(
339
+ event: TelemetryEvent,
340
+ defaults: SinkDefaults = {},
341
+ ): (string | number | null)[] {
342
+ const tsMs = event.tsMs ?? Date.now()
343
+ return [
344
+ event.eventId ?? uuidv7(tsMs),
345
+ event.requestId ?? null,
346
+ event.runId,
347
+ event.jobId ?? null,
348
+ tsMs,
349
+ event.durationMs ?? null,
350
+ event.event,
351
+ event.subsystem,
352
+ event.command ?? null,
353
+ event.accountId ?? null,
354
+ event.profile ?? null,
355
+ event.method ?? null,
356
+ event.attempt ?? null,
357
+ event.outcome,
358
+ event.errorCode ?? null,
359
+ event.err ?? null,
360
+ event.waitS ?? null,
361
+ jsonOrNull(event.req),
362
+ jsonOrNull(event.res),
363
+ event.host ?? defaults.host ?? null,
364
+ event.pid ?? defaults.pid ?? null,
365
+ event.version ?? defaults.version ?? null,
366
+ event.level ?? null,
367
+ event.msg ?? null,
368
+ jsonOrNull(event.attrs),
369
+ ]
370
+ }
371
+
372
+ const INSERT_SQL = `INSERT INTO event (
373
+ event_id, request_id, run_id, job_id, ts_ms, duration_ms, event, subsystem,
374
+ command, account_id, profile, method, attempt, outcome, error_code, err,
375
+ wait_s, req, res, host, pid, version, level, msg, attrs
376
+ ) VALUES (
377
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
378
+ )`
379
+
380
+ export class SqliteTelemetrySink implements TelemetrySink {
381
+ private droppedCount = 0
382
+ private insert: Statement | null = null
383
+ private readonly defaults: SinkDefaults
384
+
385
+ constructor(
386
+ private readonly db: Database,
387
+ defaults: SinkDefaults = {},
388
+ private readonly stderr: StderrWriter = defaultStderr,
389
+ ) {
390
+ this.defaults = {
391
+ host: defaults.host ?? safeHostname(),
392
+ pid: defaults.pid ?? process.pid,
393
+ version: defaults.version,
394
+ }
395
+ }
396
+
397
+ record(event: TelemetryEvent): void {
398
+ try {
399
+ this.insert ??= this.db.prepare(INSERT_SQL)
400
+ this.insert.run(...eventToParams(event, this.defaults))
401
+ } catch (error) {
402
+ this.droppedCount += 1
403
+ this.stderr(
404
+ `tg-sai telemetry: dropped ${String(event?.event)} event ` +
405
+ `(${this.droppedCount} dropped): ${errorMessage(error)}`,
406
+ )
407
+ }
408
+ }
409
+
410
+ dropped(): number {
411
+ return this.droppedCount
412
+ }
413
+
414
+ close(): void {
415
+ try {
416
+ this.insert?.finalize()
417
+ this.insert = null
418
+ this.db.close(true)
419
+ } catch {
420
+ // Already closed.
421
+ }
422
+ }
423
+ }
424
+
425
+ /** Sink for disabled telemetry: accepts and discards every event. */
426
+ export class NoopSink implements TelemetrySink {
427
+ record(_event: TelemetryEvent): void {}
428
+ dropped(): number {
429
+ return 0
430
+ }
431
+ close(): void {}
432
+ }
433
+
434
+ export interface OpenSinkOptions extends SinkDefaults {
435
+ path?: string
436
+ stderr?: StderrWriter
437
+ }
438
+
439
+ /**
440
+ * Returns a sink for the environment. Never throws: telemetry disabled via
441
+ * `TG_SAI_TELEMETRY=off`, or a database that fails to open, yields a
442
+ * `NoopSink` (the latter after one stderr line).
443
+ */
444
+ export function openTelemetrySink(
445
+ env: ProfileStoreEnv = process.env,
446
+ options: OpenSinkOptions = {},
447
+ ): TelemetrySink {
448
+ if (telemetryDisabled(env)) return new NoopSink()
449
+ const stderr = options.stderr ?? defaultStderr
450
+ const path = options.path ?? telemetryDbPath(env)
451
+ try {
452
+ const db = openTelemetryDb(path)
453
+ return new SqliteTelemetrySink(db, options, stderr)
454
+ } catch (error) {
455
+ stderr(`tg-sai telemetry: disabled, cannot open ${path}: ` +
456
+ errorMessage(error))
457
+ return new NoopSink()
458
+ }
459
+ }
460
+
461
+ function safeHostname(): string | undefined {
462
+ try {
463
+ return hostname()
464
+ } catch {
465
+ return undefined
466
+ }
467
+ }
468
+
469
+ function errorMessage(error: unknown): string {
470
+ return error instanceof Error ? error.message : String(error)
471
+ }