@mt-tl/tl 0.1.0

package/src/index.ts

@@ -0,0 +1,22 @@
+import { fileURLToPath } from 'node:url'
+
+export * from './tl/ir.js'
+export * from './tl/value.js'
+export * from './tl/parser.js'
+export * from './tl/crc32.js'
+export * from './rpc.js'
+export * from './wire.js'
+export * from './migrate.js'
+// Shared structured logger — the server engine, the handler layer, and your app
+// import it for one consistent log style (see docs/guide/observability.md).
+export * from './logger.js'
+// Codegen + layer tooling — consumers generate TS types and freeze layer
+// snapshots from THEIR `.tl` schema.
+export { generateSchemaTs, writeSchemaTs } from './codegen/gen-types.js'
+export { freezeLayer, type FreezeResult } from './tools/freeze-layer.js'
+
+// @mt-tl/tl ships only the fixed MTProto **protocol** schema; business `.tl`
+// lives in the consumer app. This is the protocol-only default; apps pass their
+// own schema dir to the gateway.
+/** Absolute path to the bundled MTProto protocol `.tl` schema directory. */
+export const protocolSchemaDir = fileURLToPath(new URL('../schema', import.meta.url))

package/src/logger.ts

@@ -0,0 +1,210 @@
+/**
+ * A tiny dependency-free structured logger, shared across the MTProto packages
+ * (the server engine, the handler layer, and your app — import it for one
+ * consistent log style). Levels gate output; fields are key/value context.
+ *
+ * - `LOG_LEVEL` sets the threshold (`trace`<`debug`<`info`<`warn`<`error`<`silent`).
+ * - `LOG_FORMAT=json` emits one JSON object per line (ship to a log pipeline);
+ *   anything else is a readable line for local dev.
+ * - `LOG_ERROR_STACK=true|false` forces whether `Error.stack` is serialized
+ *   (default: on for `pretty`, off for `json` — prod opts in).
+ * - `pretty` output is ANSI-colored when stdout is a TTY (dim keys, colored
+ *   level) for readability; honors `NO_COLOR` / `FORCE_COLOR`.
+ * - Tests default to `silent` (set `LOG_LEVEL` to override).
+ *
+ * Levels, by convention across this codebase:
+ * - `trace` — byte/hex protocol firehose (every recv, framing headers, decrypt).
+ * - `debug` — useful protocol events (decoded method, salt re-send, handshake step).
+ * - `info`  — request/response one-liners + lifecycle links/unlinks (sockets,
+ *             sessions, auth keys, users) + update deliveries.
+ * - `warn`  — recoverable anomalies (decode failure, unknown method, integrity
+ *             rejection, insecure config).
+ * - `error` — a request that failed or an update that could not be delivered.
+ */
+export type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'silent'
+
+const ORDER: Record<LogLevel, number> = {
+    trace: 5,
+    debug: 10,
+    info: 20,
+    warn: 30,
+    error: 40,
+    silent: 99,
+}
+
+export type Fields = Record<string, unknown>
+
+export interface Logger {
+    /** Byte/hex protocol firehose — guard heavy field building with {@link isLevelEnabled}. */
+    trace(msg: string, fields?: Fields): void
+    debug(msg: string, fields?: Fields): void
+    info(msg: string, fields?: Fields): void
+    warn(msg: string, fields?: Fields): void
+    error(msg: string, fields?: Fields): void
+    /** Returns a logger that merges `bindings` into every line (e.g. a scope/conn id). */
+    child(bindings: Fields): Logger
+    /** True when a message at `level` would be emitted — guard expensive field building. */
+    isLevelEnabled(level: LogLevel): boolean
+    /** The active threshold level. */
+    readonly level: LogLevel
+}
+
+export interface LoggerOptions {
+    level?: LogLevel
+    /** 'json' for machine-readable lines, 'pretty' for humans. */
+    format?: 'json' | 'pretty'
+    name?: string
+    bindings?: Fields
+    /**
+     * Include `Error.stack` when serializing an Error field. Default: `true` for
+     * `pretty` (dev), `false` for `json` (prod opts in). `LOG_ERROR_STACK` overrides.
+     */
+    errorStack?: boolean
+    /**
+     * ANSI-color the `pretty` output (dim keys, colored level). Default: auto — on
+     * when stdout is a TTY and `NO_COLOR` is unset (off when a custom `write` is
+     * given). Never colors `json`.
+     */
+    color?: boolean
+    /** Sink (defaults to stdout/stderr by level). Override in tests. */
+    write?: (line: string) => void
+}
+
+function defaultLevel(): LogLevel {
+    const env = process.env.LOG_LEVEL as LogLevel | undefined
+    if (env && env in ORDER) return env
+    if (process.env.VITEST || process.env.NODE_ENV === 'test') return 'silent'
+    return 'info'
+}
+
+function defaultFormat(): 'json' | 'pretty' {
+    return process.env.LOG_FORMAT === 'json' ? 'json' : 'pretty'
+}
+
+function defaultErrorStack(format: 'json' | 'pretty'): boolean {
+    const env = process.env.LOG_ERROR_STACK
+    if (env === 'true' || env === '1') return true
+    if (env === 'false' || env === '0') return false
+    return format === 'pretty'
+}
+
+function defaultColor(format: 'json' | 'pretty'): boolean {
+    if (format === 'json') return false
+    if (process.env.NO_COLOR !== undefined) return false
+    if (process.env.FORCE_COLOR) return true
+    return !!process.stdout.isTTY
+}
+
+// ANSI styling for the pretty (TTY) format.
+const RESET = '\x1b[0m'
+const DIM = '\x1b[2m'
+const BOLD = '\x1b[1m'
+const GRAY = '\x1b[90m'
+const LEVEL_COLOR: Record<Exclude<LogLevel, 'silent'>, string> = {
+    trace: '\x1b[90m', // gray
+    debug: '\x1b[36m', // cyan
+    info: '\x1b[32m', // green
+    warn: '\x1b[33m', // yellow
+    error: '\x1b[31m', // red
+}
+
+function makeSerialize(errorStack: boolean): (value: unknown) => unknown {
+    return function serialize(value: unknown): unknown {
+        if (value instanceof Error) {
+            const base: Fields = { name: value.name, message: value.message }
+            if (errorStack && value.stack) base.stack = value.stack
+            return base
+        }
+        if (typeof value === 'bigint') return value.toString()
+        return value
+    }
+}
+
+function prettyFields(fields: Fields, serialize: (v: unknown) => unknown, color: boolean): string {
+    const parts: string[] = []
+    for (const [k, v] of Object.entries(fields)) {
+        const sv = serialize(v)
+        const val = typeof sv === 'object' && sv !== null ? JSON.stringify(sv) : String(sv)
+        // Dim the key so the eye separates key from value; values stay bright.
+        parts.push(color ? `${DIM}${k}=${RESET}${val}` : `${k}=${val}`)
+    }
+    if (!parts.length) return ''
+    // Wider gap between fields when colored (the TTY view we optimize for reading).
+    const sep = color ? '  ' : ' '
+    return sep + parts.join(sep)
+}
+
+export function createLogger(options: LoggerOptions = {}): Logger {
+    const level = options.level ?? defaultLevel()
+    const format = options.format ?? defaultFormat()
+    const errorStack = options.errorStack ?? defaultErrorStack(format)
+    // A custom sink (tests/embedders) gets no color unless explicitly asked.
+    const color = options.color ?? (options.write ? false : defaultColor(format))
+    const name = options.name
+    const bindings = options.bindings ?? {}
+    const threshold = ORDER[level]
+    const serialize = makeSerialize(errorStack)
+
+    const emit = (lvl: Exclude<LogLevel, 'silent'>, msg: string, fields?: Fields) => {
+        if (ORDER[lvl] < threshold) return
+        const merged: Fields = { ...bindings, ...(fields ?? {}) }
+        const write = options.write ?? (lvl === 'error' || lvl === 'warn' ? errLine : outLine)
+        if (format === 'json') {
+            const obj: Fields = { time: new Date().toISOString(), level: lvl, ...(name ? { name } : {}), msg }
+            for (const [k, v] of Object.entries(merged)) obj[k] = serialize(v)
+            write(JSON.stringify(obj))
+        } else {
+            const ts = new Date().toISOString().slice(11, 23)
+            const lvlText = lvl.toUpperCase().padEnd(5)
+            if (color) {
+                const tag = name ? ` ${DIM}[${name}]${RESET}` : ''
+                write(
+                    `${GRAY}${ts}${RESET} ${LEVEL_COLOR[lvl]}${lvlText}${RESET}${tag} ${BOLD}${msg}${RESET}` +
+                        prettyFields(merged, serialize, true),
+                )
+            } else {
+                const tag = name ? ` [${name}]` : ''
+                write(`${ts} ${lvlText}${tag} ${msg}` + prettyFields(merged, serialize, false))
+            }
+        }
+    }
+
+    return {
+        level,
+        trace: (msg, fields) => emit('trace', msg, fields),
+        debug: (msg, fields) => emit('debug', msg, fields),
+        info: (msg, fields) => emit('info', msg, fields),
+        warn: (msg, fields) => emit('warn', msg, fields),
+        error: (msg, fields) => emit('error', msg, fields),
+        isLevelEnabled: lvl => ORDER[lvl] >= threshold,
+        child: extra =>
+            createLogger({
+                ...options,
+                level,
+                format,
+                errorStack,
+                color,
+                name,
+                bindings: { ...bindings, ...extra },
+            }),
+    }
+}
+
+function outLine(line: string): void {
+    process.stdout.write(line + '\n')
+}
+function errLine(line: string): void {
+    process.stderr.write(line + '\n')
+}
+
+/** A logger that drops everything (explicit opt-out in tests/embedders). */
+export const noopLogger: Logger = {
+    level: 'silent',
+    trace() {},
+    debug() {},
+    info() {},
+    warn() {},
+    error() {},
+    isLevelEnabled: () => false,
+    child: () => noopLogger,
+}

package/src/migrate.ts

@@ -0,0 +1,101 @@
+import type { TlObject, TlValue } from './tl/value.js'
+
+/**
+ * One rung of a predicate's migration ladder. `up` maps THIS version's shape to
+ * the NEXT version's; `down` maps the next version back to this one. The newest
+ * (canonical) rung has neither.
+ */
+export interface MigrationRung {
+    /** Layer this shape was introduced. */
+    since: number
+    up?: (obj: Record<string, unknown>) => Record<string, unknown>
+    down?: (obj: Record<string, unknown>) => Record<string, unknown>
+}
+
+/**
+ * Per-predicate migration ladders. The gateway normalizes inbound values to the
+ * canonical (newest) shape (`up`) before forwarding, and renders canonical values
+ * down to a client's layer (`down`) before encoding — so workers only ever see
+ * the canonical shape. Scales to N versions: one rung per version; adding a
+ * version appends a rung and never touches the others.
+ *
+ * Only non-additively-changed predicates need a ladder; everything else is
+ * handled by decode-union + layered-encode with no rungs (identity here).
+ */
+export class MigrationRegistry {
+    private byPredicate = new Map<string, MigrationRung[]>()
+
+    /** Register a predicate's ladder (rungs in any order; sorted by `since`). */
+    register(predicate: string, rungs: MigrationRung[]): this {
+        this.byPredicate.set(
+            predicate,
+            [...rungs].sort((a, b) => a.since - b.since),
+        )
+        return this
+    }
+
+    has(predicate: string): boolean {
+        return this.byPredicate.has(predicate)
+    }
+
+    get size(): number {
+        return this.byPredicate.size
+    }
+
+    /** Normalize a value decoded at `fromLayer` up to the canonical shape. */
+    up(value: TlValue, fromLayer: number): TlValue {
+        return this.recurse(value, obj => this.upObject(obj, fromLayer))
+    }
+
+    /** Render a canonical value down to `toLayer`'s shape. */
+    down(value: TlValue, toLayer: number): TlValue {
+        return this.recurse(value, obj => this.downObject(obj, toLayer))
+    }
+
+    // Children-first walk: normalize nested objects, then transform the parent.
+    private recurse(value: TlValue, fn: (obj: TlObject) => TlObject): TlValue {
+        if (Array.isArray(value)) return value.map(v => this.recurse(v, fn))
+        if (value && typeof value === 'object' && !Buffer.isBuffer(value) && '_' in value) {
+            const obj = value as TlObject
+            const next: TlObject = { _: obj._ }
+            for (const [k, v] of Object.entries(obj)) {
+                if (k !== '_') next[k] = this.recurse(v as TlValue, fn)
+            }
+            return fn(next)
+        }
+        return value
+    }
+
+    private upObject(obj: TlObject, fromLayer: number): TlObject {
+        const rungs = this.byPredicate.get(obj._)
+        if (!rungs) return obj
+        let cur: Record<string, unknown> = obj
+        for (let i = startRung(rungs, fromLayer); i <= rungs.length - 2; i++) {
+            const up = rungs[i]!.up
+            if (up) cur = up(cur)
+        }
+        return cur as TlObject
+    }
+
+    private downObject(obj: TlObject, toLayer: number): TlObject {
+        const rungs = this.byPredicate.get(obj._)
+        if (!rungs) return obj
+        const target = startRung(rungs, toLayer)
+        let cur: Record<string, unknown> = obj
+        for (let i = rungs.length - 2; i >= target; i--) {
+            const down = rungs[i]!.down
+            if (down) cur = down(cur)
+        }
+        return cur as TlObject
+    }
+}
+
+/** Index of the rung active at `layer` (the latest with since <= layer; else 0). */
+function startRung(rungs: MigrationRung[], layer: number): number {
+    let idx = 0
+    for (let i = 0; i < rungs.length; i++) {
+        if (rungs[i]!.since <= layer) idx = i
+        else break
+    }
+    return idx
+}

package/src/rpc.ts

@@ -0,0 +1,70 @@
+import type { JsonValue } from './tl/value.js'
+
+/** Per-request connection context the engine attaches to every business call; surfaced as `ctx.request`. */
+export interface RpcContext {
+    /** The MTProto session id (hex). */
+    sessionId: string
+    /** The connection's auth key id (hex). */
+    authKeyId: string
+    /**
+     * The **subject** bound to this auth key — your app's *internal* user id,
+     * opaque to the framework and safe to share across your services (e.g. a
+     * uuid). Set it via `ctx.login(subject)`; `undefined` for an anonymous key.
+     *
+     * This is deliberately NOT the wire `user_id` your TL schema exposes to
+     * clients (Telegram-style `int`/`long`). The framework only routes/persists
+     * by `subject`; mapping `subject ⇄ public user_id` is your app's job (see the
+     * demo's `users` module). Keeping them separate lets the public id stay an
+     * `int` while the protocol runs entirely on your internal id.
+     */
+    subject?: string
+    /** The client's negotiated TL layer. */
+    apiLayer: number
+    /** `initConnection.api_id` — the client app id, if reported. */
+    apiId?: number
+    /** `initConnection.device_model`, if reported. */
+    deviceModel?: string
+    /** `initConnection.system_version`, if reported. */
+    systemVersion?: string
+    /** `initConnection.app_version`, if reported. */
+    appVersion?: string
+    /** `initConnection.lang_code`, if reported. */
+    langCode?: string
+    /** Client IP (from the carrier / `X-Forwarded-For`), if known. */
+    ip?: string
+}
+
+/** A decoded business method call handed to the handler layer. */
+export interface RpcRequest {
+    /** Derived from the client's reqMsgId. */
+    id: string
+    /** TL method name, e.g. "dust.getConfig". */
+    method: string
+    /** Decoded params as tagged JSON (bigint -> {$bigint}, Buffer -> {$bin}). */
+    params: JsonValue
+    /** The connection context (see {@link RpcContext}). */
+    context: RpcContext
+}
+
+/**
+ * A side-effect a handler records (via `ctx.login`/`logout`/`revoke`) for the
+ * engine to apply to auth-key state, returned alongside the normal result/error.
+ * Keeps the engine agnostic to your auth scheme: a `signIn` handler returns
+ * `bindUser`, and the engine binds the `subject` (your internal user id) to the
+ * auth key.
+ */
+export type SessionEffect =
+    | { type: 'bindUser'; subject: string }
+    | { type: 'unbindUser' }
+    | { type: 'revokeKey' }
+
+/**
+ * Response envelope (transport-agnostic). Exactly one of `result` / `error` is
+ * set; `effects` may accompany either.
+ */
+export interface RpcResponse {
+    /** A TL result as tagged JSON ({ _: name, ... }, $bigint/$bin), or a primitive. */
+    result?: JsonValue
+    error?: { code: number; message: string }
+    effects?: SessionEffect[]
+}

package/src/tl/crc32.ts

@@ -0,0 +1,44 @@
+import CRC32 from 'crc-32'
+
+/**
+ * Computes a TL constructor id (crc32 of the normalized definition line),
+ * mirroring the existing backend's `get-tl-object-crc32.js` exactly so our
+ * validation agrees with the schema's declared ids.
+ *
+ * Normalization (applied repeatedly until stable):
+ *   :bytes              -> :string
+ *   ?bytes              -> ?string
+ *   #<hex>              -> (constructor id removed)
+ *   name:flags.N?true   -> (removed; presence-only flags don't affect the id)
+ *   < > and {} and ;    -> stripped / spaced
+ *   collapse double / edge spaces
+ */
+function normalizeSchemaLine(line: string): string {
+    const rules: Array<[RegExp, string]> = [
+        [/:bytes /g, ':string '],
+        [/\?bytes /g, '?string '],
+        [/#[a-f0-9]+ /g, ' '],
+        [/ [a-zA-Z0-9_]+:flags\.[0-9]+\?true/g, ''],
+        [/</g, ' '],
+        [/>/g, ' '],
+        [/;/g, ''],
+        [/\{/g, ''],
+        [/\}/g, ''],
+    ]
+
+    let out = line
+    let prev: string
+    do {
+        prev = out
+        for (const [re, to] of rules) out = out.replace(re, to)
+        out = out.replace(/ {2}/g, ' ').replace(/^ /, '').replace(/ $/, '')
+    } while (out !== prev)
+
+    return out
+}
+
+export function getTlObjectCrc32(line: string): string {
+    const hashNum = CRC32.bstr(normalizeSchemaLine(line))
+    const unsigned = hashNum < 0 ? hashNum + 0x100000000 : hashNum
+    return unsigned.toString(16).padStart(8, '0')
+}

package/src/tl/ir.ts

@@ -0,0 +1,105 @@
+/**
+ * Intermediate representation (IR) for the TL schema.
+ *
+ * A `.tl` file is parsed into a flat list of {@link TlDef} (constructors and
+ * methods). Each parameter's textual type is parsed once, at load time, into a
+ * structured {@link TlType} so the generic codec can walk it without re-parsing
+ * strings on every (de)serialization.
+ */
+
+export type TlType =
+    | { kind: 'int' } // 4 bytes LE -> number
+    | { kind: 'long' } // 8 bytes LE -> bigint
+    | { kind: 'double' } // 8 bytes IEEE754 LE -> number
+    | { kind: 'string' } // length-prefixed utf-8 -> string
+    | { kind: 'bytes' } // length-prefixed raw -> Buffer
+    | { kind: 'int128' } // 16 raw bytes -> Buffer
+    | { kind: 'int256' } // 32 raw bytes -> Buffer
+    | { kind: 'bool' } // boxed Bool -> boolean
+    | { kind: 'true' } // bare `true`, only as a flag presence marker -> boolean
+    | { kind: 'flags' } // the `#` bitmask field
+    | { kind: 'flag'; flagsField: string; bit: number; inner: TlType } // conditional field
+    | { kind: 'vector'; boxed: boolean; inner: TlType }
+    | { kind: 'object' } // `Object` / `!X` / `X` — a nested boxed object (read its ctor id)
+    | { kind: 'boxed'; name: string } // a named polymorphic type — read/write with ctor id
+    | { kind: 'bare'; name: string } // a named bare constructor (`%X` / lowercase) — no ctor id
+
+export interface TlParam {
+    name: string
+    /** Original textual type, kept for diagnostics / crc. */
+    raw: string
+    type: TlType
+}
+
+export interface TlDef {
+    /** 8-char lowercase hex constructor id. */
+    id: string
+    /** Unsigned 32-bit numeric form of {@link id}. */
+    idNum: number
+    /** predicate (constructor) or method name, e.g. `dust.getConfig`. */
+    name: string
+    kind: 'constructor' | 'method'
+    params: TlParam[]
+    /** Boxed result/return type, e.g. `dust.CalculatedExchange`. */
+    type: string
+    /** True for the immutable MTProto protocol layer (scheme_0_protocol + core). */
+    isProtocol: boolean
+}
+
+const PRIMITIVES = new Set(['int', 'long', 'double', 'string', 'bytes', 'int128', 'int256', 'Bool', 'true'])
+
+export function parseType(raw: string): TlType {
+    const t = raw.trim()
+
+    if (t === '#') return { kind: 'flags' }
+
+    // conditional field: <flagsField>.<bit>?<inner>, e.g. api_hash:flags.1?string
+    const cond = t.match(/^(\w+)\.(\d+)\?(.+)$/)
+    if (cond) {
+        return {
+            kind: 'flag',
+            flagsField: cond[1]!,
+            bit: Number(cond[2]),
+            inner: parseType(cond[3]!),
+        }
+    }
+
+    // Vector<T> (boxed) or vector<T> (bare)
+    const vec = t.match(/^([Vv])ector<(.+)>$/)
+    if (vec) {
+        return { kind: 'vector', boxed: vec[1] === 'V', inner: parseType(vec[2]!) }
+    }
+
+    if (PRIMITIVES.has(t)) {
+        switch (t) {
+            case 'int':
+                return { kind: 'int' }
+            case 'long':
+                return { kind: 'long' }
+            case 'double':
+                return { kind: 'double' }
+            case 'string':
+                return { kind: 'string' }
+            case 'bytes':
+                return { kind: 'bytes' }
+            case 'int128':
+                return { kind: 'int128' }
+            case 'int256':
+                return { kind: 'int256' }
+            case 'Bool':
+                return { kind: 'bool' }
+            case 'true':
+                return { kind: 'true' }
+        }
+    }
+
+    if (t === 'Object' || t === 'X' || t.startsWith('!')) return { kind: 'object' }
+    if (t.startsWith('%')) return { kind: 'bare', name: t.slice(1) }
+
+    // Named type. Boxed iff the final segment is capitalized (e.g. dust.DustBalance,
+    // Currency); otherwise a bare constructor reference (e.g. future_salt).
+    const lastSeg = t.includes('.') ? t.slice(t.lastIndexOf('.') + 1) : t
+    const first = lastSeg[0] ?? ''
+    if (first >= 'A' && first <= 'Z') return { kind: 'boxed', name: t }
+    return { kind: 'bare', name: t }
+}