@mt-tl/tl 0.1.0

package/src/tl/parser.ts

@@ -0,0 +1,144 @@
+import { readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import type { TlDef, TlParam } from './ir.js'
+import { parseType } from './ir.js'
+import { getTlObjectCrc32 } from './crc32.js'
+
+const LINE_RE = /^([\w.]+)#([0-9a-fA-F]+)\s*(.*?)\s*=\s*([\w.]+)\s*;\s*$/
+
+/**
+ * Core MTProto constructors that scheme_0_protocol.tl leaves commented out
+ * ("parsed manually"). They still need registry entries for id<->name lookup.
+ */
+const CORE_LINES: Array<{ kind: 'constructor' | 'method'; line: string }> = [
+    { kind: 'constructor', line: 'vector#1cb5c415 = Vector;' },
+    { kind: 'constructor', line: 'rpc_result#f35c6d01 req_msg_id:long result:Object = RpcResult;' },
+    { kind: 'constructor', line: 'msg_container#73f1f8dc messages:vector<%Message> = MessageContainer;' },
+    {
+        kind: 'constructor',
+        line: 'message#5bb8e511 msg_id:long seqno:int bytes:int body:Object = Message;',
+    },
+    { kind: 'constructor', line: 'msg_copy#e06046b2 orig_message:Message = MessageCopy;' },
+    { kind: 'constructor', line: 'gzip_packed#3072cfa1 packed_data:bytes = Object;' },
+]
+
+function parseParams(paramsStr: string): TlParam[] {
+    if (!paramsStr) return []
+    return paramsStr
+        .split(/\s+/)
+        .filter(tok => tok.includes(':') && !tok.includes('{') && !tok.includes('}'))
+        .map(tok => {
+            const idx = tok.indexOf(':')
+            const name = tok.slice(0, idx)
+            const raw = tok.slice(idx + 1)
+            return { name, raw, type: parseType(raw) }
+        })
+}
+
+export interface ParsedLine {
+    def: TlDef
+    /** declared id differs from the crc32 of the normalized line */
+    crcMismatch: boolean
+}
+
+function parseLine(
+    line: string,
+    kind: 'constructor' | 'method',
+    isProtocol: boolean,
+    validateCrc: boolean,
+): ParsedLine | null {
+    const m = line.match(LINE_RE)
+    if (!m) return null
+    const [, name, hashRaw, paramsStr, type] = m
+    const id = hashRaw!.toLowerCase().padStart(8, '0')
+    const def: TlDef = {
+        id,
+        idNum: parseInt(id, 16) >>> 0,
+        name: name!,
+        kind,
+        params: parseParams(paramsStr ?? ''),
+        type: type!,
+        isProtocol,
+    }
+    const crcMismatch = validateCrc ? getTlObjectCrc32(line.trim()) !== id : false
+    return { def, crcMismatch }
+}
+
+export interface ParseResult {
+    defs: TlDef[]
+    crcMismatches: Array<{ name: string; id: string; computed: string }>
+}
+
+export function parseTlText(text: string, isProtocol: boolean): ParseResult {
+    const defs: TlDef[] = []
+    const crcMismatches: ParseResult['crcMismatches'] = []
+    let kind: 'constructor' | 'method' = 'constructor'
+
+    for (const rawLine of text.split('\n')) {
+        const line = rawLine.trim()
+        if (!line || line.startsWith('//') || line.startsWith('*') || line.startsWith('/*')) continue
+        if (line.includes('---types---')) {
+            kind = 'constructor'
+            continue
+        }
+        if (line.includes('---functions---')) {
+            kind = 'method'
+            continue
+        }
+        const parsed = parseLine(line, kind, isProtocol, true)
+        if (!parsed) continue
+        defs.push(parsed.def)
+        if (parsed.crcMismatch) {
+            crcMismatches.push({
+                name: parsed.def.name,
+                id: parsed.def.id,
+                computed: getTlObjectCrc32(line),
+            })
+        }
+    }
+
+    return { defs, crcMismatches }
+}
+
+function parseCoreLines(): TlDef[] {
+    return CORE_LINES.map(({ kind, line }) => {
+        const parsed = parseLine(line, kind, true, false)
+        if (!parsed) throw new Error(`Failed to parse core line: ${line}`)
+        return parsed.def
+    })
+}
+
+/**
+ * Loads every `*.tl` file from a directory into the IR. `scheme_0_protocol.tl`
+ * is flagged as protocol; the manually-parsed MTProto core constructors are
+ * merged in. Duplicate constructor ids are de-duplicated (first wins).
+ */
+export function parseSchemaDir(dir: string): ParseResult {
+    const files = readdirSync(dir)
+        .filter(f => f.endsWith('.tl'))
+        .sort()
+
+    const seen = new Set<string>()
+    const defs: TlDef[] = []
+    const crcMismatches: ParseResult['crcMismatches'] = []
+
+    for (const def of parseCoreLines()) {
+        if (seen.has(def.id)) continue
+        seen.add(def.id)
+        defs.push(def)
+    }
+
+    for (const file of files) {
+        const isProtocol = file === 'scheme_0_protocol.tl'
+        const text = readFileSync(join(dir, file), 'utf-8')
+        const res = parseTlText(text, isProtocol)
+        crcMismatches.push(...res.crcMismatches)
+        for (const def of res.defs) {
+            if (seen.has(def.id)) continue
+            seen.add(def.id)
+            defs.push(def)
+        }
+    }
+
+    return { defs, crcMismatches }
+}

package/src/tl/value.ts

@@ -0,0 +1,60 @@
+/**
+ * In-code representation of a decoded TL value.
+ *
+ * A constructed type is a tagged plain object `{ _: 'predicate.name', ...fields }`
+ * (matching the existing `toPrimitiveObject` convention). Wire primitives map to:
+ *   int -> number, long/flags -> bigint|number, double -> number, string -> string,
+ *   bytes/int128/int256 -> Buffer, Bool/true -> boolean, vector -> array.
+ */
+export type TlValue = number | bigint | boolean | string | Buffer | TlObject | TlValue[] | null | undefined
+
+export interface TlObject {
+    _: string
+    [field: string]: TlValue
+}
+
+/** JSON-safe encoding so values can cross the JSON-RPC boundary. */
+export type JsonValue =
+    | number
+    | boolean
+    | string
+    | null
+    | { $bigint: string }
+    | { $bin: string }
+    | JsonValue[]
+    | { [k: string]: JsonValue }
+
+function isBuffer(v: unknown): v is Buffer {
+    return Buffer.isBuffer(v)
+}
+
+export function toJson(value: TlValue): JsonValue {
+    if (value === null || value === undefined) return null
+    if (typeof value === 'bigint') return { $bigint: value.toString() }
+    if (typeof value === 'number' || typeof value === 'boolean' || typeof value === 'string') return value
+    if (isBuffer(value)) return { $bin: value.toString('base64') }
+    if (Array.isArray(value)) return value.map(toJson)
+    // TlObject
+    const out: { [k: string]: JsonValue } = {}
+    for (const [k, v] of Object.entries(value)) out[k] = toJson(v as TlValue)
+    return out
+}
+
+export function fromJson(value: JsonValue): TlValue {
+    if (value === null) return null
+    if (typeof value === 'number' || typeof value === 'boolean' || typeof value === 'string') return value
+    if (Array.isArray(value)) return value.map(fromJson)
+    if (typeof value === 'object') {
+        if ('$bigint' in value && typeof value.$bigint === 'string') return BigInt(value.$bigint)
+        if ('$bin' in value && typeof value.$bin === 'string') return Buffer.from(value.$bin, 'base64')
+        const out: { [k: string]: TlValue } = {}
+        for (const [k, v] of Object.entries(value)) out[k] = fromJson(v as JsonValue)
+        return out as TlObject
+    }
+    return null
+}
+
+/** Pretty JSON string for terminal logging (bigint/Buffer made readable). */
+export function stringify(value: TlValue, indent = 2): string {
+    return JSON.stringify(toJson(value), null, indent)
+}

package/src/tools/freeze-layer.ts

@@ -0,0 +1,108 @@
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { parseSchemaDir } from '../tl/parser.js'
+
+interface SnapshotEntry {
+    id: string
+    predicate?: string
+    method?: string
+    params: Array<{ name: string; type: string }>
+    type: string
+}
+
+export interface FreezeResult {
+    constructors: number
+    methods: number
+    /** Path of the JSON snapshot the gateway loads. */
+    out: string
+    /** Path of the human-readable `.tl` mirror (not loaded; for inspection/diffs). */
+    tlOut: string
+    crcWarnings: number
+}
+
+/** A single `name#id p:type … = ResultType;` line. */
+function tlLine(e: SnapshotEntry): string {
+    const name = e.predicate ?? e.method ?? '?'
+    const params = e.params.map(p => `${p.name}:${p.type}`).join(' ')
+    return `${name}#${e.id}${params ? ' ' + params : ''} = ${e.type};`
+}
+
+/** Reconstruct a readable `.tl` from the frozen IR (constructors, then methods). */
+function toTlText(constructors: SnapshotEntry[], methods: SnapshotEntry[], layer: number): string {
+    const lines = [`// Frozen snapshot of layer ${layer}. Generated by \`freeze\` — do not edit.`, '']
+    lines.push(...constructors.map(tlLine))
+    if (methods.length) lines.push('', '---functions---', '', ...methods.map(tlLine))
+    return lines.join('\n') + '\n'
+}
+
+/**
+ * Freezes the `.tl` schema in `schemaDir` into a per-layer snapshot
+ * `<outDir>/scheme_<layer>.json` (loaded by the gateway) plus a human-readable
+ * `<outDir>/scheme_<layer>.tl` mirror (for inspection/diffs; not loaded). Run
+ * this when you SHIP a layer: the `.tl` you edit is the in-progress newest layer;
+ * the frozen snapshot is the historical record the gateway uses to encode for
+ * clients on that layer.
+ *
+ * The snapshot is BUSINESS-only: the immutable MTProto protocol/core types are
+ * excluded (they live in @mt-tl/tl and are loaded globally), so a business
+ * layer never carries protocol definitions.
+ *
+ * Schema ownership lives in the consumer app, so paths are explicit — apps wrap
+ * this in their own `freeze` script.
+ */
+export function freezeLayer(schemaDir: string, outDir: string, layer: number): FreezeResult {
+    const { defs, crcMismatches } = parseSchemaDir(schemaDir)
+
+    const constructors: SnapshotEntry[] = []
+    const methods: SnapshotEntry[] = []
+    const seen = new Set<string>()
+    for (const d of defs) {
+        // Per-layer snapshots are BUSINESS-only. The MTProto protocol layer
+        // (scheme_0_protocol + the manually-parsed core ctors `parseSchemaDir`
+        // injects) is immutable and layer-independent — it lives in @mt-tl/tl
+        // and the gateway always loads it globally, so it must NOT be baked into
+        // each business snapshot (mirrors Telegram's split: mtproto_api vs api).
+        if (d.isProtocol) continue
+        if (seen.has(d.id)) continue
+        seen.add(d.id)
+        const params = d.params.map(p => ({ name: p.name, type: p.raw }))
+        if (d.kind === 'method') methods.push({ id: d.id, method: d.name, params, type: d.type })
+        else constructors.push({ id: d.id, predicate: d.name, params, type: d.type })
+    }
+
+    const byId = (a: SnapshotEntry, b: SnapshotEntry) => parseInt(a.id, 16) - parseInt(b.id, 16)
+    constructors.sort(byId)
+    methods.sort(byId)
+
+    mkdirSync(outDir, { recursive: true })
+    const out = join(outDir, `scheme_${layer}.json`)
+    const tlOut = join(outDir, `scheme_${layer}.tl`)
+    writeFileSync(out, JSON.stringify({ constructors, methods }, null, 2))
+    writeFileSync(tlOut, toTlText(constructors, methods, layer))
+
+    return {
+        constructors: constructors.length,
+        methods: methods.length,
+        out,
+        tlOut,
+        crcWarnings: crcMismatches.length,
+    }
+}
+
+// CLI: `tsx freeze-layer.ts <layer> <schemaDir> <outDir>`
+//   or env: `SCHEMA_DIR=… SCHEMA_LAYERS_DIR=… tsx freeze-layer.ts <layer>`
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+    const layer = Number(process.argv[2])
+    const schemaDir = process.argv[3] || process.env.SCHEMA_DIR
+    const outDir = process.argv[4] || process.env.SCHEMA_LAYERS_DIR
+    if (!Number.isInteger(layer) || layer <= 0 || !schemaDir || !outDir) {
+        console.error('usage: freeze-layer <layer> <schemaDir> <outDir>')
+        process.exit(1)
+    }
+    const r = freezeLayer(schemaDir, outDir, layer)
+    console.log(
+        `Froze layer ${layer}: ${r.constructors} constructors, ${r.methods} methods -> ${r.out} (+ ${r.tlOut})` +
+            (r.crcWarnings ? `  (${r.crcWarnings} crc warnings)` : ''),
+    )
+}

package/src/wire.ts

@@ -0,0 +1,54 @@
+import type { JsonValue } from './tl/value.js'
+import type { RpcContext, RpcRequest, RpcResponse, SessionEffect } from './rpc.js'
+
+/**
+ * Wire format for the gateway↔worker RPC, shared so both sides agree:
+ *   request  { jsonrpc, id, method, params, context }
+ *   response { result? , error?: { code, message }, effects? }
+ * Gateway encodes requests / decodes responses; workers do the inverse.
+ */
+export function encodeRequest(req: RpcRequest): unknown {
+    return {
+        jsonrpc: '2.0',
+        id: req.id,
+        method: req.method,
+        params: req.params,
+        context: req.context,
+    }
+}
+
+export function decodeRequest(json: unknown): RpcRequest | null {
+    if (!json || typeof json !== 'object') return null
+    const o = json as Record<string, unknown>
+    if (typeof o.method !== 'string' || !o.context || typeof o.context !== 'object') return null
+    return {
+        id: o.id === undefined ? '' : String(o.id),
+        method: o.method,
+        params: (o.params ?? {}) as JsonValue,
+        context: o.context as RpcContext,
+    }
+}
+
+export function encodeResponse(res: RpcResponse): unknown {
+    return res
+}
+
+export function decodeResponse(json: unknown): RpcResponse {
+    if (!json || typeof json !== 'object') {
+        return { error: { code: 502, message: 'BAD_RESPONSE' } }
+    }
+    const o = json as Record<string, unknown>
+    const effects = Array.isArray(o.effects) ? (o.effects as SessionEffect[]) : undefined
+
+    if (o.error && typeof o.error === 'object') {
+        const e = o.error as Record<string, unknown>
+        return {
+            error: {
+                code: typeof e.code === 'number' ? e.code : 500,
+                message: typeof e.message === 'string' ? e.message : 'ERROR',
+            },
+            effects,
+        }
+    }
+    return { result: o.result as JsonValue, effects }
+}