gen-typescript-from-tolk-dev 0.1.0

package/README.md

@@ -0,0 +1,47 @@
+# Tolk-to-TypeScript wrappers generator (alpha)
+
+Currently, it uses a patched LSP to parse and Tolk source code (for MVP). Later, the compiler will be able
+to emit ABI of a contract, and LSP won't be used in this process.
+
+### Main CLI
+
+The main entrypoint accepts contract ABI as a JSON string and prints the generated TypeScript wrapper to stdout:
+
+```bash
+npx gen-typescript-from-tolk '{"contractName":"MyContract", ... }'
+```
+
+This package no longer bundles a Tolk compiler. It only supports `ABI JSON -> TypeScript wrapper`.
+
+
+### Configuring TypeScript output
+
+1. All structs/aliases/enums are generated to the output — it's unconfigurable currently. 
+If a struct can't be serialized (contains `int` for example, not int32/uint64),
+its `fromSlice` and `store` just contain `throw new Error` with a description.
+
+2. Default values of all structs are preserved:
+```
+struct WalletStorage {
+    jettonBalance: coins = 0    // <----
+    ownerAddress: address
+    minterAddress: address
+}
+```
+
+Then, `jettonBalance` key in `WalletStorage.create()` will be optional. If it's a storage or a message, it will also be optional in `fromStorage` and `sendXXX`.
+
+3. If your storage change shape after initialization (the case of NFT), you can specify an "uninited storage" shape:
+```
+type NOT_INITIALIZED_STORAGE = StorageStructNotInitialized
+```
+
+Then, `fromStorage` will accept fields from this struct, not from `STORAGE`.
+
+
+### Known issues
+
+1. Generics behave incorrectly when `T` is used inside unions (because generics are not monomorphic, they are true TypeScript generics). For example, you have `struct A<T> { v: T | int8 }` and instantiate `A<int2 | int4>`, then an actual prefix tree will be '00/01/10', but a TS wrapper serializes '0+T/1'; or, for `A<int8>` actually will be no union (`int8 | int8` = `int8`), but a TS wrapper will still serialize as '0+int8/1'. Similar corner cases appear in get methods that work via the stack. For example, `struct MyNullable<T> { v: T? }`, instantiated as `MyNullable<Point>` has no information about stack layout and type-id.
+2. Non-standard keys for maps are unsupported, like `map<Point, ...>`. Only intN/uintN/address are supported (99% use cases). Others can not be represented via `@ton/core` library at all.
+3. Custom serializers for generic structs `fun Some<T>.packToBuilder` not supported (works perfectly for non-generic structs and aliases, but not for generic).
+4. Default values for struct fields, if a field contains a union, not supported.

package/bin/generator.js

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+
+const path = require('path');
+require('ts-node').register({
+    transpileOnly: true,
+    project: path.join(__dirname, '..', 'tsconfig.json'),
+});
+
+require('../src/cli-generate-from-abi-json')
+    .runGeneratorFromAbiCli(process.argv.slice(2))
+    .catch((error) => {
+        const message = error instanceof Error ? error.message : String(error);
+        process.stderr.write(`${message}\n`);
+        process.exitCode = 1;
+    });

package/package.json

@@ -0,0 +1,32 @@
+{
+    "name": "gen-typescript-from-tolk-dev",
+    "version": "0.1.0",
+    "author": "TON Core",
+    "files": [
+        "README.md",
+        "bin",
+        "src",
+        "tsconfig.json"
+    ],
+    "bin": {
+        "gen-typescript-from-tolk": "./bin/generator.js"
+    },
+    "scripts": {
+        "build:standalone": "node scripts/build-standalone-cli.js",
+        "test": "jest --verbose"
+    },
+    "dependencies": {
+        "@ton/core": "~0",
+        "ts-node": "^10.9.2",
+        "typescript": "^5.9.2"
+    },
+    "devDependencies": {
+        "@ton/crypto": "^3.3.0",
+        "@ton/sandbox": ">=0.37.0",
+        "@ton/ton": ">=15.3.1 <16.0.0",
+        "@types/jest": "^30.0.0",
+        "@types/node": "^22.17.2",
+        "jest": "^30.0.5",
+        "ts-jest": "^29.4.1"
+    }
+}

package/src/abi-types.ts

@@ -0,0 +1,157 @@
+/*
+    ABI Types — describes the type system and symbols definitions in the Tolk language.
+    The goal is to have a full bidirectional mapping from any Tolk struct to ABI:
+  all tricky types like generics, aliases, inline unions, etc. should be supported.
+  Then, any input could be packed to a cell and unpacked back — exactly as it's done by the compiler. 
+ */
+
+// Since ABI should be JSON-serialized, all 'bigint' numbers are stored as strings.
+type bigint_as_string = string
+
+// Ty is an "ABI Type" that fully reflects the type system in Tolk.
+export type Ty =
+    | { kind: 'int' }
+    | { kind: 'intN'; n: number }
+    | { kind: 'uintN'; n: number }
+    | { kind: 'varintN', n: number }
+    | { kind: 'varuintN', n: number }
+    | { kind: 'coins' }
+    | { kind: 'bool' }
+    | { kind: 'cell' }
+    | { kind: 'builder' }
+    | { kind: 'slice' }
+    | { kind: 'string' }
+    | { kind: 'remaining' }
+    | { kind: 'address' }
+    | { kind: 'addressOpt' }
+    | { kind: 'addressExt' }
+    | { kind: 'addressAny' }
+    | { kind: 'bitsN'; n: number }
+    | { kind: 'nullLiteral' }
+    | { kind: 'callable' }
+    | { kind: 'void' }
+    | { kind: 'unknown' }
+    | { kind: 'nullable'; inner: Ty; stackTypeId?: number; stackWidth?: number }
+    | { kind: 'cellOf'; inner: Ty }
+    | { kind: 'arrayOf'; inner: Ty }
+    | { kind: 'lispListOf'; inner: Ty }
+    | { kind: 'tensor'; items: Ty[] }
+    | { kind: 'shapedTuple'; items: Ty[] }
+    | { kind: 'mapKV'; k: Ty; v: Ty }
+    | { kind: 'EnumRef'; enumName: string }
+    | { kind: 'StructRef'; structName: string; typeArgs?: Ty[] }
+    | { kind: 'AliasRef'; aliasName: string; typeArgs?: Ty[] }
+    | { kind: 'genericT'; nameT: string }
+    | { kind: 'union'; variants: UnionVariant[]; stackWidth: number }
+
+// UnionVariant exists for every `T_i` in a union type `T1 | T2 | ...`.
+// For binary serialization, a union should have a prefix tree,
+// which is either defined explicitly with struct prefixes: `struct (0x12345678) CounterIncrement`,
+// or auto-generated (implicit), e.g. `int8 | int16 | int32` is serialized as '00'+int8 / '01'+int16 / '10'+int32.
+export interface UnionVariant {
+    variantTy: Ty
+    prefixStr: string
+    prefixLen: number
+    isPrefixImplicit?: boolean
+    stackTypeId: number
+    stackWidth: number
+}
+
+// ABIConstExpression is "an expression" that occurs in fields defaults.
+// Example:
+// > struct Demo {
+// >    initial: int8 = 123
+// >    owner: address = address("UQ...")
+// > }
+// Then both fields have a default value: 
+//      initial: { kind: 'int', v: '123' }
+//      owner:   { kind: 'address', addr: 'UQ...' }
+export type ABIConstExpression =
+    | { kind: 'int'; v: bigint_as_string }
+    | { kind: 'bool'; v: boolean }
+    | { kind: 'slice'; hex: string }
+    | { kind: 'string'; str: string }
+    | { kind: 'address'; addr: string }
+    | { kind: 'tensor'; items: ABIConstExpression[] }
+    | { kind: 'shapedTuple'; items: ABIConstExpression[] }
+    | { kind: 'object'; structName: string; fields: ABIConstExpression[] }
+    | { kind: 'castTo'; inner: ABIConstExpression; castTo: Ty }
+    | { kind: 'null' }
+
+// ABICustomSerializers is present in a struct/alias if that type has custom serializers in Tolk code:
+//
+// fun SomeType.packToBuilder(self, mutate b: builder) { ... }
+// fun SomeType.unpackFromSlice(mutate s: slice): SomeAlias { ... }
+//
+// Body of these functions is not a part of ABI (it's arbitrary Tolk code).
+// To make serialization work (e.g., in TypeScript wrappers),
+// one should provide equivalent implementations in a language ABI is applied to. 
+export interface ABICustomSerializers {
+    packToBuilder: boolean
+    unpackFromSlice: boolean
+}
+
+// ABIStruct represents a Tolk struct.
+// Examples:
+//
+// > struct Point { x: int, y: int }
+// A simple struct, not serializable (because 'int', not 'int8' or similar)
+//
+// > struct Wrapper<TItem> { item: TItem }
+// A generic struct (has typeParams), fields[0].ty = { kind: 'genericT', nameT: 'TItem' }
+// 
+// > struct (0x12345678) Increment { ... }
+// Has a serialization prefix: prefixStr = '0x12345678', prefixLen = 32
+export interface ABIStruct {
+    kind: 'Struct'
+    name: string
+    typeParams?: string[]
+    prefix?: { 
+        prefixStr: string
+        prefixLen: number 
+    }
+    fields: {
+        name: string
+        ty: Ty
+        defaultValue?: ABIConstExpression
+        description?: string
+    }[]
+    customPackUnpack?: ABICustomSerializers
+}
+
+// ABIAlias represents a Tolk type alias.
+// Examples:
+// 
+// > type UserId = int32
+// A simple alias, targetTy = { kind: 'intN', n: 32 }
+//
+// > type Maybe<T> = MaybeNothing | MaybeJust<T>
+// A generic alias (has typeParams), targetTy = { kind: 'union', variants: ... }
+//
+// An alias is serialized as its target, unless it has custom serializers in Tolk code.
+export interface ABIAlias {
+    kind: 'Alias'
+    name: string
+    targetTy: Ty
+    typeParams?: string[]
+    customPackUnpack?: ABICustomSerializers
+}
+
+// ABIEnum represents a Tolk enum.
+// Examples:
+//
+// > enum Color { Red, Green, Blue }
+// Has 3 members (values '0', '1', '2'), encoded as 'uint2' (auto-calculated by the compiler).
+//
+// > enum Mode: int8 { User = 0, Admin = 127 }
+// Has 2 members, encoded as 'int8' (specified manually).
+export interface ABIEnum {
+    kind: 'Enum'
+    name: string
+    encodedAs: Ty
+    members: { 
+        name: string
+        value: bigint_as_string
+    }[]
+    customPackUnpack?: ABICustomSerializers
+}

package/src/abi.ts

@@ -0,0 +1,132 @@
+/*
+    ABI of a contract describes how the contract is "seen" by external observers:
+  its get methods, incoming messages, etc.
+    Given an ABI, it becomes possible to
+    - post messages from a client, since ABI provides sufficient info for serialization
+    - generate wrappers for TypeScript and other languages
+    - render its storage in the explorer
+    - create a UI to interact with a contract from Web/IDE
+    - etc.
+    The ABI is emitted by the Tolk compiler, along with bytecode.
+ */
+
+import { Ty, ABIStruct, ABIAlias, ABIEnum, ABIConstExpression } from './abi-types'
+
+// ABIGetMethod is a "get method" (aka "contract getter").
+// In Tolk code, getters are created with `get fun`.
+// Example:
+// > get fun calcData(owner: address): SomeStruct { ... }
+// It has one parameter with ty = { kind: 'address' },
+// its returnTy is { kind: 'StructRef', structName: 'SomeStruct' }.
+// Note, that getters are called off-chain — via the TVM stack, not via serialization.
+// (For instance, they can return 'int' or 'slice', although they are not serializable)
+export interface ABIGetMethod {
+    tvmMethodId: number
+    name: string
+    parameters: {
+        name: string
+        ty: Ty
+        description?: string
+        defaultValue?: ABIConstExpression
+    }[]
+    returnTy: Ty
+    description?: string
+}
+
+// ABIInternalMessage is "an incoming internal message" (handled by `onInternalMessage`).
+// In practice, a user describes each message as a struct:
+// > struct (0x12345678) Increment { ... }
+// > struct (0x23456789) Reset { ... }
+// Then, ABI of a contract will contain those two messages:
+// * bodyTy = { kind: 'StructRef', structName: 'Increment' }
+// * bodyTy = { kind: 'StructRef', structName: 'Reset' }
+// Theoretically, bodyTy can be something else: e.g., instantiation `Transfer<ForwardPayload>`.
+// Description, minimalMsgValue, and other properties may be set with an `@abi` attribute:
+// > @abi(...)
+// > struct (0x12345678) Increment { ... }
+export interface ABIInternalMessage {
+    bodyTy: Ty
+    description?: string
+    minimalMsgValue?: number
+    preferredSendMode?: number
+}
+
+// ABIExternalMessage is "an incoming external message" (handled by `onExternalMessage`).
+// It's either a 'slice' or some struct.
+export interface ABIExternalMessage {
+    bodyTy: Ty
+    description?: string
+}
+
+// ABIOutgoingMessage is "an outgoing internal/external message".
+// In Tolk code, those are calls to `createMessage`.
+export interface ABIOutgoingMessage {
+    bodyTy: Ty
+    description?: string
+    // todo need BounceMode? if yes, what if 2 messages of type T sent with different modes?
+}
+
+// ABIStorage defines shape of a storage.
+// Most often, it's a regular struct, serializable into a cell.
+// In the case of NFT, when a storage changes its shape (several fields appear after deployment),
+// the "initial storage" can also be expressed: it's called "storage at deployment".
+// The storage is used to visualize current contract state and to calculate its address.
+export interface ABIStorage {
+    storageTy?: Ty
+    storageAtDeploymentTy?: Ty
+}
+
+// ABIThrownError is an errCode fired by `throw` or `assert` in Tolk code.
+export interface ABIThrownError {
+    kind: 'plainInt' | 'constant' | 'enumMember'
+    name?: string
+    errCode: number
+}
+
+// ContractABI is a final result — the ABI of a TON smart contract.
+//
+// Partially, its properties may be specified by a user manually:
+// > contract MyName {
+// >    author: "Dima"
+// >    incomingMessages: SomeUnion
+// > }
+//
+// Partially, its properties are automatically calculated by the compiler:
+// - outgoingMessages, via the calls to `createMessage`
+// - thrownErrors, via `throw` and `assert` statements
+// - getMethods, essentially `get fun`
+//
+// Auto-calculated properties may be manually enriched via the `@abi` annotation:
+// > @abi({
+// >    description: "...",
+// >    minimalMsgValue: ton("0.05"),
+// > })
+// > struct SomeOutgoingMessage { ... }
+//
+// todo opened questions:
+// - clientTy for struct fields
+// - potentially missing struct fields, especially for describing FunC
+// - union types are not supported in get methods now, they have complicated stack layout
+// - TEPs and namespaces?
+export interface ContractABI {
+    contractName: string
+    author?: string
+    version?: string
+    description?: string
+
+    declarations: (ABIStruct | ABIAlias | ABIEnum)[]
+
+    storage: ABIStorage
+    incomingMessages: ABIInternalMessage[]
+    incomingExternal: ABIExternalMessage[]
+    outgoingMessages: ABIOutgoingMessage[]
+    emittedEvents: ABIOutgoingMessage[]
+    getMethods: ABIGetMethod[]
+    thrownErrors: ABIThrownError[]
+
+    compilerName: string
+    compilerVersion: string
+
+    // todo is it a part of ABI? the same question for source maps in the future
+    codeBoc64: string
+}

package/src/cli-generate-from-abi-json.ts

@@ -0,0 +1,21 @@
+import { generateTypeScriptFromAbiJson } from './generate-from-abi-json';
+
+const HELP_TEXT = `Usage:
+  generator '<contract-abi-json>'
+
+Prints the generated TypeScript wrapper to stdout.`;
+
+function parseCliArgs(argv: string[]): string {
+    if (argv.length !== 1) {
+        throw new Error(`Expected a single positional ABI JSON argument.\n\n${HELP_TEXT}`);
+    }
+    const abiJson = argv[0];
+    if (abiJson.length === 0) {
+        throw new Error(`ABI JSON argument is required.\n\n${HELP_TEXT}`);
+    }
+    return abiJson;
+}
+
+export async function runGeneratorFromAbiCli(argv: string[]): Promise<void> {
+    process.stdout.write(generateTypeScriptFromAbiJson(parseCliArgs(argv)));
+}

package/src/codegen-ctx.ts

@@ -0,0 +1,115 @@
+import type { Ty, ABIStruct, ABIAlias, ABIEnum } from './abi-types';
+import { SymbolNotFound } from './unsupported-errors';
+
+// Functions implemented in an output template, used to detect name shadowing.
+export const RUNTIME = {
+    beginCell: 'beginCell',
+    lookupPrefix: 'lookupPrefix',
+    lookupPrefixAndEat: 'lookupPrefixAndEat',
+    throwNonePrefixMatch: 'throwNonePrefixMatch',
+    loadAndCheckPrefix32: 'loadAndCheckPrefix32',
+    loadAndCheckPrefix: 'loadAndCheckPrefix',
+    loadTolkBitsN: 'loadTolkBitsN',
+    storeTolkBitsN: 'storeTolkBitsN',
+    loadTolkAddressAny: 'loadTolkAddressAny',
+    storeTolkAddressAny: 'storeTolkAddressAny',
+    loadTolkRemaining: 'loadTolkRemaining',
+    storeTolkRemaining: 'storeTolkRemaining',
+    loadCellRef: 'loadCellRef',
+    storeCellRef: 'storeCellRef',
+    loadArrayOf: 'loadArrayOf',
+    storeArrayOf: 'storeArrayOf',
+    loadLispListOf: 'loadLispListOf',
+    storeLispListOf: 'storeLispListOf',
+    storeTolkNullable: 'storeTolkNullable',
+    makeCellFrom: 'makeCellFrom',
+    createDictionaryValue: 'createDictionaryValue',
+    registerCustomPackUnpack: 'registerCustomPackUnpack',
+    ExtraSendOptions: 'ExtraSendOptions',
+    StackReader: 'StackReader',
+};
+
+// SymTable is an index of top-level declarations.
+export class SymTable {
+    private structs = new Map<string, ABIStruct>()
+    private aliases = new Map<string, ABIAlias>()
+    private enums = new Map<string, ABIEnum>()
+
+    constructor(declarations: (ABIStruct | ABIAlias | ABIEnum)[]) {
+        for (const n of declarations) {
+            if (n.kind === 'Struct')     this.structs.set(n.name, n);
+            else if (n.kind === 'Alias') this.aliases.set(n.name, n);
+            else if (n.kind === 'Enum')  this.enums.set(n.name, n);
+        }
+    }
+
+    getStruct(structName: string): ABIStruct {
+        const s = this.structs.get(structName);
+        if (s === undefined)
+            throw new SymbolNotFound('struct', structName);
+        return s;
+    }
+
+    getAlias(aliasName: string): ABIAlias {
+        const a = this.aliases.get(aliasName);
+        if (a === undefined)
+            throw new SymbolNotFound('alias', aliasName);
+        return a;
+    }
+
+    getEnum(enumName: string): ABIEnum {
+        const e = this.enums.get(enumName);
+        if (e === undefined)
+            throw new SymbolNotFound('enum', enumName);
+        return e;
+    }
+
+    getAliasTarget(aliasName: string): Ty {
+        return this.getAlias(aliasName).targetTy;
+    }
+}
+
+// CodegenCtx represents current state of TypeScript code generation.
+// It contains properties that affect final output and help resolve symbols.
+// For example, we want Tolk `int64` not be just TS `bigint`, but
+// to export `type int64 = bigint` and use `int64` directly,
+// that's why we track all occurred intN.
+// Same for other public properties, they are modified while iterating through declarations.
+export class CodegenCtx {
+    readonly symbols: SymTable
+
+    intNOccurred = new Set<string>(['int8', 'int16', 'int32', 'int256'])
+    uintNOccurred = new Set<string>(['uint8', 'uint16', 'uint32', 'uint256'])
+    varIntNOccurred = new Set<string>()
+    bitsNOccurred = new Set<string>()
+
+    has_RemainingBitsAndRefs = false
+    has_customPackUnpack = false
+    has_customDictV = false
+    has_implicitUnionPrefix = false
+    has_non32Prefixes = false
+    has_addressAny = false
+    has_arrayOf = false
+    has_lispListOf = false
+
+    stackReadsUnknown = false
+    stackReadsArrayOf = false
+    stackReadsLispListOf = false
+    stackReadsSnakeString = false
+    stackReadsTuple = false
+    stackReadsMapKV = false
+    stackReadsBuilder = false
+    stackReadsNullable = false
+    stackReadsWideNullable = false
+    stackReadsUnionType = false
+    stackReadsCellRef = false
+    stackReadsNullLiteral = false
+
+    constructor(declarations: (ABIStruct | ABIAlias | ABIEnum)[]) {
+        this.symbols = new SymTable(declarations);
+    }
+
+    sortOccurred(occurred: Set<string>): string[] {
+        return [...occurred].sort((a, b) => a.length - b.length || a.localeCompare(b));
+    }
+}

package/src/dynamic-ctx.ts

@@ -0,0 +1,55 @@
+import * as c from '@ton/core'
+import { SymTable } from './codegen-ctx'
+import { ABIGetMethod, ContractABI } from './abi'
+import { CantPackDynamic, CantUnpackDynamic } from './unsupported-errors'
+
+type CustomPackToBuilderFn<T> = (self: T, b: c.Builder) => void
+type CustomUnpackFromSliceFn<T> = (s: c.Slice) => T
+
+export class DynamicCtx {
+    readonly contractName: string;
+    readonly getMethods: ABIGetMethod[];
+    readonly symbols: SymTable;
+
+    private customSerializersRegistry: Map<string, [CustomPackToBuilderFn<any> | null, CustomUnpackFromSliceFn<any> | null]> = new Map;
+
+    constructor(contract: ContractABI) {
+        this.contractName = contract.contractName;
+        this.getMethods = contract.getMethods;
+        this.symbols = new SymTable(contract.declarations);
+    }
+
+    registerCustomPackUnpack<T>(
+        typeName: string,
+        packToBuilderFn: CustomPackToBuilderFn<T> | null,
+        unpackFromSliceFn: CustomUnpackFromSliceFn<T> | null,
+    ) {
+        if (this.customSerializersRegistry.has(typeName)) {
+            throw new Error(`Custom pack/unpack for '${typeName}' already registered`);
+        }
+        this.customSerializersRegistry.set(typeName, [packToBuilderFn, unpackFromSliceFn]);
+    }
+
+    getCustomPackFnOrThrow(typeName: string, fieldPath: string): CustomPackToBuilderFn<any> {
+        let packUnpack = this.customSerializersRegistry.get(typeName);
+        let packFn = packUnpack ? packUnpack[0] : null;
+        if (!packFn) {
+            throw new CantPackDynamic(fieldPath, `custom packToBuilder was not registered for '${typeName}'`);
+        }
+        return packFn;
+    }
+
+    getCustomUnpackFnOrThrow(typeName: string, fieldPath: string): CustomUnpackFromSliceFn<any> {
+        let packUnpack = this.customSerializersRegistry.get(typeName);
+        let unpackFn = packUnpack ? packUnpack[1] : null;
+        if (!unpackFn) {
+            throw new CantUnpackDynamic(fieldPath, `custom unpackFromSlice was not registered for '${typeName}'`);
+        }
+        return unpackFn;
+    }
+
+    findGetMethod(getMethodName: string): ABIGetMethod | undefined {
+        return this.getMethods.find(m => m.name === getMethodName);
+    }
+}
+