@typesol/cli 0.22.0 → 0.24.0

package/dist/compiler/emit/expressions.js

@@ -128,7 +128,7 @@ export class ExpressionEmitter extends StorageEmitter {
         if (ts.isNonNullExpression(node)) {
             const v = this.expr(node.expression, hint && hint.kind !== "option" ? { kind: "option", inner: hint } : hint);
             if (v.type.kind !== "option")
-                this.fail(node, "! works on Option values; check optional accounts with `x !== null`");
+                this.fail(node, "! works on Option values and optional accounts (acct!.field); check other values with `x !== null`");
             return this.unwrap(v, node);
         }
         if (ts.isPropertyAccessExpression(node) && node.questionDotToken)
@@ -154,6 +154,8 @@ export class ExpressionEmitter extends StorageEmitter {
         if (ts.isTemplateExpression(node)) {
             // `a${x}b${y}c`: one join on the heap (or a literal when every piece is known at compile time).
             const pieces = this.templatePieces(node);
+            if (pieces.only !== undefined)
+                return { c: pieces.only, type: STRING };
             if (pieces.text !== undefined)
                 return { c: `ts_str((const uint8_t *)${cString(pieces.text)}, ${Buffer.byteLength(pieces.text, "utf8")})`, type: STRING, stringLiteral: pieces.text };
             const t = this.tmp("tpl");
@@ -171,9 +173,11 @@ export class ExpressionEmitter extends StorageEmitter {
             const l = this.lookup(node.text);
             if (l) {
                 if (l.closure)
-                    this.fail(node, `"${node.text}" is a function: TypeSOL functions are called (${node.text}(…)), not passed around or stored`);
+                    this.fail(node, `"${node.text}" is a function: call it (${node.text}(…)), bind it with const, or pass it to a helper or list method; it cannot be stored or returned as a value`);
                 if (l.type.kind === "array")
                     this.fail(node, `"${node.text}" is an array; use .get(i)`);
+                if (l.type.kind === "vec")
+                    this.fail(node, `"${node.text}" is a list; use .get(i) and .length`);
                 if (l.type.kind === "remaining")
                     return { c: "", type: l.type };
                 return l.type.kind === "pubkey" && !l.ptr ? { c: `(&${l.c})`, type: l.type } : { c: l.c, type: l.type };
@@ -202,7 +206,7 @@ export class ExpressionEmitter extends StorageEmitter {
             if (this.accounts.has(node.text))
                 this.fail(node, `"${node.text}" is an account; use ${node.text}.key for its address`);
             if (calleeName(node, this.model.helperDecls, this.checker) || this.model.helpers.has(node.text))
-                this.fail(node, `"${node.text}" is a function: TypeSOL functions are called (${node.text}(…)), not passed around or stored`);
+                this.fail(node, `"${node.text}" is a function: call it (${node.text}(…)), bind it with const, or pass it to a helper or list method; it cannot be stored or returned as a value`);
             this.fail(node, `unknown name "${node.text}"`);
         }
         if (ts.isPropertyAccessExpression(node))
@@ -261,6 +265,13 @@ export class ExpressionEmitter extends StorageEmitter {
         if (field === "length") {
             if (this.isRemaining(node.expression))
                 return { c: `((uint64_t)(params->ka_num - ${this.ix.accounts.length}))`, type: U64 };
+            // The number of elements a map / filter chain gives: counted, nothing stored.
+            if (this.isPipeline(node.expression)) {
+                const n = this.tmp("count");
+                this.line(`uint64_t ${n} = 0;`);
+                this.eachElement(node.expression, node, { index: `__ln${this.temp++}`, ci: this.tmp("ln_i") }, () => this.line(`${n}++;`));
+                return { c: n, type: U64 };
+            }
             const arr = this.ref(node.expression);
             if (arr?.type.kind === "array")
                 return { c: `${arr.type.len}ULL`, type: U64 };
@@ -446,16 +457,20 @@ export class ExpressionEmitter extends StorageEmitter {
                 lit(known);
             else {
                 const mark = this.lines.length;
-                const c = this.toText(span.expression);
+                const converted = { yes: false };
+                const c = this.toText(span.expression, converted);
                 for (const p of parts)
                     if (p.c !== undefined)
                         p.c = this.settle({ c: p.c, type: STRING }, mark).c;
-                parts.push({ c });
+                parts.push({ c, converted: converted.yes });
             }
             lit(span.literal.text);
         }
         if (parts.every((p) => p.text !== undefined))
             return { text: parts.map((p) => p.text).join("") };
+        // `${n}` alone: the text a conversion just made (or a constant one) is already a string of its own; no join.
+        if (parts.length === 1 && parts[0].converted)
+            return { only: parts[0].c };
         const strOf = (text) => `ts_str((const uint8_t *)${cString(text)}, ${Buffer.byteLength(text, "utf8")})`;
         const arr = this.tmp("parts");
         this.line(`TS_Str ${arr}[${parts.length}] = {${parts.map((p) => (p.text !== undefined ? strOf(p.text) : p.c)).join(", ")}};`);
@@ -485,10 +500,13 @@ export class ExpressionEmitter extends StorageEmitter {
         }
         return undefined;
     }
-    toText(node) {
+    /** `${node}`'s text; `converted.yes` when it was made from a non-string value (new text, not a view of other text). */
+    toText(node, converted) {
         const v = this.expr(node);
         if (v.type.kind === "string")
             return v.c;
+        if (converted)
+            converted.yes = true;
         if (v.type.kind === "option" && (v.type.inner.kind === "int" || v.type.inner.kind === "bool")) {
             // `${opt}`: the value, or "null" like JS.
             const inner = v.type.inner;
@@ -548,12 +566,7 @@ export class ExpressionEmitter extends StorageEmitter {
             (ts.isBigIntLiteral(node.whenFalse) || ts.isPrefixUnaryExpression(node.whenFalse));
         if (!type && !bothLiteral && (ts.isBigIntLiteral(node.whenTrue) || node.whenTrue.kind === ts.SyntaxKind.NullKeyword)) {
             // Type the non-literal branch first, without emitting it, to give the literal its type.
-            const saved = { lines: this.lines.length, hoisted: this.hoisted.length, temp: this.temp, diagnostics: this.reporter.diagnostics.length };
-            const probe = this.reporter.guard(() => this.expr(node.whenFalse));
-            this.lines.length = saved.lines;
-            this.hoisted.length = saved.hoisted; // a probed helper call may hoist declarations
-            this.temp = saved.temp;
-            this.reporter.diagnostics.length = saved.diagnostics;
+            const probe = this.probe(() => this.expr(node.whenFalse));
             if (probe && probe.literal === undefined && !probe.isNull)
                 type = probe.type;
         }

package/dist/compiler/emit/function.js

@@ -3,7 +3,7 @@ import { snakeCase } from "../analyze.js";
 import { decodeAddress } from "../../keys.js";
 import { INT_INFO, isScalar, isWritable, scalarSize, typeName, } from "../model.js";
 import { borshIdentical, borshMinSize } from "./borsh.js";
-import { MAX_ACCOUNTS, ATA_PROGRAM_BYTES, cBytes, cString, optC, fieldC, storageDecl, helperSignature } from "./shared.js";
+import { MAX_ACCOUNTS, ATA_PROGRAM_BYTES, cBytes, cString, optC, fieldC, storageDecl, helperSignature, emptySizeC, vecLenCheck } from "./shared.js";
 import { StatementEmitter } from "./statements.js";
 /**
  * One instruction or helper as a C function: the instruction prologue (account checks, constraints and
@@ -42,7 +42,7 @@ export class FunctionEmitter extends StatementEmitter {
                 head.push(`  TS_TRY(ts_check_writable(${v}));`);
             // A @layout borsh account only has to hold its smallest Borsh form (unpacking checks the rest).
             if (k.kind === "account")
-                head.push(`  TS_TRY(ts_check_program_account(${v}, params->program_id, TS_DISC_${k.struct.name}, ${k.struct.layout === "borsh" ? borshMinSize(k.struct, this.model) : `sizeof(TSS_${k.struct.name})`}));`);
+                head.push(`  TS_TRY(ts_check_program_account(${v}, params->program_id, TS_DISC_${k.struct.name}, ${k.struct.layout === "borsh" ? borshMinSize(k.struct, this.model) : emptySizeC(k.struct)}));`);
             // A token account created by this instruction is checked where it is created (create/createIfNeeded).
             if (k.kind === "token" && !k.init)
                 head.push(`  TS_TRY(ts_check_token_account(${v}));`);
@@ -106,7 +106,7 @@ export class FunctionEmitter extends StatementEmitter {
                 if (k.struct.vec) {
                     head.push(`  uint64_t entry_${a.name} = ${v}->data_len;`, `  (void)entry_${a.name};`);
                     if (k.kind === "account")
-                        head.push(`  TS_TRY(ts_check_vec_len(${v}, sizeof(TSS_${k.struct.name}), d_${a.name}->${k.struct.vec.name}__len, TS_ELEM_${k.struct.name}));`);
+                        head.push(...vecLenCheck(k.struct, v, `d_${a.name}`));
                 }
             }
             // realloc(x, …) needs the size the account had when the instruction started (the 10 KiB growth limit).
@@ -150,14 +150,16 @@ export class FunctionEmitter extends StatementEmitter {
         // The loader passes each account once and marks repeats as duplicates, which share its record (ts_deserialize),
         // so two parameters name the same account exactly when their pointers are equal.
         const writable = ix.accounts.filter((a) => isWritable(a.kind));
+        const pairs = [];
         for (let i = 0; i < writable.length; i++)
             for (let j = i + 1; j < writable.length; j++) {
                 const present = [writable[i], writable[j]]
                     .filter((a) => a.optional)
                     .map((a) => `a_${a.name} && `)
                     .join("");
-                head.push(`  if (${present}a_${writable[i].name} == a_${writable[j].name}) TS_FAIL(TS_ERR_DUPLICATE_MUT, "TypeSOL: ${writable[i].name} and ${writable[j].name} must be different accounts");`);
+                pairs.push(`if (${present}a_${writable[i].name} == a_${writable[j].name}) TS_FAIL(TS_ERR_DUPLICATE_MUT, "TypeSOL: ${writable[i].name} and ${writable[j].name} must be different accounts");`);
             }
+        head.push(...pairs.map((p) => `  ${p}`));
         // Arguments: Borsh, read sequentially (strings make later offsets dynamic).
         if (ix.args.length)
             head.push(`  uint64_t off = ${ix.discriminator.length};`);
@@ -219,6 +221,8 @@ export class FunctionEmitter extends StatementEmitter {
                 const size = this.fieldBytes(elem);
                 need("4");
                 head.push(`  TS_VecArg arg_${a.name};`, `  arg_${a.name}.len = (uint32_t)ts_read_le(${at}, 4);`, `  off += 4;`);
+                if (a.type.cap !== undefined)
+                    head.push(`  if (arg_${a.name}.len > ${a.type.cap}) TS_FAIL(TS_ERR_INSTRUCTION_DATA, "TypeSOL: more elements than the FixedVec argument holds");`);
                 need(`(uint64_t)arg_${a.name}.len * ${size}`);
                 head.push(`  arg_${a.name}.ptr = ${at};`, `  off += (uint64_t)arg_${a.name}.len * ${size};`);
                 // Struct elements are read in place, so check them all now; scalars are checked when read.

package/dist/compiler/emit/program.js

@@ -1,4 +1,4 @@
-import { MAX_ACCOUNTS, cBytes, scalarStorageC, optC, storageDecl, helperSignature, fixedVecC } from "./shared.js";
+import { MAX_ACCOUNTS, cBytes, scalarStorageC, optC, storageDecl, helperSignature, fixedVecC, laterVecC } from "./shared.js";
 import { FunctionEmitter } from "./function.js";
 import { emitBorshCodecs } from "./borsh.js";
 /**
@@ -96,9 +96,14 @@ export function emitProgram(analysis, opts = {}) {
             return;
         emitted.add(s.name);
         s.fields.forEach((f) => deps(f.type));
+        // Growable Vecs after the first live past the C struct (see StructDef.vecs): a header type each.
+        const later = new Set((s.vecs ?? []).slice(1).map((v) => v.name));
+        for (const v of (s.vecs ?? []).slice(1))
+            out.push(`typedef struct __attribute__((packed)) {`, `  uint32_t len;`, `  ${storageDecl(v.elem, "items[]")};`, `} ${laterVecC(s.name, v.name)};`);
         out.push(`typedef struct __attribute__((packed)) {`);
         for (const f of s.fields)
-            out.push(`  ${storageDecl(f.type, f.name)};`);
+            if (!later.has(f.name))
+                out.push(`  ${storageDecl(f.type, f.name)};`);
         if (!s.fields.length)
             out.push(`  uint8_t __empty[0];`); // a marker account (Anchor `pub struct NftAuthority {}`): discriminator only
         out.push(`} TSS_${s.name};`);
@@ -113,6 +118,8 @@ export function emitProgram(analysis, opts = {}) {
         emitStruct(s);
     for (const e of model.dataEnums.values())
         deps({ kind: "dataEnum", name: e.name });
+    for (const t of model.localVecs.values())
+        deps(t);
     for (const k of model.constants.values())
         if (typeof k.value === "object")
             out.push(`static const SolPubkey TSK_${k.name} = {${cBytes(k.value.pubkey)}};`);

package/dist/compiler/emit/shared.d.ts

@@ -1,5 +1,5 @@
 import ts from "typescript";
-import { type AccountParam, type FieldType, type Helper, type IntName, type ParamKind, type ScalarType, type ValueType } from "../model.js";
+import { type AccountParam, type FieldType, type Helper, type IntName, type ParamKind, type ScalarType, type StructDef, type ValueType } from "../model.js";
 /** Maximum accounts the generated entrypoint deserializes. */
 export declare const MAX_ACCOUNTS = 32;
 export declare const MAX_SEEDS = 15;
@@ -46,16 +46,27 @@ export interface Ref {
     vec?: {
         len: string;
         items: string;
+        /** The account's size without this Vec's elements (so with `n` of them: 8 + fixed + n × elemSize). */
         fixed: string;
         elemSize: number;
         resizable: boolean;
         cap?: number;
+        /** Growable Vecs stored after this one move when it grows or shrinks: where its elements end, and their paths. */
+        tail?: {
+            end: string;
+            dataEnd: string;
+            paths: string[];
+        };
     };
 }
 export interface Local {
     c: string;
+    /** A value, or a local FixedArray / FixedVec (`fixedArray<T, N>(…)`, `const ys = xs.map(…)`) held in place. */
     type: ValueType | (FieldType & {
         kind: "array";
+    }) | (FieldType & {
+        kind: "vec";
+        cap: number;
     });
     mutable: boolean;
     /** Pubkey held as a `const SolPubkey *` (helper parameters) rather than a value. */
@@ -64,15 +75,26 @@ export interface Local {
     readonly?: boolean;
     /** For struct / data-enum references: the storage paths they alias (e.g. "book.asks[]"), and the declaration. */
     paths?: string[];
-    decl?: ts.VariableDeclaration;
+    decl?: ts.VariableDeclaration | ts.ParameterDeclaration;
     /** A local closure (`const f = (…) => …` reading the enclosing function's variables), expanded where it is called. */
     closure?: Closure;
 }
 /** A closure and the scopes it was declared in (its free names resolve there, wherever it is called). */
 export interface Closure {
-    node: ts.ArrowFunction | ts.FunctionExpression;
+    /** The function expanded: a closure, or a helper expanded where it is called because it was given one. */
+    node: ts.ArrowFunction | ts.FunctionExpression | ts.FunctionDeclaration;
     name: string;
     scopes: Map<string, Local>[];
+    /**
+     * The instruction arguments and accounts its body may name: those where it was written for a closure, none for a
+     * helper or a function a helper made (they see only their parameters and the module).
+     */
+    env: {
+        args: Map<string, Local>;
+        accounts: Map<string, AccountParam>;
+    };
+    /** The `typesol` imports of the file it is written in: its type names mean what they mean there. */
+    imports: Map<string, string>;
 }
 export declare const cBytes: (bytes: number[]) => string;
 /** C expressions that nothing in a TypeSOL expression can change: temporaries, locals, arguments, account keys, constants. */
@@ -102,6 +124,24 @@ export interface EmitOptions {
     /** Log a marker and the remaining compute units before every statement (`typesol build --profile`). */
     profile?: boolean;
 }
+/** C type of a growable Vec stored after the first one of struct `s`: `{ uint32_t len; T items[]; }`. */
+export declare function laterVecC(s: string, field: string): string;
+/** The account size of struct `s` with every growable Vec empty, as C (8-byte discriminator not included). */
+export declare function emptySizeC(s: StructDef): string;
+/**
+ * Where growable Vec `k` of a struct at `base` (a C lvalue of type TSS_<S>) starts, as C: its length and elements.
+ * The first is a member of the struct; each further one starts right after the previous one's elements.
+ */
+export declare function vecPlace(s: StructDef, base: string, k: number): {
+    len: string;
+    items: string;
+    end: string;
+};
+/**
+ * C lines checking that an account's size matches its growable Vecs' lengths (`d`: its TSS_<S> pointer, `a`: its
+ * TsAccount). One Vec: ts_check_vec_len; more: each length read only once the bytes before it are known to exist.
+ */
+export declare function vecLenCheck(s: StructDef, a: string, d: string, pad?: string): string[];
 /** C typedef name of a FixedVec<T, N>: `{ uint32_t len; T items[N]; }`. */
 export declare function fixedVecC(t: FieldType & {
     kind: "vec";

package/dist/compiler/emit/shared.js

@@ -202,6 +202,49 @@ export function helperSignature(h) {
         params.push(h.returns.kind === "struct" ? `TSS_${h.returns.name} *ret` : h.returns.kind === "string" ? `TS_Str *ret` : `${scalarC(h.returns)} *ret`);
     return `static uint64_t fn_${h.name}(${params.join(", ")})`;
 }
+/** C type of a growable Vec stored after the first one of struct `s`: `{ uint32_t len; T items[]; }`. */
+export function laterVecC(s, field) {
+    return `TSW_${s}__${field}`;
+}
+/** The account size of struct `s` with every growable Vec empty, as C (8-byte discriminator not included). */
+export function emptySizeC(s) {
+    const n = s.vecs?.length ?? 0;
+    return n > 1 ? `(sizeof(TSS_${s.name}) + ${4 * (n - 1)})` : `sizeof(TSS_${s.name})`;
+}
+/**
+ * Where growable Vec `k` of a struct at `base` (a C lvalue of type TSS_<S>) starts, as C: its length and elements.
+ * The first is a member of the struct; each further one starts right after the previous one's elements.
+ */
+export function vecPlace(s, base, k) {
+    const vs = s.vecs;
+    const first = vs[0];
+    let end = `((uint8_t *)${base}.${first.name} + (uint64_t)${base}.${first.name}__len * ${first.elemSize})`;
+    let place = { len: `${base}.${first.name}__len`, items: `${base}.${first.name}`, end };
+    for (let j = 1; j <= k; j++) {
+        const v = vs[j];
+        const h = `((${laterVecC(s.name, v.name)} *)${end})`;
+        end = `((uint8_t *)${h}->items + (uint64_t)${h}->len * ${v.elemSize})`;
+        place = { len: `${h}->len`, items: `${h}->items`, end };
+    }
+    return place;
+}
+/**
+ * C lines checking that an account's size matches its growable Vecs' lengths (`d`: its TSS_<S> pointer, `a`: its
+ * TsAccount). One Vec: ts_check_vec_len; more: each length read only once the bytes before it are known to exist.
+ */
+export function vecLenCheck(s, a, d, pad = "  ") {
+    const vs = s.vecs ?? [];
+    if (vs.length === 1)
+        return [`${pad}TS_TRY(ts_check_vec_len(${a}, sizeof(TSS_${s.name}), ${d}->${vs[0].name}__len, TS_ELEM_${s.name}));`];
+    const o = `${d.replace(/\W/g, "")}__vo`;
+    const lines = [`${pad}{`, `${pad}  uint64_t ${o} = sizeof(TSS_${s.name}) + (uint64_t)${d}->${vs[0].name}__len * ${vs[0].elemSize};`];
+    for (const v of vs.slice(1)) {
+        lines.push(`${pad}  if (${a}->data_len < 8 + ${o} + 4) TS_FAIL(TS_ERR_ACCOUNT_TOO_SMALL, "TypeSOL: account size does not match its Vec lengths");`);
+        lines.push(`${pad}  ${o} += 4 + (uint64_t)((${laterVecC(s.name, v.name)} *)(${a}->data + 8 + ${o}))->len * ${v.elemSize};`);
+    }
+    lines.push(`${pad}  if (${a}->data_len != 8 + ${o}) TS_FAIL(TS_ERR_ACCOUNT_TOO_SMALL, "TypeSOL: account size does not match its Vec lengths");`, `${pad}}`);
+    return lines;
+}
 /** C typedef name of a FixedVec<T, N>: `{ uint32_t len; T items[N]; }`. */
 export function fixedVecC(t) {
     return `TSF_${fixedVecName(t)}`;

package/dist/compiler/emit/statements.d.ts

@@ -1,5 +1,6 @@
 import ts from "typescript";
-import { type Value, type Closure } from "./shared.js";
+import { type FieldType } from "../model.js";
+import { type Value, type Closure, type Ref } from "./shared.js";
 import { CpiEmitter } from "./cpi.js";
 /**
  * Statements: variables, assignment, control flow and return.
@@ -17,6 +18,42 @@ export declare abstract class StatementEmitter extends CpiEmitter {
      * As in JS, the length is read again before every step, so a body that pushes or pops sees the new length.
      */
     protected forOf(stmt: ts.ForOfStatement): void;
+    /** The function an identifier names (a closure, a local function or a helper), as something to expand; else undefined. */
+    protected functionNamed(id: ts.Identifier): Closure | undefined;
+    /** `xs.map(…)` / `xs.filter(…)` (or a chain of them): a list built element by element, not stored anywhere yet. */
+    protected isPipeline(e: ts.Expression): e is ts.CallExpression;
+    /** The list under a chain of map / filter calls, and the calls in the order they run. */
+    protected pipelineStages(e: ts.Expression): {
+        base: ts.Expression;
+        stages: {
+            kind: "map" | "filter";
+            fn: ts.ArrowFunction | ts.FunctionExpression;
+        }[];
+    };
+    /**
+     * Run `body` for each element of a list, or of a chain of `map` / `filter` calls on one (`xs.filter(f).map(g)`):
+     * one index loop over the list (its length read every step, as `for … of`), each callback expanded in it. A
+     * filter skips to the next element, a map binds its result, so a chain builds no list. `body` gets the element (as
+     * `list.get(i)` gives it, structs by reference, or the mapped value) and its index in the chain's output (after a
+     * filter, the count of elements it let through, as JS gives callbacks).
+     */
+    protected eachElement(list: ts.Expression, at: ts.Node, names: {
+        index: string;
+        ci: string;
+        next?: string;
+        alwaysNext?: boolean;
+    }, body: (element: ts.Expression, index: ts.Expression) => void): void;
+    /**
+     * `const ys = xs.map(…)` / `xs.filter(…)`: the chain's elements in a local FixedVec, as big as the list it starts
+     * from (a FixedArray's length, a FixedVec's capacity) or as annotated (`const ys: FixedVec<u64, 16> = …`, needed
+     * when it starts from a Vec). It lives on the stack, so it holds at most 2 KiB; one more element than it holds aborts.
+     */
+    protected localList(d: ts.VariableDeclaration, init: ts.CallExpression, c: string): FieldType & {
+        kind: "vec";
+        cap: number;
+    };
+    /** The element type of a Vec, FixedArray, FixedVec or Vec argument. */
+    protected listElem(list: ts.Expression): FieldType;
     /** Whether `e` is the remaining accounts (or a const bound to them). */
     protected isRemaining(e: ts.Expression): boolean;
     /**
@@ -30,7 +67,7 @@ export declare abstract class StatementEmitter extends CpiEmitter {
      * was declared with, so a block around the call that declares the same name does not change what it reads, and
      * writes to captured `let`s are seen by the enclosing function. Returns the result (undefined for no value).
      */
-    protected expandClosure(c: Closure, args: readonly ts.Expression[], node: ts.Node): Value | undefined;
+    protected expandClosure(c: Closure, args: readonly ts.Expression[], node: ts.Node, into?: Ref): Value | undefined;
     /**
      * `xs.forEach / some / every / find / findIndex / reduce / includes / indexOf` on a Vec, FixedArray, FixedVec or Vec
      * argument: an index loop (the length read each step, as `for … of`), the callback (an arrow written at the call)