@aztec-foundation/constants-codegen 0.0.1-commit.b66364b

package/README.md

@@ -0,0 +1,49 @@
+# Constants codegen
+
+This directory will contain the standalone cross-language generator for Aztec protocol constants.
+
+## Version 1 interface
+
+The command reads a Noir source file and writes one of the supported outputs.
+
+```text
+constants-codegen \
+  [--input <constants.nr>] \
+  [--selection <selection.json>] \
+  (--typescript <output.ts> | --cpp <output.hpp> | --pil <output.pil> | --solidity <output.sol> | --rust <output.rs>)
+```
+
+- `--input` defaults to `inputs/constants.nr` under the package root, the file `scripts/embed-inputs.sh` copies in
+  as part of the package build, so both the published npm package and the built in-repo package work without
+  `--input`. In-repo callers invoke `scripts/generate.sh`, which refreshes the embedded copy from the monorepo
+  sources before running the CLI.
+- Exactly one output option is required. Run the command once per desired output.
+- `--selection` filters the output to the selected symbols. Without it, the output contains every supported symbol
+  from the input.
+- Relative paths given as arguments are resolved from the caller's working directory.
+- Invalid arguments, an unreadable input, an unsupported expression, or an output failure produce a diagnostic on
+  stderr and a nonzero exit status.
+
+A selection file is a JSON array naming Noir source symbols, including the `DOM_SEP__` prefix for domain
+separators:
+
+```json
+["ARCHIVE_HEIGHT", "MAX_.*_PER_TX", "DOM_SEP__MERKLE_HASH"]
+```
+
+An entry that is not a valid symbol name is treated as a regular expression selecting every symbol whose whole
+name matches it. Duplicate entries, invalid patterns, unknown symbols, and patterns that match no symbol are rejected.
+
+Rust emits all parsed constants and domain separators: values that fit `u128` become `pub const NAME: u128` items,
+and larger field-sized values become `pub const NAME: &str` hex-string items.
+
+## Compatibility target
+
+The implementation must preserve the symbols and values currently checked in at:
+
+- `yarn-project/constants/src/constants.gen.ts`
+- `barretenberg/cpp/src/barretenberg/aztec/aztec_constants.hpp`
+- `barretenberg/cpp/pil/vm2/constants_gen.pil`
+- `l1-contracts/src/core/libraries/ConstantsGen.sol`
+
+Generator instructions and formatter-only whitespace may change intentionally.

package/dest/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dest/cli.js

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+import { existsSync, mkdirSync, readFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
+import { evaluateExpressions, generateCppConstants, generatePilConstants, generateRustConstants, generateSolidityConstants, generateTypescriptConstants, parseNoirFile, } from "./generator.js";
+import { readSymbolSelection, selectSymbols } from "./selection.js";
+function run(args) {
+    const { values } = parseArgs({
+        args,
+        allowPositionals: false,
+        options: {
+            input: { type: 'string' },
+            selection: { type: 'string' },
+            typescript: { type: 'string' },
+            cpp: { type: 'string' },
+            pil: { type: 'string' },
+            solidity: { type: 'string' },
+            rust: { type: 'string' },
+        },
+        strict: true,
+    });
+    // The default input is embedded by scripts/embed-inputs.sh: prepack ships it in the published
+    // tarball, while in-repo callers pass --input (via scripts/generate.sh).
+    const defaultInput = fileURLToPath(new URL('../inputs/constants.nr', import.meta.url));
+    const input = values.input ?? (existsSync(defaultInput) ? defaultInput : undefined);
+    if (!input) {
+        throw new Error('--input is required when the package has no embedded inputs');
+    }
+    const generators = [
+        [values.typescript, generateTypescriptConstants],
+        [values.cpp, generateCppConstants],
+        [values.pil, generatePilConstants],
+        [values.solidity, generateSolidityConstants],
+        [values.rust, generateRustConstants],
+    ];
+    const outputs = generators.filter(([path]) => path !== undefined);
+    if (outputs.length !== 1) {
+        throw new Error('exactly one output option is required');
+    }
+    const [outputPath, generate] = outputs[0];
+    const { constantsExpressions, domainSeparatorEnum } = parseNoirFile(readFileSync(input, 'utf8'));
+    const parsedContent = {
+        constants: evaluateExpressions(constantsExpressions),
+        domainSeparatorEnum,
+    };
+    const outputContent = values.selection
+        ? selectSymbols(parsedContent, readSymbolSelection(values.selection))
+        : parsedContent;
+    mkdirSync(dirname(outputPath), { recursive: true });
+    generate(outputContent, outputPath);
+}
+try {
+    run(process.argv.slice(2));
+}
+catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    console.error(`constants-codegen: ${message}`);
+    process.exitCode = 1;
+}

package/dest/generator.d.ts

@@ -0,0 +1,131 @@
+/**
+ * Parsed content.
+ */
+export interface ParsedContent {
+    /**
+     * Constants of the form "CONSTANT_NAME: number_as_string".
+     */
+    constants: {
+        [key: string]: string;
+    };
+    /**
+     * DomainSeparatorEnum.
+     */
+    domainSeparatorEnum: {
+        [key: string]: number;
+    };
+}
+/**
+ * Raw expressions parsed from a Noir file, prior to evaluation. Keeping expressions unevaluated lets callers merge
+ * constants from multiple files and resolve cross-file references in a single evaluation pass.
+ */
+export interface ParsedExpressions {
+    /** Ordered list of constant name and expression pairs. */
+    constantsExpressions: [string, string][];
+    /** DomainSeparatorEnum members. */
+    domainSeparatorEnum: {
+        [key: string]: number;
+    };
+}
+/**
+ * Processes a collection of constants and generates code to export them as TypeScript constants.
+ *
+ * @param constants - An object containing key-value pairs representing constants.
+ * @returns A string containing code that exports the constants as TypeScript constants.
+ */
+export declare function processConstantsTS(constants: {
+    [key: string]: string;
+}): string;
+/**
+ * Processes a collection of constants and generates code to export them as cpp constants.
+ * Required to ensure consistency between the constants used in pil and used in the vm witness generator.
+ *
+ * @param constants - An object containing key-value pairs representing constants.
+ * @returns A string containing code that exports the constants as cpp constants.
+ */
+export declare function processConstantsCpp(constants: {
+    [key: string]: string;
+}, generatorIndices: {
+    [key: string]: number;
+}): string;
+/**
+ * Processes a collection of constants and generates code to export them as PIL constants.
+ * Required to ensure consistency between the constants used in pil and used in the vm witness generator.
+ *
+ * @param constants - An object containing key-value pairs representing constants.
+ * @returns A string containing code that exports the constants as cpp constants.
+ */
+export declare function processConstantsPil(constants: {
+    [key: string]: string;
+}, generatorIndices: {
+    [key: string]: number;
+}): string;
+/**
+ * Processes an enum and generates code to export it as a TypeScript enum.
+ *
+ * @param enumName - The name of the enum.
+ * @param enumValues - An object containing key-value pairs representing enum values.
+ * @returns A string containing code that exports the enum as a TypeScript enum.
+ */
+export declare function processEnumTS(enumName: string, enumValues: {
+    [key: string]: number;
+}): string;
+/**
+ * Processes a collection of constants and generates code to export them as Solidity constants.
+ *
+ * @param constants - An object containing key-value pairs representing constants.
+ * @param generatorIndices - An object containing key-value pairs representing domain separator indices.
+ * @param prefix - A prefix to add to the constant names.
+ * @returns A string containing code that exports the constants as Noir constants.
+ */
+export declare function processConstantsSolidity(constants: {
+    [key: string]: string;
+}, generatorIndices: {
+    [key: string]: number;
+}, prefix?: string): string;
+/**
+ * Processes a collection of constants and generates code to export them as Rust constants.
+ *
+ * @param constants - An object containing key-value pairs representing constants.
+ * @param generatorIndices - An object containing key-value pairs representing domain separator indices.
+ * @returns A string containing code that exports the constants as Rust constants.
+ */
+export declare function processConstantsRust(constants: {
+    [key: string]: string;
+}, generatorIndices: {
+    [key: string]: number;
+}): string;
+/**
+ * Generate the constants file in Typescript.
+ */
+export declare function generateTypescriptConstants({ constants, domainSeparatorEnum }: ParsedContent, targetPath: string): void;
+/**
+ * Generate the constants file in C++.
+ */
+export declare function generateCppConstants({ constants, domainSeparatorEnum }: ParsedContent, targetPath: string): void;
+/**
+ * Generate the constants file in PIL.
+ */
+export declare function generatePilConstants({ constants, domainSeparatorEnum }: ParsedContent, targetPath: string): void;
+/**
+ * Generate the constants file in Solidity.
+ */
+export declare function generateSolidityConstants({ constants, domainSeparatorEnum }: ParsedContent, targetPath: string): void;
+/**
+ * Generate the constants file in Rust.
+ */
+export declare function generateRustConstants({ constants, domainSeparatorEnum }: ParsedContent, targetPath: string): void;
+/**
+ * Parse the content of the constants file in Noir.
+ */
+export declare function parseNoirFile(fileContent: string): ParsedExpressions;
+/**
+ * Converts constants defined as expressions to constants with actual values.
+ * @param expressions Ordered list of expressions of the type: "CONSTANT_NAME: expression".
+ *   where the expression is a string that can be evaluated to a number.
+ *   For example: "CONSTANT_NAME: 2 + 2" or "CONSTANT_NAME: CONSTANT_A * CONSTANT_B".
+ * @returns Parsed expressions of the form: "CONSTANT_NAME: number_as_string".
+ */
+export declare function evaluateExpressions(expressions: [string, string][]): {
+    [key: string]: string;
+};

package/dest/generator.js

@@ -0,0 +1,291 @@
+import * as fs from 'node:fs';
+/**
+ * Processes a collection of constants and generates code to export them as TypeScript constants.
+ *
+ * @param constants - An object containing key-value pairs representing constants.
+ * @returns A string containing code that exports the constants as TypeScript constants.
+ */
+export function processConstantsTS(constants) {
+    const code = [];
+    Object.entries(constants).forEach(([key, value]) => {
+        code.push(`export const ${key} = ${+value > Number.MAX_SAFE_INTEGER ? value + 'n' : value};`);
+    });
+    return code.join('\n');
+}
+/**
+ * Processes a collection of constants and generates code to export them as cpp constants.
+ * Required to ensure consistency between the constants used in pil and used in the vm witness generator.
+ *
+ * @param constants - An object containing key-value pairs representing constants.
+ * @returns A string containing code that exports the constants as cpp constants.
+ */
+export function processConstantsCpp(constants, generatorIndices) {
+    const code = [];
+    Object.entries(constants).forEach(([key, value]) => {
+        if (BigInt(value) <= 2n ** 31n - 1n) {
+            code.push(`#define ${key} ${value}`);
+        }
+        else if (BigInt(value) <= 2n ** 64n - 1n) {
+            code.push(`#define ${key} 0x${BigInt(value).toString(16)}`); // hex literals
+        }
+        else {
+            code.push(`#define ${key} "0x${BigInt(value).toString(16).padStart(64, '0')}"`); // stringify large numbers
+        }
+    });
+    Object.entries(generatorIndices).forEach(([key, value]) => {
+        code.push(`#define DOM_SEP__${key} ${value}UL`);
+    });
+    return code.join('\n');
+}
+/**
+ * Processes a collection of constants and generates code to export them as PIL constants.
+ * Required to ensure consistency between the constants used in pil and used in the vm witness generator.
+ *
+ * @param constants - An object containing key-value pairs representing constants.
+ * @returns A string containing code that exports the constants as cpp constants.
+ */
+export function processConstantsPil(constants, generatorIndices) {
+    const code = [];
+    Object.entries(constants).forEach(([key, value]) => {
+        code.push(`    pol ${key} = ${value};`);
+    });
+    Object.entries(generatorIndices).forEach(([key, value]) => {
+        code.push(`    pol DOM_SEP__${key} = ${value};`);
+    });
+    return code.join('\n');
+}
+/**
+ * Processes an enum and generates code to export it as a TypeScript enum.
+ *
+ * @param enumName - The name of the enum.
+ * @param enumValues - An object containing key-value pairs representing enum values.
+ * @returns A string containing code that exports the enum as a TypeScript enum.
+ */
+export function processEnumTS(enumName, enumValues) {
+    const code = [];
+    code.push(`export enum ${enumName} {`);
+    Object.entries(enumValues).forEach(([key, value]) => {
+        code.push(`  ${key} = ${value},`);
+    });
+    code.push('}');
+    return code.join('\n');
+}
+/**
+ * Processes a collection of constants and generates code to export them as Solidity constants.
+ *
+ * @param constants - An object containing key-value pairs representing constants.
+ * @param generatorIndices - An object containing key-value pairs representing domain separator indices.
+ * @param prefix - A prefix to add to the constant names.
+ * @returns A string containing code that exports the constants as Noir constants.
+ */
+export function processConstantsSolidity(constants, generatorIndices, prefix = '') {
+    const code = [];
+    Object.entries(constants).forEach(([key, value]) => {
+        code.push(`  uint256 internal constant ${prefix}${key} = ${value};`);
+    });
+    Object.entries(generatorIndices).forEach(([key, value]) => {
+        code.push(`  uint256 internal constant ${prefix}DOM_SEP__${key} = ${value};`);
+    });
+    return code.join('\n');
+}
+/**
+ * Processes a collection of constants and generates code to export them as Rust constants.
+ *
+ * @param constants - An object containing key-value pairs representing constants.
+ * @param generatorIndices - An object containing key-value pairs representing domain separator indices.
+ * @returns A string containing code that exports the constants as Rust constants.
+ */
+export function processConstantsRust(constants, generatorIndices) {
+    const code = [];
+    Object.entries(constants).forEach(([key, value]) => {
+        if (BigInt(value) <= 2n ** 128n - 1n) {
+            code.push(`pub const ${key}: u128 = ${value};`);
+        }
+        else {
+            // Field-sized values exceed u128, so they are emitted as hex strings.
+            code.push(`pub const ${key}: &str = "0x${BigInt(value).toString(16).padStart(64, '0')}";`);
+        }
+    });
+    Object.entries(generatorIndices).forEach(([key, value]) => {
+        code.push(`pub const DOM_SEP__${key}: u128 = ${value};`);
+    });
+    return code.join('\n');
+}
+/**
+ * Generate the constants file in Typescript.
+ */
+export function generateTypescriptConstants({ constants, domainSeparatorEnum }, targetPath) {
+    const result = [
+        '// GENERATED FILE - DO NOT EDIT',
+        processConstantsTS(constants),
+        processEnumTS('DomainSeparator', domainSeparatorEnum),
+    ].join('\n');
+    fs.writeFileSync(targetPath, result);
+}
+/**
+ * Generate the constants file in C++.
+ */
+export function generateCppConstants({ constants, domainSeparatorEnum }, targetPath) {
+    const resultCpp = `// GENERATED FILE - DO NOT EDIT
+#pragma once
+
+${processConstantsCpp(constants, domainSeparatorEnum)}
+`;
+    fs.writeFileSync(targetPath, resultCpp);
+}
+/**
+ * Generate the constants file in PIL.
+ */
+export function generatePilConstants({ constants, domainSeparatorEnum }, targetPath) {
+    const resultPil = `// GENERATED FILE - DO NOT EDIT
+namespace constants;
+${processConstantsPil(constants, domainSeparatorEnum)}
+\n`;
+    fs.writeFileSync(targetPath, resultPil);
+}
+/**
+ * Generate the constants file in Solidity.
+ */
+export function generateSolidityConstants({ constants, domainSeparatorEnum }, targetPath) {
+    const resultSolidity = `// GENERATED FILE - DO NOT EDIT
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2023 Aztec Labs.
+pragma solidity >=0.8.27;
+
+/**
+ * @title Constants Library
+ * @author Aztec Labs
+ * @notice Library that contains constants used throughout the Aztec protocol
+ */
+library Constants {
+  // Prime field modulus
+  uint256 internal constant P =
+    21888242871839275222246405745257275088548364400416034343698204186575808495617;
+
+${processConstantsSolidity(constants, domainSeparatorEnum)}
+}\n`;
+    fs.writeFileSync(targetPath, resultSolidity);
+}
+/**
+ * Generate the constants file in Rust.
+ */
+export function generateRustConstants({ constants, domainSeparatorEnum }, targetPath) {
+    const resultRust = `// GENERATED FILE - DO NOT EDIT
+${processConstantsRust(constants, domainSeparatorEnum)}
+`;
+    fs.writeFileSync(targetPath, resultRust);
+}
+/**
+ * Parse the content of the constants file in Noir.
+ */
+export function parseNoirFile(fileContent) {
+    const constantsExpressions = [];
+    const domainSeparatorEnum = {};
+    const emptyExpression = () => ({ name: '', content: [] });
+    let expression = emptyExpression();
+    fileContent.split('\n').forEach(l => {
+        // Line comments are stripped so they never leak into expressions, where they would swallow
+        // the rest of the expression when it is later evaluated as JavaScript.
+        const line = l.replace(/\/\/.*$/, '').trim();
+        if (!line) {
+            // Empty line.
+            return;
+        }
+        if (line.match(/^\/\/|^\s*\/?\*/)) {
+            // Comment.
+            return;
+        }
+        {
+            const [, name, _type, value, end] = line.match(/global\s+(\w+)(\s*:\s*\w+)?\s*=\s*([^;]*)(;)?/) || [];
+            if (name && value) {
+                const [, indexName] = name.match(/DOM_SEP__(\w+)/) || [];
+                if (indexName) {
+                    // Generator index.
+                    domainSeparatorEnum[indexName] = +value;
+                }
+                else if (end) {
+                    // A single line of expression.
+                    constantsExpressions.push([name, value]);
+                }
+                else {
+                    // The first line of an expression.
+                    expression = { name, content: [value] };
+                }
+                return;
+            }
+            else if (name) {
+                // This case happens if we have only a name, with the value being on the next line
+                expression = { name, content: [] };
+                return;
+            }
+        }
+        if (expression.name) {
+            // The expression continues...
+            const [, content, end] = line.match(/\s*([^;]+)(;)?/) || [];
+            expression.content.push(content);
+            if (end) {
+                // The last line of an expression.
+                constantsExpressions.push([expression.name, expression.content.join('')]);
+                expression = emptyExpression();
+            }
+            return;
+        }
+        if (!line.includes('use crate')) {
+            // eslint-disable-next-line no-console
+            console.warn(`Unknown content: ${line}`);
+        }
+    });
+    return { constantsExpressions, domainSeparatorEnum };
+}
+/**
+ * Converts constants defined as expressions to constants with actual values.
+ * @param expressions Ordered list of expressions of the type: "CONSTANT_NAME: expression".
+ *   where the expression is a string that can be evaluated to a number.
+ *   For example: "CONSTANT_NAME: 2 + 2" or "CONSTANT_NAME: CONSTANT_A * CONSTANT_B".
+ * @returns Parsed expressions of the form: "CONSTANT_NAME: number_as_string".
+ */
+export function evaluateExpressions(expressions) {
+    const constants = {};
+    const knownBigInts = ['AZTEC_EPOCH_DURATION', 'FEE_RECIPIENT_LENGTH'];
+    // Create JS expressions. It is not as easy as just evaluating the expression!
+    // We basically need to convert everything to BigInts, otherwise things don't fit.
+    // However, (1) the bigints need to be initialized from strings; (2) everything needs to
+    // be a bigint, even the actual constant values!
+    const prelude = expressions
+        .map(([name, rhs]) => {
+        const guardedRhs = rhs
+            // Remove 'as u8', 'as u32' and 'as u64' castings
+            .replaceAll(' as u8', '')
+            .replaceAll(' as u32', '')
+            .replaceAll(' as u64', '')
+            // Remove the 'AztecAddress::from_field(...)' pattern.
+            // Also copes with the noir formatter re-formatting over multiple lines.
+            .replace(/AztecAddress::from_field\(\s*(0x[a-fA-F0-9]+|\d+)\s*,?\s*\)/gs, '$1')
+            // We make some space around the parentheses, so that constant numbers are still split.
+            .replace(/\(/g, '( ')
+            .replace(/\)/g, ' )')
+            // We also make some space around common operators
+            .replace(/\+/g, ' + ')
+            .replace(/(?<!\/)\*(?!\/)/, ' * ')
+            // We split the expression into terms...
+            .split(/\s+/)
+            // ...and then we convert each term to a BigInt if it is a number.
+            .map(term => {
+            // Remove underscores from numeric literals (e.g., 6_000_000 -> 6000000)
+            const termWithoutUnderscores = term.replace(/_/g, '');
+            return isNaN(+termWithoutUnderscores) ? term : `BigInt('${termWithoutUnderscores}')`;
+        })
+            // .. also, we convert the known bigints to BigInts.
+            .map(term => (knownBigInts.includes(term) ? `BigInt(${term})` : term))
+            // We join the terms back together.
+            .join(' ');
+        return `var ${name} = ${guardedRhs};`;
+    })
+        .join('\n');
+    // Extract each value from the expressions. Observe that this will still be a string,
+    // so that we can then choose to express it as BigInt or Number depending on the size.
+    for (const [name, _] of expressions) {
+        constants[name] = eval(prelude + `; BigInt(${name}).toString()`);
+    }
+    return constants;
+}

package/dest/selection.d.ts

@@ -0,0 +1,7 @@
+import type { ParsedContent } from './generator.ts';
+/** Source symbols to include in one generated output. */
+export type SymbolSelection = string[];
+/** Reads and validates a symbol selection JSON file. */
+export declare function readSymbolSelection(path: string): SymbolSelection;
+/** Filters parsed Noir content to the symbols requested for one output. */
+export declare function selectSymbols(content: ParsedContent, selection: SymbolSelection): ParsedContent;

package/dest/selection.js

@@ -0,0 +1,68 @@
+import { readFileSync } from 'node:fs';
+/** Conventional prefix for domain separators. */
+const DOMAIN_SEPARATOR_PREFIX = 'DOM_SEP__';
+// Symbol names contain no regex metacharacters, so any entry that is not a
+// valid name can only be intended as a pattern.
+function isExactName(entry) {
+    return /^[A-Za-z_]\w*$/.test(entry);
+}
+function compilePattern(entry) {
+    return new RegExp(`^(?:${entry})$`);
+}
+/** Reads and validates a symbol selection JSON file. */
+export function readSymbolSelection(path) {
+    let value;
+    try {
+        value = JSON.parse(readFileSync(path, 'utf8'));
+    }
+    catch (error) {
+        throw new Error(`could not read selection ${path}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    if (!Array.isArray(value) || !value.every(entry => typeof entry === 'string')) {
+        throw new Error(`selection ${path} must be a JSON array of strings`);
+    }
+    for (const entry of value) {
+        if (!isExactName(entry)) {
+            try {
+                compilePattern(entry);
+            }
+            catch {
+                throw new Error(`invalid pattern '${entry}' in ${path}`);
+            }
+        }
+    }
+    const duplicate = value.find((entry, index) => value.indexOf(entry) !== index);
+    if (duplicate) {
+        throw new Error(`duplicate entry '${duplicate}' in ${path}`);
+    }
+    return value;
+}
+/** Filters parsed Noir content to the symbols requested for one output. */
+export function selectSymbols(content, selection) {
+    const sourceNames = [
+        ...Object.keys(content.constants),
+        ...Object.keys(content.domainSeparatorEnum).map(name => DOMAIN_SEPARATOR_PREFIX + name),
+    ];
+    const sourceNameSet = new Set(sourceNames);
+    const selected = new Set();
+    for (const entry of selection) {
+        if (isExactName(entry)) {
+            if (!sourceNameSet.has(entry)) {
+                throw new Error(`unknown symbol '${entry}' in selection`);
+            }
+            selected.add(entry);
+        }
+        else {
+            const pattern = compilePattern(entry);
+            const matches = sourceNames.filter(name => pattern.test(name));
+            if (matches.length === 0) {
+                throw new Error(`pattern '${entry}' in selection matched no symbols`);
+            }
+            matches.forEach(name => selected.add(name));
+        }
+    }
+    return {
+        constants: Object.fromEntries(Object.entries(content.constants).filter(([name]) => selected.has(name))),
+        domainSeparatorEnum: Object.fromEntries(Object.entries(content.domainSeparatorEnum).filter(([name]) => selected.has(DOMAIN_SEPARATOR_PREFIX + name))),
+    };
+}