@aztec/txe 5.0.0-private.20260319 → 5.0.0-rc.2

package/src/index.ts

@@ -1,62 +1,33 @@
-import { SchnorrAccountContractArtifact } from '@aztec/accounts/schnorr';
-import { type NoirCompiledContract, loadContractArtifact } from '@aztec/aztec.js/abi';
-import { AztecAddress } from '@aztec/aztec.js/addresses';
-import {
-  type ContractInstanceWithAddress,
-  getContractInstanceFromInstantiationParams,
-} from '@aztec/aztec.js/contracts';
 import { Fr } from '@aztec/aztec.js/fields';
-import { PublicKeys, deriveKeys } from '@aztec/aztec.js/keys';
-import { createSafeJsonRpcServer } from '@aztec/foundation/json-rpc/server';
 import type { Logger } from '@aztec/foundation/log';
-import { openTmpStore } from '@aztec/kv-store/lmdb-v2';
-import { protocolContractNames } from '@aztec/protocol-contracts';
-import { BundledProtocolContractsProvider } from '@aztec/protocol-contracts/providers/bundle';
-import { ContractStore } from '@aztec/pxe/server';
-import { computeArtifactHash } from '@aztec/stdlib/contract';
-import type { ContractArtifactWithHash } from '@aztec/stdlib/contract';
+import { cloneEphemeralStoreFrom } from '@aztec/kv-store/lmdb-v2';
+import type { ProtocolContractName } from '@aztec/protocol-contracts';
+import { ContractStore } from '@aztec/pxe/client/lazy';
 import type { ApiSchemaFor } from '@aztec/stdlib/schemas';
 import { zodFor } from '@aztec/stdlib/schemas';
 
-import { createHash } from 'crypto';
-import { createReadStream } from 'fs';
-import { readFile, readdir } from 'fs/promises';
-import { join, parse } from 'path';
+import { join } from 'path';
 import { z } from 'zod';
 
+// Side-effect import: registers the msgpackr Fr extension for the bundled `Fr` class. Must
+// be loaded before any `sendMessage` call. See msgpackr_fr_extension.ts for the why.
+import './msgpackr_fr_extension.js';
 import { type TXEOracleFunctionName, TXESession } from './txe_session.js';
 import {
   type ForeignCallArgs,
   ForeignCallArgsSchema,
-  type ForeignCallArray,
   type ForeignCallResult,
   ForeignCallResultSchema,
-  type ForeignCallSingle,
-  addressFromSingle,
-  fromArray,
-  fromSingle,
-  toSingle,
-} from './util/encoding.js';
+} from './utils/encoding.js';
+import { TXEArtifactResolver } from './utils/txe_artifact_resolver.js';
 
-const sessions = new Map<number, TXESession>();
+// Protocol contracts TXE registers in its contract store. Only AuthRegistry is needed for the
+// current test suites; add a contract here if a lookup against a `0x000…00X` address fails.
+export const TXE_REQUIRED_PROTOCOL_CONTRACTS: ProtocolContractName[] = [];
 
-/*
- * TXE typically has to load the same contract artifacts over and over again for multiple tests,
- * so we cache them here to avoid loading from disk repeatedly.
- *
- * The in-flight map coalesces concurrent requests for the same cache key so that
- * computeArtifactHash (very expensive) is only run once even under parallelism.
- */
-const TXEArtifactsCache = new Map<
-  string,
-  { artifact: ContractArtifactWithHash; instance: ContractInstanceWithAddress }
->();
-const TXEArtifactsCacheInFlight = new Map<
-  string,
-  Promise<{ artifact: ContractArtifactWithHash; instance: ContractInstanceWithAddress }>
->();
+const sessions = new Map<number, TXESession>();
 
-type TXEForeignCallInput = {
+export type TXEForeignCallInput = {
   session_id: number;
   function: TXEOracleFunctionName;
   root_path: string;
@@ -64,10 +35,13 @@ type TXEForeignCallInput = {
   inputs: ForeignCallArgs;
 };
 
-const TXEForeignCallInputSchema = zodFor<TXEForeignCallInput>()(
+export const TXEForeignCallInputSchema: z.ZodType<TXEForeignCallInput> = zodFor<TXEForeignCallInput>()(
   z.object({
+    // Nargo generates session_id as a u64, which may exceed Number.MAX_SAFE_INTEGER.
+    // Zod 4's `.int()` enforces the safe-integer bound, so we drop it here and only require
+    // the value to be a non-negative number (it is used solely as a Map key).
     // eslint-disable-next-line camelcase
-    session_id: z.number().int().nonnegative(),
+    session_id: z.number().nonnegative(),
     function: z.string() as z.ZodType<TXEOracleFunctionName>,
     // eslint-disable-next-line camelcase
     root_path: z.string(),
@@ -77,205 +51,101 @@ const TXEForeignCallInputSchema = zodFor<TXEForeignCallInput>()(
   }),
 );
 
-class TXEDispatcher {
-  private contractStore!: ContractStore;
-
-  constructor(private logger: Logger) {}
-
-  private fastHashFile(path: string) {
-    return new Promise(resolve => {
-      const fd = createReadStream(path);
-      const hash = createHash('sha1');
-      hash.setEncoding('hex');
-
-      fd.on('end', function () {
-        hash.end();
-        resolve(hash.read());
-      });
-
-      fd.pipe(hash);
-    });
-  }
-
-  async #processDeployInputs({ inputs, root_path: rootPath, package_name: packageName }: TXEForeignCallInput) {
-    const [contractPath, initializer] = inputs.slice(0, 2).map(input =>
-      fromArray(input as ForeignCallArray)
-        .map(char => String.fromCharCode(char.toNumber()))
-        .join(''),
-    );
-
-    const decodedArgs = fromArray(inputs[3] as ForeignCallArray);
-    const secret = fromSingle(inputs[4] as ForeignCallSingle);
-    const salt = fromSingle(inputs[5] as ForeignCallSingle);
-    const deployer = addressFromSingle(inputs[6] as ForeignCallSingle);
-    const publicKeys = secret.equals(Fr.ZERO) ? PublicKeys.default() : (await deriveKeys(secret)).publicKeys;
-    const publicKeysHash = await publicKeys.hash();
-
-    let artifactPath = '';
-    const { dir: contractDirectory, base: contractFilename } = parse(contractPath);
-    if (contractDirectory) {
-      if (contractDirectory.includes('@')) {
-        // We're deploying a contract that belongs in a workspace
-        // env.deploy("../path/to/workspace/root@packageName/contractName")
-        const [workspace, pkg] = contractDirectory.split('@');
-        const targetPath = join(rootPath, workspace, '/target');
-        this.logger.debug(`Looking for compiled artifact in workspace ${targetPath}`);
-        artifactPath = join(targetPath, `${pkg}-${contractFilename}.json`);
-      } else {
-        // We're deploying a standalone external contract
-        // env.deploy("../path/to/contract/root/contractName")
-        const targetPath = join(rootPath, contractDirectory, '/target');
-        this.logger.debug(`Looking for compiled artifact in ${targetPath}`);
-        [artifactPath] = (await readdir(targetPath)).filter(file => file.endsWith(`-${contractFilename}.json`));
-      }
-    } else {
-      // We're deploying a local contract
-      // env.deploy("contractName")
-      artifactPath = join(rootPath, './target', `${packageName}-${contractFilename}.json`);
-    }
-
-    const fileHash = await this.fastHashFile(artifactPath);
-
-    const cacheKey = `${contractDirectory ?? ''}-${contractFilename}-${initializer}-${decodedArgs
-      .map(arg => arg.toString())
-      .join('-')}-${publicKeysHash}-${salt}-${deployer}-${fileHash}`;
-
-    let instance;
-    let artifact: ContractArtifactWithHash;
-
-    if (TXEArtifactsCache.has(cacheKey)) {
-      this.logger.debug(`Using cached artifact for ${cacheKey}`);
-      ({ artifact, instance } = TXEArtifactsCache.get(cacheKey)!);
-    } else {
-      if (!TXEArtifactsCacheInFlight.has(cacheKey)) {
-        this.logger.debug(`Loading compiled artifact ${artifactPath}`);
-        const compute = async () => {
-          const artifactJSON = JSON.parse(await readFile(artifactPath, 'utf-8')) as NoirCompiledContract;
-          const artifactWithoutHash = loadContractArtifact(artifactJSON);
-          const computedArtifact: ContractArtifactWithHash = {
-            ...artifactWithoutHash,
-            // Artifact hash is *very* expensive to compute, so we do it here once
-            // and the TXE contract data provider can cache it
-            artifactHash: await computeArtifactHash(artifactWithoutHash),
-          };
-          this.logger.debug(
-            `Deploy ${computedArtifact.name} with initializer ${initializer}(${decodedArgs}) and public keys hash ${publicKeysHash.toString()}`,
-          );
-          const computedInstance = await getContractInstanceFromInstantiationParams(computedArtifact, {
-            constructorArgs: decodedArgs,
-            skipArgsDecoding: true,
-            salt,
-            publicKeys,
-            constructorArtifact: initializer ? initializer : undefined,
-            deployer,
-          });
-          const result = { artifact: computedArtifact, instance: computedInstance };
-          TXEArtifactsCache.set(cacheKey, result);
-          TXEArtifactsCacheInFlight.delete(cacheKey);
-          return result;
-        };
-        TXEArtifactsCacheInFlight.set(cacheKey, compute());
-      }
-      ({ artifact, instance } = await TXEArtifactsCacheInFlight.get(cacheKey)!);
-    }
+export interface TXEDispatcherOptions {
+  /**
+   * Path to an LMDB directory holding the required protocol contracts (see
+   * {@link TXE_REQUIRED_PROTOCOL_CONTRACTS}) and the SchnorrAccount artifact. When set, the
+   * dispatcher clones this directory into a fresh tmpdir on first use instead of registering
+   * the contracts itself.
+   */
+  contractStoreSourceDir: string;
+  /**
+   * Class id (hex) of the SchnorrAccount artifact pre-registered in the shared LMDB. The
+   * {@link TXEArtifactResolver} looks the artifact up from the cloned store via this class id
+   * instead of recomputing it via `getSchnorrAccountContractArtifact()` + `computeArtifactHash()`.
+   */
+  schnorrClassId: string;
+}
 
-    inputs.splice(0, 1, artifact, instance, toSingle(secret));
+export class TXEDispatcher {
+  private contractStore!: ContractStore;
+  private artifactResolver!: TXEArtifactResolver;
+  private readonly contractStoreSourceDir: string;
+  private readonly schnorrClassId: Fr;
+
+  constructor(
+    private logger: Logger,
+    opts: TXEDispatcherOptions,
+  ) {
+    this.contractStoreSourceDir = opts.contractStoreSourceDir;
+    this.schnorrClassId = Fr.fromString(opts.schnorrClassId);
   }
 
-  async #processAddAccountInputs({ inputs }: TXEForeignCallInput) {
-    const secret = fromSingle(inputs[0] as ForeignCallSingle);
-
-    const cacheKey = `SchnorrAccountContract-${secret}`;
-
-    let artifact: ContractArtifactWithHash;
-    let instance;
-
-    if (TXEArtifactsCache.has(cacheKey)) {
-      this.logger.debug(`Using cached artifact for ${cacheKey}`);
-      ({ artifact, instance } = TXEArtifactsCache.get(cacheKey)!);
-    } else {
-      if (!TXEArtifactsCacheInFlight.has(cacheKey)) {
-        const compute = async () => {
-          const keys = await deriveKeys(secret);
-          const args = [
-            keys.publicKeys.masterIncomingViewingPublicKey.x,
-            keys.publicKeys.masterIncomingViewingPublicKey.y,
-          ];
-          const computedArtifact: ContractArtifactWithHash = {
-            ...SchnorrAccountContractArtifact,
-            // Artifact hash is *very* expensive to compute, so we do it here once
-            // and the TXE contract data provider can cache it
-            artifactHash: await computeArtifactHash(SchnorrAccountContractArtifact),
-          };
-          const computedInstance = await getContractInstanceFromInstantiationParams(computedArtifact, {
-            constructorArgs: args,
-            skipArgsDecoding: true,
-            salt: Fr.ONE,
-            publicKeys: keys.publicKeys,
-            constructorArtifact: 'constructor',
-            deployer: AztecAddress.ZERO,
-          });
-          const result = { artifact: computedArtifact, instance: computedInstance };
-          TXEArtifactsCache.set(cacheKey, result);
-          TXEArtifactsCacheInFlight.delete(cacheKey);
-          return result;
-        };
-        TXEArtifactsCacheInFlight.set(cacheKey, compute());
-      }
-      ({ artifact, instance } = await TXEArtifactsCacheInFlight.get(cacheKey)!);
+  /**
+   * Clones the pre-populated LMDB at `contractStoreSourceDir` into a fresh per-instance tmpdir
+   * on first use, so this dispatcher has a writable store already containing the required
+   * protocol contracts + SchnorrAccount. Idempotent — subsequent calls are no-ops.
+   */
+  private async warmUp(): Promise<void> {
+    if (this.contractStore) {
+      return;
     }
-
-    inputs.splice(0, 0, artifact, instance);
+    const t0 = Date.now();
+    const kvStore = await cloneEphemeralStoreFrom(
+      join(this.contractStoreSourceDir, 'data.mdb'),
+      'txe-contracts',
+      undefined,
+      2,
+    );
+    this.contractStore = new ContractStore(kvStore);
+    this.artifactResolver = new TXEArtifactResolver(this.contractStore, this.schnorrClassId);
+    this.logger.debug('Cloned shared protocol-contracts store', { totalMs: Date.now() - t0 });
   }
 
   // eslint-disable-next-line camelcase
   async resolve_foreign_call(callData: TXEForeignCallInput): Promise<ForeignCallResult> {
-    const { session_id: sessionId, function: functionName, inputs } = callData;
+    const {
+      session_id: sessionId,
+      function: functionName,
+      inputs,
+      root_path: rootPath,
+      package_name: packageName,
+    } = callData;
     this.logger.debug(`Calling ${functionName} on session ${sessionId}`);
 
     if (!sessions.has(sessionId)) {
       this.logger.debug(`Creating new session ${sessionId}`);
-      if (!this.contractStore) {
-        const kvStore = await openTmpStore('txe-contracts');
-        this.contractStore = new ContractStore(kvStore);
-        const provider = new BundledProtocolContractsProvider();
-        for (const name of protocolContractNames) {
-          const { instance, artifact } = await provider.getProtocolContractArtifact(name);
-          await this.contractStore.addContractArtifact(artifact);
-          await this.contractStore.addContractInstance(instance);
-        }
-        this.logger.debug('Registered protocol contracts in shared contract store');
-      }
-      sessions.set(sessionId, await TXESession.init(this.contractStore));
-    }
-
-    switch (functionName) {
-      case 'aztec_txe_deploy': {
-        await this.#processDeployInputs(callData);
-        break;
-      }
-      case 'aztec_txe_addAccount': {
-        await this.#processAddAccountInputs(callData);
-        break;
-      }
+      await this.warmUp();
+      sessions.set(sessionId, await TXESession.init(this.contractStore, this.artifactResolver, rootPath, packageName));
     }
 
     return await sessions.get(sessionId)!.processFunction(functionName, inputs);
   }
+
+  /**
+   * Releases a session and its resources (per-session LMDB + `NativeWorldStateService`).
+   * Called by the dispatcher pool when nargo closes its TCP connection for a test (see
+   * `rpc_server.ts`'s socket tracker). No-op if the session was never created — that happens
+   * when nargo opens a connection but errors before sending a request.
+   */
+  async disposeSession(sessionId: number): Promise<void> {
+    const session = sessions.get(sessionId);
+    if (!session) {
+      return;
+    }
+    sessions.delete(sessionId);
+    await session.dispose();
+  }
 }
 
-const TXEDispatcherApiSchema: ApiSchemaFor<TXEDispatcher> = {
+/** Diagnostic-only: number of sessions currently held by this worker. */
+export function activeSessionCount(): number {
+  return sessions.size;
+}
+
+export const TXEDispatcherApiSchema: ApiSchemaFor<TXEDispatcher> = {
   // eslint-disable-next-line camelcase
-  resolve_foreign_call: z.function().args(TXEForeignCallInputSchema).returns(ForeignCallResultSchema),
+  resolve_foreign_call: z.function({ input: z.tuple([TXEForeignCallInputSchema]), output: ForeignCallResultSchema }),
+  // disposeSession is invoked over IPC from the worker, not via RPC; required by ApiSchemaFor.
+  disposeSession: z.function({ input: z.tuple([z.number().nonnegative()]), output: z.void() }),
 };
-
-/**
- * Creates an RPC server that forwards calls to the TXE.
- * @param logger - Logger to output to
- * @returns A TXE RPC server.
- */
-export function createTXERpcServer(logger: Logger) {
-  return createSafeJsonRpcServer(new TXEDispatcher(logger), TXEDispatcherApiSchema, {
-    http200OnError: true,
-  });
-}

package/src/msgpackr_fr_extension.ts

@@ -0,0 +1,23 @@
+import { Fr } from '@aztec/foundation/curves/bn254';
+
+import { addExtension } from 'msgpackr';
+
+// `@aztec/native/msgpack_channel` registers a msgpackr extension that packs `Fr` instances to
+// their raw 32-byte buffer for the C++ world-state side to deserialize. Because `@aztec/native`
+// is externalized in TXE's esbuild config (it loads a `.node` binary that can't be bundled),
+// the `Fr` class the extension is keyed on belongs to the *external* `@aztec/foundation`
+// module instance — a different class identity than the `Fr` bundled into the TXE worker /
+// bin. msgpackr's extension match uses `instanceof`, so bundled `Fr` instances slip through
+// and msgpackr falls back to `Fr.toJSON()` (returns `Fr.toString()` — a `0x...` hex string).
+// The C++ side then sees a string where a 32-byte binary was expected and throws
+// `msgpack::type_error` whose `what()` is literally `"std::bad_cast"`.
+//
+// Fix: register the same extension under the BUNDLED `Fr` class identity. msgpackr's extension
+// table holds an entry per `Class`, so both Fr classes now resolve to the same `write` callback
+// and produce the same wire format. Imported as a side-effect from `./index.js` so both the
+// worker (via `./worker.ts`) and the main-thread RPC server (via `./rpc_server.ts`) get the
+// registration before any `sendMessage` runs.
+addExtension({
+  Class: Fr,
+  write: (fr: Fr) => fr.toBuffer(),
+});

package/src/oracle/interfaces.ts

@@ -1,11 +1,14 @@
-import type { ContractArtifact } from '@aztec/aztec.js/abi';
 import { CompleteAddress } from '@aztec/aztec.js/addresses';
-import type { ContractInstanceWithAddress } from '@aztec/aztec.js/contracts';
 import { TxHash } from '@aztec/aztec.js/tx';
 import { BlockNumber } from '@aztec/foundation/branded-types';
 import type { Fr } from '@aztec/foundation/curves/bn254';
+import type { EthAddress } from '@aztec/foundation/eth-address';
+import type { TaggingSecretStrategy } from '@aztec/pxe/server';
+import type { Option } from '@aztec/pxe/simulator';
 import type { EventSelector, FunctionSelector } from '@aztec/stdlib/abi';
 import type { AztecAddress } from '@aztec/stdlib/aztec-address';
+import type { GasSettings } from '@aztec/stdlib/gas';
+import type { PrivateLog } from '@aztec/stdlib/logs';
 import type { UInt64 } from '@aztec/stdlib/types';
 
 // These interfaces complement the ones defined in PXE, and combined with those contain the full list of oracles used by
@@ -36,6 +39,15 @@ export interface IAvmExecutionOracle {
   nullifierExists(siloedNullifier: Fr): Promise<boolean>;
   storageWrite(slot: Fr, value: Fr): Promise<void>;
   storageRead(slot: Fr, contractAddress: AztecAddress): Promise<Fr>;
+  getContractInstanceDeployer(address: AztecAddress): Promise<{ member: Fr; exists: boolean }>;
+  getContractInstanceClassId(address: AztecAddress): Promise<{ member: Fr; exists: boolean }>;
+  getContractInstanceInitializationHash(address: AztecAddress): Promise<{ member: Fr; exists: boolean }>;
+  getContractInstanceImmutablesHash(address: AztecAddress): Promise<{ member: Fr; exists: boolean }>;
+  returndataSize(): Promise<Fr>;
+  returndataCopy(rdOffset: number, copySize: number): Promise<Fr[]>;
+  call(l2Gas: number, daGas: number, address: AztecAddress, argsLength: number, args: Fr[]): Promise<void>;
+  staticCall(l2Gas: number, daGas: number, address: AztecAddress, argsLength: number, args: Fr[]): Promise<void>;
+  successCopy(): Promise<boolean>;
 }
 
 /**
@@ -49,37 +61,53 @@ export interface ITxeExecutionOracle {
   getNextBlockTimestamp(): Promise<UInt64>;
   advanceBlocksBy(blocks: number): Promise<void>;
   advanceTimestampBy(duration: UInt64): void;
-  deploy(artifact: ContractArtifact, instance: ContractInstanceWithAddress, foreignSecret: Fr): Promise<void>;
+  deploy(
+    contractPath: string,
+    initializer: string,
+    args: Fr[],
+    secret: Fr,
+    salt: Fr,
+    deployer: AztecAddress,
+  ): Promise<Fr[]>;
   createAccount(secret: Fr): Promise<CompleteAddress>;
-  addAccount(artifact: ContractArtifact, instance: ContractInstanceWithAddress, secret: Fr): Promise<CompleteAddress>;
+  addAccount(secret: Fr): Promise<CompleteAddress>;
   addAuthWitness(address: AztecAddress, messageHash: Fr): Promise<void>;
+  sendL1ToL2Message(content: Fr, secretHash: Fr, sender: EthAddress, recipient: AztecAddress): Promise<Fr>;
+  setTaggingSecretStrategy(strategy: Option<TaggingSecretStrategy>): void;
   getLastBlockTimestamp(): Promise<bigint>;
   getLastTxEffects(): Promise<{
     txHash: TxHash;
     noteHashes: Fr[];
     nullifiers: Fr[];
+    privateLogs: PrivateLog[];
   }>;
   getPrivateEvents(selector: EventSelector, contractAddress: AztecAddress, scope: AztecAddress): Promise<Fr[][]>;
   privateCallNewFlow(
-    from: AztecAddress,
+    from: AztecAddress | undefined,
     targetContractAddress: AztecAddress,
     functionSelector: FunctionSelector,
     args: Fr[],
     argsHash: Fr,
     isStaticCall: boolean,
+    additionalScopes: AztecAddress[],
     jobId: string,
-  ): Promise<Fr[]>;
+    authorizedUtilityCallTargets: AztecAddress[],
+    gasSettings: GasSettings,
+  ): Promise<{ returnValues: Fr[]; offchainEffects: Fr[][] }>;
   executeUtilityFunction(
+    from: AztecAddress | undefined,
     targetContractAddress: AztecAddress,
     functionSelector: FunctionSelector,
     args: Fr[],
     jobId: string,
+    authorizedUtilityCallTargets: AztecAddress[],
   ): Promise<Fr[]>;
   publicCallNewFlow(
-    from: AztecAddress,
+    from: AztecAddress | undefined,
     targetContractAddress: AztecAddress,
     calldata: Fr[],
     isStaticCall: boolean,
+    gasSettings: GasSettings,
   ): Promise<Fr[]>;
   // TODO(F-335): Drop this from here as it's not a real oracle handler - it's only called from
   // RPCTranslator::txeGetPrivateEvents and never from Noir.