@proto-kit/common 0.1.1-develop.211 → 0.1.1-develop.2137

package/package.json

@@ -3,32 +3,34 @@
   "license": "MIT",
   "private": false,
   "type": "module",
-  "version": "0.1.1-develop.211+662735c",
+  "version": "0.1.1-develop.2137+8a7eca31",
   "scripts": {
     "build": "tsc -p tsconfig.json",
     "dev": "tsc -p tsconfig.json --watch",
     "lint": "eslint ./src ./test",
-    "test:file": "node --experimental-vm-modules --experimental-wasm-modules --experimental-wasm-threads ../../node_modules/jest/bin/jest.js",
-    "test": "npm run test:file -- ./src/** ./test/**",
-    "test:watch": "npm run test:file -- ./src/** ./test/** --watch"
+    "test:file": "node --experimental-vm-modules --experimental-wasm-modules ../../node_modules/jest/bin/jest.js",
+    "test": "npm run test:file -- ./test/**",
+    "test:watch": "npm run test:file -- ./test/** --watch"
   },
   "main": "dist/index.js",
   "publishConfig": {
     "access": "public"
   },
   "dependencies": {
+    "cachedir": "^2.4.0",
     "lodash": "^4.17.21",
     "loglevel": "^1.8.1",
     "reflect-metadata": "^0.1.13",
+    "ts-mixer": "^6.0.3",
     "typescript-memoize": "^1.1.1"
   },
   "peerDependencies": {
-    "snarkyjs": "0.12.0",
-    "tsyringe": "^4.7.0"
+    "o1js": "^2.10.0",
+    "tsyringe": "^4.10.0"
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
     "@types/lodash": "^4.14.194"
   },
-  "gitHead": "662735cd072170aa01a9f6040e89293a3e3fc8a9"
+  "gitHead": "8a7eca319272a15162dc4ad04bdc134b1017716d"
 }

package/src/cache/CacheManifest.ts

@@ -0,0 +1,57 @@
+import path from "node:path";
+import fs from "node:fs";
+
+import { injectable, singleton } from "tsyringe";
+import cachedir from "cachedir";
+
+@injectable()
+@singleton()
+export class CacheManifest {
+  public manifestFile(): string {
+    return path.format({
+      dir: cachedir("o1js"),
+      name: "protokit-cache-manifest",
+      ext: "json",
+    });
+  }
+
+  manifestRead = false;
+
+  manifest: string[] = [];
+
+  private readManifest(): string[] {
+    const file = this.manifestFile();
+    if (fs.existsSync(file)) {
+      return JSON.parse(fs.readFileSync(file).toString());
+    }
+    return [];
+  }
+
+  private ensureManifestRead() {
+    if (!this.manifestRead) {
+      this.manifest = this.readManifest();
+      this.manifestRead = true;
+    }
+  }
+
+  public getManifest() {
+    this.ensureManifestRead();
+
+    return this.manifest;
+  }
+
+  public writeToManifest(program: string) {
+    this.ensureManifestRead();
+
+    if (!this.manifest.includes(program)) {
+      this.manifest.push(program);
+      fs.writeFileSync(this.manifestFile(), JSON.stringify(this.manifest));
+    }
+  }
+
+  public includes(program: string): boolean {
+    this.ensureManifestRead();
+
+    return this.manifest.includes(program);
+  }
+}

package/src/cache/ProxyCache.ts

@@ -0,0 +1,30 @@
+import { CacheHeader, Cache as O1Cache } from "o1js";
+
+export class ProxyCache implements O1Cache {
+  private realCache = O1Cache.FileSystemDefault;
+
+  private log: string[] = [];
+
+  canWrite = true;
+
+  debug = false;
+
+  cacheDirectory = this.realCache.cacheDirectory;
+
+  public read(header: CacheHeader): Uint8Array | undefined {
+    return this.realCache.read(header);
+  }
+
+  public write(header: CacheHeader, value: Uint8Array): void {
+    this.log.push(header.persistentId);
+    return this.realCache.write(header, value);
+  }
+
+  public getLog() {
+    return this.log;
+  }
+
+  public startLog() {
+    this.log = [];
+  }
+}

package/src/cache/RemoteCache.ts

@@ -0,0 +1,9 @@
+import { Readable } from "node:stream";
+
+export interface RemoteCache {
+  storeObject(program: string, object: string, file: Readable): Promise<void>;
+
+  getObjects(program: string): Promise<string[]>;
+
+  readObject(program: string, object: string): Promise<Readable>;
+}

package/src/cache/RemoteCacheCompiler.ts

@@ -0,0 +1,131 @@
+import fs from "node:fs";
+import path from "node:path";
+
+import { inject, injectable, Lifecycle, scoped } from "tsyringe";
+import cachedir from "cachedir";
+
+import { log } from "../log";
+import { mapSequential } from "../utils";
+import {
+  CompileArtifact,
+  PlainZkProgram,
+} from "../zkProgrammable/ZkProgrammable";
+
+import { RemoteCache } from "./RemoteCache";
+import { ProxyCache } from "./ProxyCache";
+import { CacheManifest } from "./CacheManifest";
+
+@injectable()
+@scoped(Lifecycle.ContainerScoped)
+export class RemoteCacheCompiler {
+  public constructor(
+    @inject("RemoteCache", { isOptional: true })
+    private readonly remoteCache: RemoteCache | undefined,
+    private readonly manifest: CacheManifest
+  ) {
+    if (remoteCache === undefined) {
+      log.debug(
+        "No remote cache configured, only using local file system cache for circuits"
+      );
+    } else {
+      log.debug("Using remote cache for circuit caching");
+    }
+  }
+
+  private getFileFromObjectName(object: string): string {
+    const dir = cachedir("o1js");
+
+    return path.format({
+      dir,
+      base: object,
+    });
+  }
+
+  private async download(remoteCache: RemoteCache, name: string) {
+    const objects = await remoteCache.getObjects(name);
+
+    log.debug(`Downloading ${objects.length} cached objects for ${name}`);
+
+    await mapSequential(objects, async (object) => {
+      const readable = await remoteCache.readObject(name, object);
+      const file = this.getFileFromObjectName(object);
+
+      const writeStream = fs.createWriteStream(file);
+      readable.pipe(writeStream);
+
+      await new Promise<void>((res) => {
+        writeStream.on("close", res);
+      });
+    });
+  }
+
+  private async uploadFile(
+    remoteCache: RemoteCache,
+    program: string,
+    fileName: string
+  ) {
+    const file = path.resolve(cachedir("o1js"), fileName);
+    const readStream = fs.createReadStream(file);
+    await remoteCache.storeObject(program, fileName, readStream);
+  }
+
+  private async upload(
+    remoteCache: RemoteCache,
+    name: string,
+    identifiers: string[]
+  ) {
+    await mapSequential(identifiers, async (identifier) => {
+      await this.uploadFile(remoteCache, name, identifier);
+      await this.uploadFile(remoteCache, name, `${identifier}.header`);
+    });
+  }
+
+  private isSRSFile(file: string): boolean {
+    return file.includes("srs-") || file.includes("lagrange-");
+  }
+
+  private async compileWithRemoteCache(
+    remoteCache: RemoteCache,
+    program: Pick<PlainZkProgram, "name" | "compile">
+  ) {
+    const { name } = program;
+
+    if (!this.manifest.includes("srs")) {
+      await this.download(remoteCache, "srs");
+    }
+
+    if (!this.manifest.includes(name)) {
+      await this.download(remoteCache, name);
+    }
+
+    const cache = new ProxyCache();
+    cache.startLog();
+
+    const result = await program.compile({
+      cache,
+    });
+
+    const files = cache.getLog();
+    log.debug("Uploading files", files);
+
+    const srsFiles = files.filter((file) => this.isSRSFile(file));
+    await this.upload(remoteCache, "srs", srsFiles);
+    this.manifest.writeToManifest("srs");
+
+    const circuitFiles = files.filter((file) => !this.isSRSFile(file));
+    await this.upload(remoteCache, name, circuitFiles);
+    this.manifest.writeToManifest(name);
+
+    return result;
+  }
+
+  public async compileWithCache(
+    program: Pick<PlainZkProgram, "name" | "compile">
+  ): Promise<CompileArtifact> {
+    if (this.remoteCache !== undefined) {
+      return await this.compileWithRemoteCache(this.remoteCache, program);
+    } else {
+      return await program.compile();
+    }
+  }
+}

package/src/compiling/AtomicCompileHelper.ts

@@ -0,0 +1,71 @@
+import { inject, injectable } from "tsyringe";
+
+import {
+  AreProofsEnabled,
+  CompileArtifact,
+  MOCK_VERIFICATION_KEY,
+} from "../zkProgrammable/ZkProgrammable";
+import { isSubtypeOfName } from "../utils";
+import { TypedClass } from "../types";
+import { log } from "../log";
+import { RemoteCacheCompiler } from "../cache/RemoteCacheCompiler";
+
+export type ArtifactRecord = Record<string, CompileArtifact>;
+
+export type CompileTarget = {
+  name: string;
+  compile: () => Promise<CompileArtifact>;
+};
+
+@injectable()
+export class AtomicCompileHelper {
+  public constructor(
+    @inject("AreProofsEnabled")
+    private readonly areProofsEnabled: AreProofsEnabled,
+    private readonly remoteCacheCompiler: RemoteCacheCompiler
+  ) {}
+
+  private compilationPromises: {
+    [key: string]: Promise<CompileArtifact>;
+  } = {};
+
+  public async compileContract(
+    contract: CompileTarget,
+    overrideProofsEnabled?: boolean
+  ): Promise<CompileArtifact> {
+    let newPromise = false;
+    const { name } = contract;
+    if (this.compilationPromises[name] === undefined) {
+      const proofsEnabled =
+        overrideProofsEnabled ?? this.areProofsEnabled.areProofsEnabled;
+
+      // We only care about proofs enabled here if it's a contract, because
+      // in all other cases, ZkProgrammable already handles this switch, and we
+      // want to preserve the artifact layout (which might be more than one
+      // entry for ZkProgrammables)
+      if (
+        proofsEnabled ||
+        !isSubtypeOfName(
+          // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+          contract as unknown as TypedClass<any>,
+          "SmartContract"
+        )
+      ) {
+        log.time(`Compiling ${name}`);
+        this.compilationPromises[name] =
+          this.remoteCacheCompiler.compileWithCache(contract);
+        newPromise = true;
+      } else {
+        log.debug(`Compiling ${name} - mock`);
+        this.compilationPromises[name] = Promise.resolve({
+          verificationKey: MOCK_VERIFICATION_KEY,
+        });
+      }
+    }
+    const result = await this.compilationPromises[name];
+    if (newPromise) {
+      log.timeEnd.info(`Compiling ${name}`);
+    }
+    return result;
+  }
+}

package/src/compiling/CompilableModule.ts

@@ -0,0 +1,6 @@
+import type { CompileRegistry } from "./CompileRegistry";
+import type { ArtifactRecord } from "./AtomicCompileHelper";
+
+export interface CompilableModule {
+  compile(registry: CompileRegistry): Promise<ArtifactRecord | void>;
+}

package/src/compiling/CompileRegistry.ts

@@ -0,0 +1,64 @@
+import { injectable, singleton } from "tsyringe";
+
+import { CompileArtifact } from "../zkProgrammable/ZkProgrammable";
+
+import {
+  ArtifactRecord,
+  AtomicCompileHelper,
+  CompileTarget,
+} from "./AtomicCompileHelper";
+
+/**
+ * The CompileRegistry compiles "compilable modules"
+ * (i.e. zkprograms, contracts or contractmodules)
+ * while making sure they don't get compiled twice in the same process in parallel.
+ */
+@injectable()
+@singleton()
+export class CompileRegistry {
+  public constructor(private readonly compiler: AtomicCompileHelper) {}
+
+  private artifacts: ArtifactRecord = {};
+
+  private inForceProverBlock = 0;
+
+  /**
+   * This function forces compilation even if the artifact itself is in the registry.
+   * Basically the statement is: The artifact along is not enough, we need to
+   * actually have the prover compiled.
+   * This is true for non-sideloaded circuit dependencies.
+   */
+  public async forceProverExists<R>(
+    f: (registry: CompileRegistry) => Promise<R>
+  ): Promise<R> {
+    this.inForceProverBlock += 1;
+    const result = await f(this);
+    this.inForceProverBlock -= 1;
+    return result;
+  }
+
+  public async compile(target: CompileTarget, nameOverride?: string) {
+    const name = nameOverride ?? target.name;
+    if (this.artifacts[name] === undefined || this.inForceProverBlock > 0) {
+      const artifact = await this.compiler.compileContract(target);
+      this.artifacts[name] = artifact;
+      return artifact;
+    }
+    return this.artifacts[name];
+  }
+
+  public getArtifact(name: string): CompileArtifact | undefined {
+    return this.artifacts[name];
+  }
+
+  public addArtifactsRaw(artifacts: ArtifactRecord) {
+    this.artifacts = {
+      ...this.artifacts,
+      ...artifacts,
+    };
+  }
+
+  public getAllArtifacts() {
+    return this.artifacts;
+  }
+}

package/src/compiling/services/ChildVerificationKeyService.ts

@@ -0,0 +1,26 @@
+import { injectable, Lifecycle, scoped } from "tsyringe";
+
+import { CompileRegistry } from "../CompileRegistry";
+
+@injectable()
+@scoped(Lifecycle.ContainerScoped)
+export class ChildVerificationKeyService {
+  private compileRegistry?: CompileRegistry;
+
+  public setCompileRegistry(registry: CompileRegistry) {
+    this.compileRegistry = registry;
+  }
+
+  public getVerificationKey(name: string) {
+    if (this.compileRegistry === undefined) {
+      throw new Error("CompileRegistry hasn't been set yet");
+    }
+    const artifact = this.compileRegistry.getArtifact(name);
+    if (artifact === undefined) {
+      throw new Error(
+        `Verification Key for child program ${name} not found in registry`
+      );
+    }
+    return artifact.verificationKey;
+  }
+}

package/src/config/ChildContainerCreatable.ts

@@ -0,0 +1,5 @@
+import { ChildContainerProvider } from "./ChildContainerProvider";
+
+export interface ChildContainerCreatable {
+  create: (childContainerProvider: ChildContainerProvider) => void;
+}

package/src/config/ChildContainerProvider.ts

@@ -0,0 +1,5 @@
+import { DependencyContainer } from "tsyringe";
+
+export interface ChildContainerProvider {
+  (): DependencyContainer;
+}

package/src/config/ConfigurableModule.ts

@@ -1,3 +1,8 @@
+import { noop } from "../utils";
+
+import { ChildContainerProvider } from "./ChildContainerProvider";
+import type { BaseModuleInstanceType } from "./ModuleContainer";
+
 const errors = {
   configNotSet: (moduleName: string) =>
     new Error(
@@ -6,7 +11,7 @@ const errors = {
 };
 
 // defines how presets can be provided, either a function or a record
-export type Preset<Config> = Config | ((...args: any[]) => Config);
+export type Preset<Config> = Config | ((...args: unknown[]) => Config);
 export type Presets<Config> = Record<string, Preset<Config>>;
 
 // describes the interface of a configurable module
@@ -14,10 +19,14 @@ export interface Configurable<Config> {
   config: Config;
 }
 
+export type NoConfig = Record<never, never>;
+
 /**
  * Used by various module sub-types that may need to be configured
  */
-export class ConfigurableModule<Config> implements Configurable<Config> {
+export class ConfigurableModule<Config = NoConfig>
+  implements BaseModuleInstanceType
+{
   /**
    * Store the config separately, so that we can apply additional
    * checks when retrieving it via the getter
@@ -36,6 +45,10 @@ export class ConfigurableModule<Config> implements Configurable<Config> {
   public set config(config: Config) {
     this.currentConfig = config;
   }
+
+  public create(childContainerProvider: ChildContainerProvider): void {
+    noop();
+  }
 }
 
 // Helps ensure that the target class implements static presets