@anterior/brrr 0.1.1

package/src/codec.test.ts

@@ -0,0 +1,91 @@
+import { suite, test } from "node:test";
+import { deepStrictEqual, notDeepStrictEqual } from "node:assert/strict";
+import type { Codec } from "./codec.ts";
+
+export async function codecContractTest(codec: Codec) {
+  await suite("store-contract", async () => {
+    const cases: Record<string, [unknown[], unknown[]]> = {
+      simple: [
+        [1, 2],
+        [1, 2],
+      ],
+      "simple-object": [[{ b: 2, a: 1 }], [{ a: 1, b: 2 }]],
+      "nested object": [
+        [{ b: { c: 3, a: 1 }, a: 2 }],
+        [{ a: 2, b: { a: 1, c: 3 } }],
+      ],
+      "array of objects": [
+        [
+          [
+            { b: 2, a: 1 },
+            { d: 4, c: 3 },
+          ],
+        ],
+        [
+          [
+            { a: 1, b: 2 },
+            { c: 3, d: 4 },
+          ],
+        ],
+      ],
+      "complex object": [
+        [{ a: 1, b: { c: 3, d: [4, 5] }, e: "6" }],
+        [{ e: "6", b: { d: [4, 5], c: 3 }, a: 1 }],
+      ],
+      nulls: [
+        [{ a: null, b: [1, 2, null], c: { d: null } }],
+        [{ c: { d: null }, b: [1, 2, null], a: null }],
+      ],
+      undefineds: [
+        [{ a: undefined, b: [1, 2, undefined], c: { d: undefined } }],
+        [{ c: { d: undefined }, b: [1, 2, undefined], a: undefined }],
+      ],
+      empties: [[{ a: {}, b: [] }], [{ b: [], a: {} }]],
+      "mixed types": [
+        [{ a: 1, b: "2", c: true, d: null }],
+        [{ d: null, c: true, b: "2", a: 1 }],
+      ],
+    } as const;
+
+    await suite("deterministic call hash", async () => {
+      for (const [name, args] of Object.entries(cases)) {
+        await test(name, async () => {
+          // sanity check
+          deepStrictEqual(...args);
+
+          deepStrictEqual(
+            await codec.encodeCall("foo", args[0]),
+            await codec.encodeCall("foo", args[1]),
+          );
+        });
+      }
+    });
+
+    await suite(
+      "round trip: encodeCall -> invokeTask -> decodeReturn",
+      async () => {
+        async function identify<T>(a: T): Promise<T> {
+          return a;
+        }
+
+        for (const [name, args] of Object.entries(cases)) {
+          await test(name, async () => {
+            const call = await codec.encodeCall(identify.name, [args[0]]);
+            const result = await codec.invokeTask(call, identify);
+            const decoded = await codec.decodeReturn(identify.name, result);
+            deepStrictEqual(decoded, await identify(args[1]));
+          });
+        }
+      },
+    );
+
+    await test("different arguments produce different hashes", async () => {
+      const a = await codec.encodeCall("foo", [1, 2]);
+      const b = await codec.encodeCall("foo", [2, 1]);
+      const c = await codec.encodeCall("bar", [1, 2]);
+      notDeepStrictEqual(a, b);
+      notDeepStrictEqual(a, c);
+      notDeepStrictEqual(b, c);
+    });
+  });
+}

package/src/codec.ts

@@ -0,0 +1,12 @@
+import type { Call } from "./call.ts";
+
+export interface Codec {
+  encodeCall<A extends unknown[]>(taskName: string, args: A): Promise<Call>;
+
+  invokeTask<A extends unknown[], R>(
+    call: Call,
+    task: (...args: A) => Promise<R>,
+  ): Promise<Uint8Array>;
+
+  decodeReturn(taskName: string, payload: Uint8Array): unknown;
+}

package/src/connection.ts

@@ -0,0 +1,186 @@
+import type { Call } from "./call.ts";
+import { type Cache, Memory, type Store } from "./store.ts";
+import { SpawnLimitError } from "./errors.ts";
+import { randomUUID } from "node:crypto";
+import type { Publisher, Subscriber } from "./emitter.ts";
+import { BrrrShutdownSymbol, BrrrTaskDoneEventSymbol } from "./symbol.ts";
+import { PendingReturn, ScheduleMessage, TaggedTuple } from "./tagged-tuple.ts";
+
+export interface DeferredCall {
+  readonly topic: string | undefined;
+  readonly call: Call;
+}
+
+export class Defer {
+  public readonly calls: DeferredCall[];
+
+  public constructor(...calls: DeferredCall[]) {
+    this.calls = calls;
+  }
+}
+
+export interface Request {
+  readonly call: Call;
+}
+
+export interface Response {
+  readonly payload: Uint8Array;
+}
+
+type RequestHandler = (
+  request: Request,
+  connection: Connection,
+) => Promise<Response | Defer>;
+
+export class Connection {
+  public readonly cache: Cache;
+  public readonly memory: Memory;
+  public readonly emitter: Publisher;
+  public readonly spawnLimit = 10_000;
+
+  public constructor(store: Store, cache: Cache, emitter: Publisher) {
+    this.cache = cache;
+    this.memory = new Memory(store);
+    this.emitter = emitter;
+  }
+
+  public async putJob(topic: string, job: ScheduleMessage): Promise<void> {
+    if ((await this.cache.incr(`brrr_count/${job.rootId}`)) > this.spawnLimit) {
+      throw new SpawnLimitError(this.spawnLimit, job.rootId, job.callHash);
+    }
+    await this.emitter.emit(topic, TaggedTuple.encodeToString(job));
+  }
+
+  public async scheduleRaw(topic: string, call: Call): Promise<void> {
+    if (await this.memory.hasValue(call.callHash)) {
+      return;
+    }
+    await this.memory.setCall(call);
+    const rootId = randomUUID().replaceAll("-", "");
+    await this.putJob(topic, new ScheduleMessage(rootId, call.callHash));
+  }
+
+  public async readRaw(callHash: string): Promise<Uint8Array | undefined> {
+    return this.memory.getValue(callHash);
+  }
+}
+
+export class Server extends Connection {
+  public constructor(store: Store, cache: Cache, emitter: Publisher) {
+    super(store, cache, emitter);
+  }
+
+  public async loop(
+    topic: string,
+    handler: RequestHandler,
+    getMessage: () => Promise<string | typeof BrrrShutdownSymbol | undefined>,
+  ) {
+    while (true) {
+      const message = await getMessage();
+      if (!message) {
+        continue;
+      }
+      if (message === BrrrShutdownSymbol) {
+        break;
+      }
+      const call = await this.handleMessage(handler, topic, message);
+      if (call) {
+        await this.emitter.emitEventSymbol?.(BrrrTaskDoneEventSymbol, call);
+      }
+    }
+  }
+
+  protected async handleMessage(
+    requestHandler: RequestHandler,
+    topic: string,
+    payload: string,
+  ): Promise<Call | undefined> {
+    const message = TaggedTuple.decodeFromString(ScheduleMessage, payload);
+    const call = await this.memory.getCall(message.callHash);
+    const handled = await requestHandler({ call }, this);
+    if (handled instanceof Defer) {
+      await Promise.all(
+        handled.calls.map((child) => {
+          return this.scheduleCallNested(topic, child, message);
+        }),
+      );
+      return;
+    }
+    await this.memory.setValue(message.callHash, handled.payload);
+    let spawnLimitError: SpawnLimitError;
+    await this.memory.withPendingReturnsRemove(
+      message.callHash,
+      async (returns) => {
+        for (const pending of returns) {
+          try {
+            await this.scheduleReturnCall(pending);
+          } catch (err) {
+            if (err instanceof SpawnLimitError) {
+              spawnLimitError = err;
+              continue;
+            }
+            throw err;
+          }
+        }
+        if (spawnLimitError) {
+          throw spawnLimitError;
+        }
+      },
+    );
+    return call;
+  }
+
+  private async scheduleReturnCall(
+    pendingReturn: PendingReturn,
+  ): Promise<void> {
+    const job = new ScheduleMessage(
+      pendingReturn.rootId,
+      pendingReturn.callHash,
+    );
+    await this.putJob(pendingReturn.topic, job);
+  }
+
+  private async scheduleCallNested(
+    topic: string,
+    child: DeferredCall,
+    parent: ScheduleMessage,
+  ): Promise<void> {
+    await this.memory.setCall(child.call);
+    const callHash = child.call.callHash;
+    const pendingReturn = new PendingReturn(
+      parent.rootId,
+      parent.callHash,
+      topic,
+    );
+    const shouldSchedule = await this.memory.addPendingReturns(
+      callHash,
+      pendingReturn,
+    );
+    if (shouldSchedule) {
+      const job = new ScheduleMessage(parent.rootId, callHash);
+      await this.putJob(child.topic || topic, job);
+    }
+  }
+}
+
+export class SubscriberServer extends Server {
+  public override readonly emitter: Publisher & Subscriber;
+
+  public constructor(
+    store: Store,
+    cache: Cache,
+    emitter: Publisher & Subscriber,
+  ) {
+    super(store, cache, emitter);
+    this.emitter = emitter;
+  }
+
+  public listen(topic: string, handler: RequestHandler) {
+    this.emitter.on(topic, async (callId: string): Promise<void> => {
+      const result = await this.handleMessage(handler, topic, callId);
+      if (result) {
+        await this.emitter.emitEventSymbol?.(BrrrTaskDoneEventSymbol, result);
+      }
+    });
+  }
+}

package/src/emitter.ts

@@ -0,0 +1,20 @@
+import type { Call } from "./call.ts";
+import { BrrrTaskDoneEventSymbol } from "./symbol.ts";
+
+export interface Publisher {
+  emit(topic: string, callId: string): Promise<void>;
+
+  emitEventSymbol?(
+    event: typeof BrrrTaskDoneEventSymbol,
+    call: Call,
+  ): Promise<void>;
+}
+
+export interface Subscriber {
+  on(topic: string, listener: (callId: string) => void): void;
+
+  onEventSymbol?(
+    event: typeof BrrrTaskDoneEventSymbol,
+    listener: (call: Call) => void,
+  ): void;
+}

package/src/errors.ts

@@ -0,0 +1,50 @@
+import type { MemKey } from "./store.ts";
+import type { Tagged } from "./tagged-tuple.ts";
+
+abstract class BrrrError extends Error {
+  protected constructor(message: string) {
+    super();
+    this.name = this.constructor.name;
+    this.message = message;
+  }
+}
+
+export class NotFoundError extends BrrrError {
+  public constructor(key: MemKey) {
+    super(`Not found: ${key.type}/${key.callHash}`);
+  }
+}
+
+export class CasRetryLimitReachedError extends BrrrError {
+  public constructor(retryLimit: number) {
+    super(`CAS retry limit reached (${retryLimit})`);
+  }
+}
+
+export class SpawnLimitError extends BrrrError {
+  public constructor(limit: number, rootId: string, callHash: string) {
+    super(
+      `Spawn limit of ${limit} reached for rootId ${rootId} and callHash ${callHash}`,
+    );
+  }
+}
+
+export class TaskNotFoundError extends BrrrError {
+  public constructor(taskName: string) {
+    super(`Task not found: ${taskName}`);
+  }
+}
+
+export class TagMismatchError extends BrrrError {
+  public constructor(clz: Tagged) {
+    super(`Tag mismatch for ${clz.name}: expected ${clz.tag}`);
+  }
+}
+
+export class MalformedTaggedTupleError extends BrrrError {
+  public constructor(clz: Tagged) {
+    super(
+      `Malformed tagged tuple for ${clz.name}, expected ${clz.length} elements`,
+    );
+  }
+}

package/src/internal-codecs.ts

@@ -0,0 +1,28 @@
+import bencode from "bencode";
+import { Buffer } from "node:buffer";
+import type { Encoding } from "node:crypto";
+import { TextDecoder, TextEncoder } from "node:util";
+
+/**
+ * Brrr uses UTF-8 for encoding
+ */
+export const encoding = "utf-8" as const satisfies Encoding;
+
+/**
+ * Bencode encoding and decoding utility.
+ */
+export const bencoder = {
+  encode(data: unknown): Uint8Array {
+    return bencode.encode(data);
+  },
+  decode(data: Uint8Array, _encoding?: typeof encoding): unknown {
+    const buffer = Buffer.from(data);
+    return bencode.decode(buffer, _encoding);
+  },
+} as const;
+
+/**
+ * Exports TextEncoder and TextDecoder instances for UTF-8 encoding.
+ */
+export const encoder = new TextEncoder();
+export const decoder = new TextDecoder(encoding);

package/src/local-app.ts

@@ -0,0 +1,93 @@
+import { SubscriberServer } from "./connection.ts";
+import {
+  AppWorker,
+  type Handlers,
+  type NoAppTask,
+  type StripLeadingActiveWorker,
+  type TaskIdentifier,
+  taskIdentifierToName,
+} from "./app.ts";
+import type { Codec } from "./codec.ts";
+import {
+  InMemoryCache,
+  InMemoryEmitter,
+  InMemoryStore,
+} from "./backends/in-memory.ts";
+import { NotFoundError } from "./errors.ts";
+import { BrrrTaskDoneEventSymbol } from "./symbol.ts";
+
+export class LocalApp {
+  public readonly topic: string;
+  public readonly server: SubscriberServer;
+  public readonly app: AppWorker;
+
+  private hasRun = false;
+
+  public constructor(topic: string, server: SubscriberServer, app: AppWorker) {
+    this.topic = topic;
+    this.server = server;
+    this.app = app;
+  }
+
+  public schedule<A extends unknown[], R>(
+    handler: Parameters<typeof this.app.schedule<A, R>>[0],
+  ): NoAppTask<A, void> {
+    return this.app.schedule(handler, this.topic);
+  }
+
+  public read<A extends unknown[], R>(
+    ...args: Parameters<typeof this.app.read<A, R>>
+  ): NoAppTask<A, R> {
+    return this.app.read(...args);
+  }
+
+  public run(): void {
+    if (this.hasRun) {
+      throw new Error("LocalApp has already been run");
+    }
+    this.hasRun = true;
+    this.server.listen(this.topic, this.app.handle);
+  }
+}
+
+export class LocalBrrr {
+  private readonly topic: string;
+  private readonly handlers: Handlers;
+  private readonly codec: Codec;
+
+  public constructor(topic: string, handlers: Handlers, codec: Codec) {
+    this.topic = topic;
+    this.handlers = handlers;
+    this.codec = codec;
+  }
+
+  public run<A extends unknown[], R>(taskIdentifier: TaskIdentifier<A, R>) {
+    const store = new InMemoryStore();
+    const cache = new InMemoryCache();
+    const emitter = new InMemoryEmitter();
+    const server = new SubscriberServer(store, cache, emitter);
+    const worker = new AppWorker(this.codec, server, this.handlers);
+    const localApp = new LocalApp(this.topic, server, worker);
+    const taskName = taskIdentifierToName(taskIdentifier, this.handlers);
+    return async (...args: StripLeadingActiveWorker<A>): Promise<R> => {
+      localApp.run();
+      await localApp.schedule(taskName)(...args);
+      const call = await this.codec.encodeCall(taskName, args);
+      return new Promise((resolve) => {
+        emitter.onEventSymbol(BrrrTaskDoneEventSymbol, async ({ callHash }) => {
+          if (callHash === call.callHash) {
+            const payload = await server.readRaw(callHash);
+            if (!payload) {
+              throw new NotFoundError({
+                type: "value",
+                callHash,
+              });
+            }
+            const result = this.codec.decodeReturn(taskName, payload) as R;
+            resolve(result);
+          }
+        });
+      });
+    };
+  }
+}

package/src/naive-json-codec.test.ts

@@ -0,0 +1,9 @@
+import { suite, test } from "node:test";
+import { NaiveJsonCodec } from "./naive-json-codec.ts";
+import { codecContractTest } from "./codec.test.ts";
+
+await suite(import.meta.filename, async () => {
+  await test(NaiveJsonCodec.name, async () => {
+    await codecContractTest(new NaiveJsonCodec());
+  });
+});

package/src/naive-json-codec.ts

@@ -0,0 +1,72 @@
+import { type BinaryToTextEncoding, createHash } from "node:crypto";
+import type { Call } from "./call.ts";
+import type { Codec } from "./codec.ts";
+import { parse, stringify } from "superjson";
+import { decoder, encoder } from "./internal-codecs.ts";
+
+/**
+ * Naive JSON codec that uses `superjson` for serialization and deserialization.
+ *
+ * It tries its best to ensure that the serialized data is deterministic by
+ * sorting object keys recursively before serialization, but it's not
+ * reccommended for production use; the primary purpose of this codec is
+ * executable documentation.
+ */
+export class NaiveJsonCodec implements Codec {
+  public static readonly algorithm = "sha256";
+  public static readonly binaryToTextEncoding =
+    "hex" satisfies BinaryToTextEncoding;
+
+  public async decodeReturn(_: string, payload: Uint8Array): Promise<unknown> {
+    const decoded = decoder.decode(payload);
+    return parse(decoded);
+  }
+
+  public async encodeCall<A extends unknown[]>(
+    taskName: string,
+    args: A,
+  ): Promise<Call> {
+    const sortedArgs = args.map(NaiveJsonCodec.sortObjectKeys);
+    const data = stringify(sortedArgs);
+    const payload = encoder.encode(data);
+    const callHash = await this.hashCall(taskName, sortedArgs);
+    return { taskName, payload, callHash };
+  }
+
+  public async invokeTask<A extends unknown[], R>(
+    call: Call,
+    task: (...args: A) => Promise<R>,
+  ): Promise<Uint8Array> {
+    const decoded = decoder.decode(call.payload);
+    const args = parse<A>(decoded);
+    const result = await task(...args);
+    const resultJson = stringify(result);
+    return encoder.encode(resultJson);
+  }
+
+  private async hashCall<A extends unknown>(
+    taskName: string,
+    args: A,
+  ): Promise<string> {
+    const data = stringify([taskName, args]);
+    return createHash(NaiveJsonCodec.algorithm)
+      .update(data)
+      .digest(NaiveJsonCodec.binaryToTextEncoding);
+  }
+
+  private static sortObjectKeys<T>(unordered: T): T {
+    if (!unordered || typeof unordered !== "object") {
+      return unordered;
+    }
+    if (Array.isArray(unordered)) {
+      return unordered.map(NaiveJsonCodec.sortObjectKeys) as T;
+    }
+    const entries = Object.keys(unordered)
+      .sort()
+      .map((key) => [
+        key,
+        NaiveJsonCodec.sortObjectKeys(unordered[key as keyof typeof unordered]),
+      ]);
+    return Object.fromEntries(entries);
+  }
+}