@apibara/protocol 0.4.8 → 2.0.0-beta.0

package/src/proto/testing.ts

@@ -1,11 +1,146 @@
-import { Message, Field } from 'protobufjs/light'
+// Code generated by protoc-gen-ts_proto. DO NOT EDIT.
+// versions:
+//   protoc-gen-ts_proto  v1.176.0
+//   protoc               unknown
+// source: testing.proto
 
-export class TestFilter extends Message<TestFilter> {
-  @Field.d(1, 'uint64')
-  // @ts-ignore
-  public num: number
+/* eslint-disable */
+import _m0 from "protobufjs/minimal";
 
-  @Field.d(2, 'string')
-  // @ts-ignore
-  public text: string
+export const protobufPackage = "dna.v2.testing";
+
+export interface MockFilter {
+  readonly filter?: string | undefined;
+}
+
+export interface MockBlock {
+  readonly data?: string | undefined;
+}
+
+function createBaseMockFilter(): MockFilter {
+  return { filter: undefined };
+}
+
+export const MockFilter = {
+  encode(message: MockFilter, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.filter !== undefined) {
+      writer.uint32(10).string(message.filter);
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): MockFilter {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseMockFilter() as any;
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.filter = reader.string();
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): MockFilter {
+    return { filter: isSet(object.filter) ? globalThis.String(object.filter) : undefined };
+  },
+
+  toJSON(message: MockFilter): unknown {
+    const obj: any = {};
+    if (message.filter !== undefined) {
+      obj.filter = message.filter;
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<MockFilter>): MockFilter {
+    return MockFilter.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<MockFilter>): MockFilter {
+    const message = createBaseMockFilter() as any;
+    message.filter = object.filter ?? undefined;
+    return message;
+  },
+};
+
+function createBaseMockBlock(): MockBlock {
+  return { data: undefined };
+}
+
+export const MockBlock = {
+  encode(message: MockBlock, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.data !== undefined) {
+      writer.uint32(10).string(message.data);
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): MockBlock {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseMockBlock() as any;
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.data = reader.string();
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): MockBlock {
+    return { data: isSet(object.data) ? globalThis.String(object.data) : undefined };
+  },
+
+  toJSON(message: MockBlock): unknown {
+    const obj: any = {};
+    if (message.data !== undefined) {
+      obj.data = message.data;
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<MockBlock>): MockBlock {
+    return MockBlock.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<MockBlock>): MockBlock {
+    const message = createBaseMockBlock() as any;
+    message.data = object.data ?? undefined;
+    return message;
+  },
+};
+
+type Builtin = Date | Function | Uint8Array | string | number | boolean | bigint | undefined;
+
+export type DeepPartial<T> = T extends Builtin ? T
+  : T extends globalThis.Array<infer U> ? globalThis.Array<DeepPartial<U>>
+  : T extends ReadonlyArray<infer U> ? ReadonlyArray<DeepPartial<U>>
+  : T extends { readonly $case: string }
+    ? { [K in keyof Omit<T, "$case">]?: DeepPartial<T[K]> } & { readonly $case: T["$case"] }
+  : T extends {} ? { [K in keyof T]?: DeepPartial<T[K]> }
+  : Partial<T>;
+
+function isSet(value: any): boolean {
+  return value !== null && value !== undefined;
 }

package/src/rate.ts

@@ -0,0 +1,54 @@
+/** Track data rate using high precision timers. */
+export class RateGauge {
+  private interval: number;
+  private prev?: bigint;
+  private rateMs?: number;
+  private var: number;
+
+  constructor(intervalSeconds: number) {
+    // Convert seconds to milliseconds.
+    this.interval = intervalSeconds * 1_000;
+    this.var = 0;
+  }
+
+  public record(items: number) {
+    // Compute the exponential moving average of the rate.
+    const prev = this.prev;
+    const now = process.hrtime.bigint();
+    this.prev = now;
+
+    if (!prev) {
+      return;
+    }
+
+    const deltaMs = Number(now - prev) / 1_000_000;
+    // rate in items/ms.
+    const rateMs = items / deltaMs;
+
+    if (this.rateMs === undefined) {
+      this.rateMs = rateMs;
+      this.var = 0;
+      return;
+    }
+
+    const alpha = 1 - Math.exp(-deltaMs / this.interval);
+    this.rateMs = alpha * rateMs + (1 - alpha) * this.rateMs;
+
+    const diff = rateMs - this.rateMs;
+    const incr = alpha * diff;
+    this.var = (1 - alpha) * (this.var + incr * diff);
+  }
+
+  /** Returns the average rate per second. */
+  public average() {
+    if (this.rateMs === undefined) {
+      return undefined;
+    }
+    return this.rateMs * 1_000;
+  }
+
+  /** Returns the variance. */
+  public variance() {
+    return this.var;
+  }
+}

package/src/status.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  StatusResponse,
+  statusRequestFromProto,
+  statusRequestToProto,
+  statusResponseFromProto,
+  statusResponseToProto,
+} from "./status";
+
+describe("StatusRequest", () => {
+  describe("proto", () => {
+    it("should encode and decode", () => {
+      const proto = statusRequestToProto({});
+      expect(proto).toMatchInlineSnapshot("{}");
+      const back = statusRequestFromProto(proto);
+      expect(back).toEqual({});
+    });
+  });
+});
+
+describe("StatusResponse", () => {
+  describe("proto", () => {
+    it("should encode and decode", () => {
+      const response = StatusResponse.make({
+        currentHead: {
+          orderKey: 123n,
+        },
+        lastIngested: {
+          orderKey: 123n,
+        },
+      });
+
+      const proto = statusResponseToProto(response);
+      expect(proto).toMatchInlineSnapshot(`
+        {
+          "currentHead": {
+            "orderKey": 123n,
+            "uniqueKey": Uint8Array [],
+          },
+          "lastIngested": {
+            "orderKey": 123n,
+            "uniqueKey": Uint8Array [],
+          },
+        }
+      `);
+      const back = statusResponseFromProto(proto);
+      expect(back).toEqual(response);
+    });
+  });
+});

package/src/status.ts

@@ -0,0 +1,22 @@
+import { Schema } from "@effect/schema";
+
+import { Cursor } from "./common";
+
+/** The request to the `status` endpoint. */
+export const StatusRequest = Schema.Struct({});
+
+export type StatusRequest = typeof StatusRequest.Type;
+
+export const statusRequestToProto = Schema.encodeSync(StatusRequest);
+export const statusRequestFromProto = Schema.decodeSync(StatusRequest);
+
+/** The response from the `status` endpoint. */
+export const StatusResponse = Schema.Struct({
+  currentHead: Schema.optional(Cursor),
+  lastIngested: Schema.optional(Cursor),
+});
+
+export type StatusResponse = typeof StatusResponse.Type;
+
+export const statusResponseToProto = Schema.encodeSync(StatusResponse);
+export const statusResponseFromProto = Schema.decodeSync(StatusResponse);

package/src/stream.test-d.ts

@@ -0,0 +1,33 @@
+import { Schema } from "@effect/schema";
+import { test } from "vitest";
+
+import { Data } from "./stream";
+
+const Inner = Schema.Struct({
+  data: Schema.String,
+});
+
+const Good = Schema.transform(Schema.Uint8ArrayFromSelf, Schema.NullOr(Inner), {
+  decode(value) {
+    throw new Error("not implemented");
+  },
+  encode(value) {
+    throw new Error("not implemented");
+  },
+});
+
+const Bad = Schema.transform(Schema.Uint8ArrayFromSelf, Inner, {
+  decode(value) {
+    throw new Error("not implemented");
+  },
+  encode(value) {
+    throw new Error("not implemented");
+  },
+});
+
+test("Data", () => {
+  const GoodData = Data(Good);
+
+  // @ts-expect-error
+  const BadData = Data(Bad);
+});

package/src/stream.test.ts

@@ -0,0 +1,238 @@
+import { describe, expect, it } from "vitest";
+
+import { Schema } from "@effect/schema";
+
+import {
+  Heartbeat,
+  Invalidate,
+  StreamDataRequest,
+  StreamDataResponse,
+  SystemMessage,
+} from "./stream";
+
+const InnerData = Schema.Struct({
+  value: Schema.String,
+});
+
+const TestData = Schema.transform(
+  Schema.Uint8ArrayFromSelf,
+  Schema.NullOr(InnerData),
+  {
+    decode(bytes) {
+      const value = new TextDecoder().decode(bytes);
+      return { value };
+    },
+    encode(value) {
+      if (value === null) {
+        return new Uint8Array();
+      }
+      return new TextEncoder().encode(value.value);
+    },
+  },
+);
+
+const TestStreamDataRequest = StreamDataRequest(TestData);
+
+const encodeTestRequest = Schema.encodeSync(TestStreamDataRequest);
+const decodeTestRequest = Schema.decodeSync(TestStreamDataRequest);
+
+const TestStreamDataResponse = StreamDataResponse(TestData);
+
+type TestStreamDataResponse = typeof TestStreamDataResponse.Type;
+
+const encodeTestResponse = Schema.encodeSync(TestStreamDataResponse);
+const decodeTestResponse = Schema.decodeSync(TestStreamDataResponse);
+
+describe("StreamDataRequest", () => {
+  it("encodes and decodes", () => {
+    const request = TestStreamDataRequest.make({
+      finality: "accepted",
+      startingCursor: {
+        orderKey: 5_000_000n,
+      },
+      filter: [{ value: "hello" }, { value: "world" }],
+    });
+
+    const proto = encodeTestRequest(request);
+    expect(proto).toMatchInlineSnapshot(`
+      {
+        "filter": [
+          Uint8Array [
+            104,
+            101,
+            108,
+            108,
+            111,
+          ],
+          Uint8Array [
+            119,
+            111,
+            114,
+            108,
+            100,
+          ],
+        ],
+        "finality": 2,
+        "startingCursor": {
+          "orderKey": 5000000n,
+          "uniqueKey": Uint8Array [],
+        },
+      }
+    `);
+    const back = decodeTestRequest(proto);
+    expect(request).toEqual(back);
+  });
+});
+
+describe("StreamDataResponse", () => {
+  const encode = encodeTestResponse;
+  const decode = decodeTestResponse;
+
+  describe(".data", () => {
+    it("encodes and decodes", () => {
+      const message: TestStreamDataResponse = {
+        _tag: "data",
+        data: {
+          finality: "accepted",
+          data: [{ value: "hello" }, { value: "world" }],
+        },
+      } as const;
+
+      const proto = encode(message);
+      expect(proto).toMatchInlineSnapshot(`
+        {
+          "$case": "data",
+          "data": {
+            "data": [
+              Uint8Array [
+                104,
+                101,
+                108,
+                108,
+                111,
+              ],
+              Uint8Array [
+                119,
+                111,
+                114,
+                108,
+                100,
+              ],
+            ],
+            "finality": 2,
+          },
+        }
+      `);
+      const back = decode(proto);
+      expect(back).toEqual(message);
+    });
+  });
+
+  describe(".invalidate", () => {
+    it("encodes and decodes", () => {
+      const invalidate = Invalidate.make({
+        _tag: "invalidate",
+        invalidate: {
+          cursor: {
+            orderKey: 5_000_000n,
+            uniqueKey: "0x1234567890",
+          },
+        },
+      });
+
+      const proto = encode(invalidate);
+      expect(proto).toMatchInlineSnapshot(`
+        {
+          "$case": "invalidate",
+          "invalidate": {
+            "cursor": {
+              "orderKey": 5000000n,
+              "uniqueKey": Uint8Array [
+                18,
+                52,
+                86,
+                120,
+                144,
+              ],
+            },
+          },
+        }
+      `);
+      const back = decode(proto);
+      expect(back).toEqual(invalidate);
+    });
+  });
+
+  describe(".heartbeat", () => {
+    it("encodes and decodes", () => {
+      const heartbeat = Heartbeat.make({
+        _tag: "heartbeat",
+      });
+
+      const proto = encode(heartbeat);
+      expect(proto).toMatchInlineSnapshot(`
+        {
+          "$case": "heartbeat",
+        }
+      `);
+      const back = decode(proto);
+      expect(back).toEqual(heartbeat);
+    });
+  });
+
+  describe(".systemMessage", () => {
+    it("encodes and decodes stdout", () => {
+      const message = SystemMessage.make({
+        _tag: "systemMessage",
+        systemMessage: {
+          output: {
+            _tag: "stdout",
+            stdout: "hello",
+          },
+        },
+      });
+
+      const proto = encode(message);
+      expect(proto).toMatchInlineSnapshot(`
+        {
+          "$case": "systemMessage",
+          "systemMessage": {
+            "output": {
+              "$case": "stdout",
+              "stdout": "hello",
+            },
+          },
+        }
+      `);
+      const back = decode(proto);
+      expect(back).toEqual(message);
+    });
+
+    it("encodes and decodes stderr", () => {
+      const message = SystemMessage.make({
+        _tag: "systemMessage",
+        systemMessage: {
+          output: {
+            _tag: "stderr",
+            stderr: "hello",
+          },
+        },
+      });
+
+      const proto = encode(message);
+      expect(proto).toMatchInlineSnapshot(`
+        {
+          "$case": "systemMessage",
+          "systemMessage": {
+            "output": {
+              "$case": "stderr",
+              "stderr": "hello",
+            },
+          },
+        }
+      `);
+      const back = decode(proto);
+      expect(back).toEqual(message);
+    });
+  });
+});

package/src/stream.ts

@@ -0,0 +1,128 @@
+import { Schema } from "@effect/schema";
+
+import { Cursor } from "./common";
+import * as proto from "./proto";
+
+/** Data finality. */
+export const DataFinality = Schema.transform(
+  Schema.Enums(proto.stream.DataFinality),
+  Schema.Literal("finalized", "accepted", "pending", "unknown"),
+  {
+    decode(value) {
+      const enumMap = {
+        [proto.stream.DataFinality.FINALIZED]: "finalized",
+        [proto.stream.DataFinality.ACCEPTED]: "accepted",
+        [proto.stream.DataFinality.PENDING]: "pending",
+        [proto.stream.DataFinality.UNKNOWN]: "unknown",
+        [proto.stream.DataFinality.UNRECOGNIZED]: "unknown",
+      } as const;
+
+      return enumMap[value] ?? "unknown";
+    },
+    encode(value) {
+      const enumMap = {
+        finalized: proto.stream.DataFinality.FINALIZED,
+        accepted: proto.stream.DataFinality.ACCEPTED,
+        pending: proto.stream.DataFinality.PENDING,
+        unknown: proto.stream.DataFinality.UNKNOWN,
+      };
+
+      return enumMap[value] ?? proto.stream.DataFinality.UNKNOWN;
+    },
+  },
+);
+
+export type DataFinality = typeof DataFinality.Type;
+
+/** Create a `StreamDataRequest` with the given filter schema. */
+export const StreamDataRequest = <TA, TR>(
+  filter: Schema.Schema<TA, Uint8Array, TR>,
+) =>
+  Schema.Struct({
+    finality: Schema.optional(DataFinality),
+    startingCursor: Schema.optional(Cursor),
+    filter: Schema.mutable(Schema.Array(filter)),
+  });
+
+export type StreamDataRequest<TA> = {
+  finality?: DataFinality | undefined;
+  startingCursor?: Cursor | undefined;
+  filter: TA[];
+};
+
+export const Invalidate = Schema.Struct({
+  _tag: tag("invalidate"),
+  invalidate: Schema.Struct({
+    cursor: Schema.optional(Cursor),
+  }),
+});
+
+export type Invalidate = typeof Invalidate.Type;
+
+export const Heartbeat = Schema.Struct({
+  _tag: tag("heartbeat"),
+});
+
+export type Heartbeat = typeof Heartbeat.Type;
+
+export const StdOut = Schema.Struct({
+  _tag: tag("stdout"),
+  stdout: Schema.String,
+});
+
+export type StdOut = typeof StdOut.Type;
+
+export const StdErr = Schema.Struct({
+  _tag: tag("stderr"),
+  stderr: Schema.String,
+});
+
+export type StdErr = typeof StdErr.Type;
+
+export const SystemMessage = Schema.Struct({
+  _tag: tag("systemMessage"),
+  systemMessage: Schema.Struct({
+    output: Schema.optional(Schema.Union(StdOut, StdErr)),
+  }),
+});
+
+export type SystemMessage = typeof SystemMessage.Type;
+
+export const Data = <TA, TR>(
+  schema: Schema.Schema<TA | null, Uint8Array, TR>,
+) =>
+  Schema.Struct({
+    _tag: tag("data"),
+    data: Schema.Struct({
+      cursor: Schema.optional(Cursor),
+      endCursor: Schema.optional(Cursor),
+      finality: DataFinality,
+      data: Schema.Array(schema),
+    }),
+  });
+
+export const StreamDataResponse = <TA, TR>(
+  data: Schema.Schema<TA | null, Uint8Array, TR>,
+) => Schema.Union(Data(data), Invalidate, Heartbeat, SystemMessage);
+
+const ResponseWithoutData = Schema.Union(Invalidate, Heartbeat, SystemMessage);
+type ResponseWithoutData = typeof ResponseWithoutData.Type;
+
+export type StreamDataResponse<TA> =
+  | ResponseWithoutData
+  | {
+      _tag: "data";
+      data: {
+        cursor?: Cursor | undefined;
+        endCursor?: Cursor | undefined;
+        finality: DataFinality;
+        data: readonly (TA | null)[];
+      };
+    };
+
+function tag<T extends string>(tag: T) {
+  return Schema.Literal(tag).pipe(
+    Schema.propertySignature,
+    Schema.fromKey("$case"),
+  );
+}