@prisma/streams-server 0.0.1 → 0.1.1

package/src/objectstore/mock_r2.ts

@@ -0,0 +1,269 @@
+import { createHash } from "node:crypto";
+import { mkdirSync, readFileSync, writeFileSync, unlinkSync, openSync, closeSync, readSync, copyFileSync, createReadStream } from "node:fs";
+import { dirname, join } from "node:path";
+import type { GetOptions, ObjectStore, PutResult } from "./interface";
+import { dsError } from "../util/ds_error.ts";
+
+export type MockR2Faults = {
+  putDelayMs?: number;
+  getDelayMs?: number;
+  headDelayMs?: number;
+  listDelayMs?: number;
+  failPutPrefix?: string; // fail PUT when key starts with prefix
+  failPutEvery?: number; // fail every Nth PUT
+  failGetEvery?: number; // fail every Nth GET
+  failHeadEvery?: number;
+  failListEvery?: number;
+  timeoutPutEvery?: number;
+  timeoutGetEvery?: number;
+  timeoutHeadEvery?: number;
+  timeoutListEvery?: number;
+  partialGetEvery?: number; // return truncated bytes every Nth GET
+};
+
+export type MockR2Options = {
+  faults?: MockR2Faults;
+  maxInMemoryBytes?: number; // spill when object exceeds this
+  spillDir?: string; // optional directory for large objects
+};
+
+type StoredObject = {
+  etag: string;
+  size: number;
+  bytes?: Uint8Array;
+  path?: string;
+};
+
+function sleep(ms: number): Promise<void> {
+  if (ms <= 0) return Promise.resolve();
+  return new Promise((res) => setTimeout(res, ms));
+}
+
+export class MockR2Store implements ObjectStore {
+  private readonly data = new Map<string, StoredObject>();
+  private readonly faults: MockR2Faults;
+  private readonly maxInMemoryBytes: number;
+  private readonly spillDir?: string;
+
+  private putCount = 0;
+  private getCount = 0;
+  private headCount = 0;
+  private listCount = 0;
+  private memBytes = 0;
+
+  constructor(opts: MockR2Options | MockR2Faults = {}) {
+    if ("failPutEvery" in opts || "putDelayMs" in opts) {
+      this.faults = opts as MockR2Faults;
+      this.maxInMemoryBytes = Number.POSITIVE_INFINITY;
+    } else {
+      const o = opts as MockR2Options;
+      this.faults = o.faults ?? {};
+      this.maxInMemoryBytes = o.maxInMemoryBytes ?? Number.POSITIVE_INFINITY;
+      this.spillDir = o.spillDir;
+    }
+  }
+
+  private mkEtag(bytes: Uint8Array): string {
+    return createHash("sha256").update(bytes).digest("hex");
+  }
+
+  private async hashFile(path: string): Promise<string> {
+    const hash = createHash("sha256");
+    await new Promise<void>((resolve, reject) => {
+      const stream = createReadStream(path);
+      stream.on("data", (chunk) => hash.update(chunk));
+      stream.on("error", (err) => reject(err));
+      stream.on("end", () => resolve());
+    });
+    return hash.digest("hex");
+  }
+
+  private mkPath(key: string): string {
+    const hex = createHash("sha256").update(key).digest("hex");
+    return this.spillDir ? join(this.spillDir, `${hex}.bin`) : hex;
+  }
+
+  private shouldSpill(len: number): boolean {
+    return this.spillDir != null && len > this.maxInMemoryBytes;
+  }
+
+  private maybeFail(count: number, every?: number, msg?: string): void {
+    if (every && count % every === 0) throw dsError(msg ?? "MockR2: injected failure");
+  }
+
+  private maybeTimeout(count: number, every?: number, msg?: string): void {
+    if (every && count % every === 0) throw dsError(msg ?? "MockR2: injected timeout");
+  }
+
+  async put(key: string, bytes: Uint8Array): Promise<PutResult> {
+    this.putCount++;
+    if (this.faults.failPutPrefix && key.startsWith(this.faults.failPutPrefix)) {
+      throw dsError(`MockR2: injected PUT failure for ${key}`);
+    }
+    this.maybeFail(this.putCount, this.faults.failPutEvery, `MockR2: injected PUT failure for ${key}`);
+    this.maybeTimeout(this.putCount, this.faults.timeoutPutEvery, `MockR2: injected PUT timeout for ${key}`);
+    await sleep(this.faults.putDelayMs ?? 0);
+
+    const copy = new Uint8Array(bytes);
+    const etag = this.mkEtag(copy);
+    const size = copy.byteLength;
+
+    const existing = this.data.get(key);
+    if (existing?.bytes) this.memBytes -= existing.bytes.byteLength;
+    if (existing?.path) {
+      try { unlinkSync(existing.path); } catch { /* ignore */ }
+    }
+
+    if (this.shouldSpill(size)) {
+      const path = this.mkPath(key);
+      mkdirSync(dirname(path), { recursive: true });
+      writeFileSync(path, copy);
+      this.data.set(key, { etag, size, path });
+    } else {
+      this.memBytes += size;
+      this.data.set(key, { etag, size, bytes: copy });
+    }
+
+    return { etag };
+  }
+
+  async putFile(key: string, path: string, size: number): Promise<PutResult> {
+    this.putCount++;
+    if (this.faults.failPutPrefix && key.startsWith(this.faults.failPutPrefix)) {
+      throw dsError(`MockR2: injected PUT failure for ${key}`);
+    }
+    this.maybeFail(this.putCount, this.faults.failPutEvery, `MockR2: injected PUT failure for ${key}`);
+    this.maybeTimeout(this.putCount, this.faults.timeoutPutEvery, `MockR2: injected PUT timeout for ${key}`);
+    await sleep(this.faults.putDelayMs ?? 0);
+
+    const etag = await this.hashFile(path);
+
+    const existing = this.data.get(key);
+    if (existing?.bytes) this.memBytes -= existing.bytes.byteLength;
+    if (existing?.path) {
+      try { unlinkSync(existing.path); } catch { /* ignore */ }
+    }
+
+    if (this.shouldSpill(size)) {
+      const dest = this.mkPath(key);
+      mkdirSync(dirname(dest), { recursive: true });
+      copyFileSync(path, dest);
+      this.data.set(key, { etag, size, path: dest });
+    } else {
+      const bytes = new Uint8Array(await Bun.file(path).arrayBuffer());
+      this.memBytes += bytes.byteLength;
+      this.data.set(key, { etag, size, bytes });
+    }
+
+    return { etag };
+  }
+
+  async get(key: string, opts: GetOptions = {}): Promise<Uint8Array | null> {
+    this.getCount++;
+    this.maybeFail(this.getCount, this.faults.failGetEvery, `MockR2: injected GET failure for ${key}`);
+    this.maybeTimeout(this.getCount, this.faults.timeoutGetEvery, `MockR2: injected GET timeout for ${key}`);
+    await sleep(this.faults.getDelayMs ?? 0);
+
+    const entry = this.data.get(key);
+    if (!entry) return null;
+
+    const range = opts.range;
+    const total = entry.size;
+    const start = range?.start ?? 0;
+    const end = range?.end ?? total - 1;
+    if (start >= total) return new Uint8Array(0);
+    const clampEnd = Math.min(end, total - 1);
+    const length = Math.max(0, clampEnd - start + 1);
+
+    let out: Uint8Array;
+    if (entry.bytes) {
+      out = entry.bytes.slice(start, start + length);
+    } else if (entry.path) {
+      if (length === total) {
+        out = new Uint8Array(readFileSync(entry.path));
+      } else {
+        const fd = openSync(entry.path, "r");
+        try {
+          const buf = new Uint8Array(length);
+          readSync(fd, buf, 0, length, start);
+          out = buf;
+        } finally {
+          closeSync(fd);
+        }
+      }
+    } else {
+      return null;
+    }
+
+    if (this.faults.partialGetEvery && this.getCount % this.faults.partialGetEvery === 0) {
+      const half = Math.max(0, Math.floor(out.byteLength / 2));
+      return out.slice(0, half);
+    }
+
+    return out;
+  }
+
+  async head(key: string): Promise<{ etag: string; size: number } | null> {
+    this.headCount++;
+    this.maybeFail(this.headCount, this.faults.failHeadEvery, `MockR2: injected HEAD failure for ${key}`);
+    this.maybeTimeout(this.headCount, this.faults.timeoutHeadEvery, `MockR2: injected HEAD timeout for ${key}`);
+    await sleep(this.faults.headDelayMs ?? 0);
+
+    const v = this.data.get(key);
+    if (!v) return null;
+    return { etag: v.etag, size: v.size };
+  }
+
+  async delete(key: string): Promise<void> {
+    const v = this.data.get(key);
+    if (v?.bytes) this.memBytes -= v.bytes.byteLength;
+    if (v?.path) {
+      try { unlinkSync(v.path); } catch { /* ignore */ }
+    }
+    this.data.delete(key);
+  }
+
+  async list(prefix: string): Promise<string[]> {
+    this.listCount++;
+    this.maybeFail(this.listCount, this.faults.failListEvery, "MockR2: injected LIST failure");
+    this.maybeTimeout(this.listCount, this.faults.timeoutListEvery, "MockR2: injected LIST timeout");
+    await sleep(this.faults.listDelayMs ?? 0);
+
+    const out: string[] = [];
+    for (const k of this.data.keys()) {
+      if (k.startsWith(prefix)) out.push(k);
+    }
+    out.sort();
+    return out;
+  }
+
+  // Helpers for tests
+  has(key: string): boolean {
+    return this.data.has(key);
+  }
+
+  size(): number {
+    return this.data.size;
+  }
+
+  memoryBytes(): number {
+    return this.memBytes;
+  }
+
+  stats(): { puts: number; gets: number; heads: number; lists: number; memoryBytes: number } {
+    return {
+      puts: this.putCount,
+      gets: this.getCount,
+      heads: this.headCount,
+      lists: this.listCount,
+      memoryBytes: this.memBytes,
+    };
+  }
+
+  resetStats(): void {
+    this.putCount = 0;
+    this.getCount = 0;
+    this.headCount = 0;
+    this.listCount = 0;
+  }
+}

package/src/objectstore/null.ts

@@ -0,0 +1,32 @@
+import type { GetOptions, ObjectStore, PutResult } from "./interface";
+import { dsError } from "../util/ds_error.ts";
+
+function disabled(op: string, key?: string): never {
+  throw dsError(`object store disabled in local mode (${op}${key ? `: ${key}` : ""})`);
+}
+
+export class NullObjectStore implements ObjectStore {
+  async put(key: string, _data: Uint8Array, _opts?: { contentType?: string; contentLength?: number }): Promise<PutResult> {
+    return disabled("put", key);
+  }
+
+  async putFile(key: string, _path: string, _size: number, _opts?: { contentType?: string }): Promise<PutResult> {
+    return disabled("putFile", key);
+  }
+
+  async get(key: string, _opts?: GetOptions): Promise<Uint8Array | null> {
+    return disabled("get", key);
+  }
+
+  async head(key: string): Promise<{ etag: string; size: number } | null> {
+    return disabled("head", key);
+  }
+
+  async delete(key: string): Promise<void> {
+    return disabled("delete", key);
+  }
+
+  async list(prefix: string): Promise<string[]> {
+    return disabled("list", prefix);
+  }
+}

package/src/objectstore/r2.ts

@@ -0,0 +1,128 @@
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import type { GetOptions, ObjectStore, PutResult } from "./interface";
+import { dsError } from "../util/ds_error.ts";
+
+export type R2Config = {
+  accountId: string;
+  bucket: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+  region?: string;
+};
+
+function sha256Hex(data: Uint8Array | string): string {
+  return createHash("sha256").update(data).digest("hex");
+}
+
+async function sha256FileHex(path: string): Promise<string> {
+  const hash = createHash("sha256");
+  await new Promise<void>((resolve, reject) => {
+    const stream = createReadStream(path);
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("error", (err) => reject(err));
+    stream.on("end", () => resolve());
+  });
+  return hash.digest("hex");
+}
+
+function stripQuotes(value: string | null): string {
+  if (!value) return "";
+  return value.replace(/^\"|\"$/g, "");
+}
+
+export class R2ObjectStore implements ObjectStore {
+  private readonly client: Bun.S3Client;
+
+  constructor(cfg: R2Config) {
+    this.client = new Bun.S3Client({
+      bucket: cfg.bucket,
+      accessKeyId: cfg.accessKeyId,
+      secretAccessKey: cfg.secretAccessKey,
+      region: cfg.region ?? "auto",
+      endpoint: `https://${cfg.accountId}.r2.cloudflarestorage.com`,
+    });
+  }
+
+  private file(key: string): Bun.S3File {
+    return this.client.file(key);
+  }
+
+  private wrapError(op: string, key: string, err: unknown): never {
+    const message = String((err as any)?.message ?? err);
+    throw dsError(`R2 ${op} failed for ${key}: ${message}`);
+  }
+
+  async put(key: string, data: Uint8Array, opts: { contentType?: string; contentLength?: number } = {}): Promise<PutResult> {
+    try {
+      await this.file(key).write(data, { type: opts.contentType });
+      const stat = await this.file(key).stat();
+      return { etag: stripQuotes(stat.etag) || sha256Hex(data) };
+    } catch (err) {
+      this.wrapError("PUT", key, err);
+    }
+  }
+
+  async putFile(key: string, path: string, _size: number, opts: { contentType?: string } = {}): Promise<PutResult> {
+    try {
+      await this.file(key).write(Bun.file(path), { type: opts.contentType });
+      const stat = await this.file(key).stat();
+      return { etag: stripQuotes(stat.etag) || (await sha256FileHex(path)) };
+    } catch (err) {
+      this.wrapError("PUT", key, err);
+    }
+  }
+
+  async get(key: string, opts: GetOptions = {}): Promise<Uint8Array | null> {
+    try {
+      const file = this.file(key);
+      if (!(await file.exists())) return null;
+      const body =
+        opts.range == null
+          ? file
+          : file.slice(opts.range.start, opts.range.end == null ? undefined : opts.range.end + 1);
+      return new Uint8Array(await body.arrayBuffer());
+    } catch (err) {
+      this.wrapError("GET", key, err);
+    }
+  }
+
+  async head(key: string): Promise<{ etag: string; size: number } | null> {
+    try {
+      const file = this.file(key);
+      if (!(await file.exists())) return null;
+      const stat = await file.stat();
+      return { etag: stripQuotes(stat.etag), size: stat.size };
+    } catch (err) {
+      this.wrapError("HEAD", key, err);
+    }
+  }
+
+  async delete(key: string): Promise<void> {
+    try {
+      const file = this.file(key);
+      if (!(await file.exists())) return;
+      await file.delete();
+    } catch (err) {
+      this.wrapError("DELETE", key, err);
+    }
+  }
+
+  async list(prefix: string): Promise<string[]> {
+    try {
+      const keys: string[] = [];
+      let continuationToken: string | undefined;
+      for (;;) {
+        const res = await this.client.list({ prefix, continuationToken });
+        for (const entry of res.contents ?? []) {
+          keys.push(entry.key);
+        }
+        if (!res.isTruncated || !res.nextContinuationToken) break;
+        continuationToken = res.nextContinuationToken;
+      }
+      return keys;
+    } catch (err) {
+      this.wrapError("LIST", prefix, err);
+    }
+  }
+}

package/src/offset.ts

@@ -0,0 +1,70 @@
+import { decodeCrockfordBase32Fixed26Result, encodeCrockfordBase32Fixed26Result } from "./util/base32_crockford";
+import { readU32BE, writeU32BE } from "./util/endian";
+import { Result } from "better-result";
+import { dsError } from "./util/ds_error.ts";
+
+export type ParsedOffset =
+  | { kind: "start" }
+  | { kind: "seq"; epoch: number; seq: bigint; inBlock: number };
+
+const DEFAULT_EPOCH = 0;
+
+export function parseOffsetResult(input: string | null | undefined): Result<ParsedOffset, { kind: "invalid_offset"; message: string }> {
+  if (input == null || input === "") {
+    return Result.err({ kind: "invalid_offset", message: "missing offset" });
+  }
+  if (input === "-1") return Result.ok({ kind: "start" });
+
+  if (input.length !== 26) {
+    return Result.err({ kind: "invalid_offset", message: `invalid offset length: ${input.length}` });
+  }
+
+  const bytesRes = decodeCrockfordBase32Fixed26Result(input);
+  if (Result.isError(bytesRes)) return Result.err({ kind: "invalid_offset", message: bytesRes.error.message });
+  const bytes = bytesRes.value;
+  const epoch = readU32BE(bytes, 0);
+  const hi = readU32BE(bytes, 4);
+  const lo = readU32BE(bytes, 8);
+  const inBlock = readU32BE(bytes, 12);
+  // Protocol offsets are shifted by +1 so we never emit reserved "-1".
+  const rawSeq = (BigInt(hi) << 32n) | BigInt(lo);
+  const seq = rawSeq - 1n;
+  return Result.ok({ kind: "seq", epoch, seq, inBlock });
+}
+
+export function parseOffset(input: string | null | undefined): ParsedOffset {
+  const res = parseOffsetResult(input);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}
+
+export function encodeOffsetResult(epoch: number, seq: bigint, inBlock = 0): Result<string, { kind: "invalid_offset"; message: string }> {
+  if (seq < -1n) return Result.err({ kind: "invalid_offset", message: "invalid offset" });
+  const bytes = new Uint8Array(16);
+  writeU32BE(bytes, 0, epoch >>> 0);
+  const rawSeq = seq + 1n;
+  const hi = Number((rawSeq >> 32n) & 0xffffffffn);
+  const lo = Number(rawSeq & 0xffffffffn);
+  writeU32BE(bytes, 4, hi);
+  writeU32BE(bytes, 8, lo);
+  writeU32BE(bytes, 12, inBlock >>> 0);
+  const encodedRes = encodeCrockfordBase32Fixed26Result(bytes);
+  if (Result.isError(encodedRes)) return Result.err({ kind: "invalid_offset", message: encodedRes.error.message });
+  return Result.ok(encodedRes.value);
+}
+
+export function encodeOffset(epoch: number, seq: bigint, inBlock = 0): string {
+  const res = encodeOffsetResult(epoch, seq, inBlock);
+  if (Result.isError(res)) throw dsError(res.error.message);
+  return res.value;
+}
+
+export function canonicalizeOffset(input: string): string {
+  const p = parseOffset(input);
+  if (p.kind === "start") return encodeOffset(DEFAULT_EPOCH, -1n, 0);
+  return encodeOffset(p.epoch, p.seq, p.inBlock);
+}
+
+export function offsetToSeqOrNeg1(p: ParsedOffset): bigint {
+  return p.kind === "start" ? -1n : p.seq;
+}