@tungthedev/streams-server 0.2.0

package/src/objectstore/mock_r2.ts

@@ -0,0 +1,269 @@
+import { createHash } from "node:crypto";
+import { mkdirSync, readFileSync, writeFileSync, unlinkSync, openSync, closeSync, readSync, copyFileSync, createReadStream } from "node:fs";
+import { dirname, join } from "node:path";
+import type { GetOptions, ObjectStore, PutResult } from "./interface";
+import { dsError } from "../util/ds_error.ts";
+
+export type MockR2Faults = {
+  putDelayMs?: number;
+  getDelayMs?: number;
+  headDelayMs?: number;
+  listDelayMs?: number;
+  failPutPrefix?: string; // fail PUT when key starts with prefix
+  failPutEvery?: number; // fail every Nth PUT
+  failGetEvery?: number; // fail every Nth GET
+  failHeadEvery?: number;
+  failListEvery?: number;
+  timeoutPutEvery?: number;
+  timeoutGetEvery?: number;
+  timeoutHeadEvery?: number;
+  timeoutListEvery?: number;
+  partialGetEvery?: number; // return truncated bytes every Nth GET
+};
+
+export type MockR2Options = {
+  faults?: MockR2Faults;
+  maxInMemoryBytes?: number; // spill when object exceeds this
+  spillDir?: string; // optional directory for large objects
+};
+
+type StoredObject = {
+  etag: string;
+  size: number;
+  bytes?: Uint8Array;
+  path?: string;
+};
+
+function sleep(ms: number): Promise<void> {
+  if (ms <= 0) return Promise.resolve();
+  return new Promise((res) => setTimeout(res, ms));
+}
+
+export class MockR2Store implements ObjectStore {
+  private readonly data = new Map<string, StoredObject>();
+  private readonly faults: MockR2Faults;
+  private readonly maxInMemoryBytes: number;
+  private readonly spillDir?: string;
+
+  private putCount = 0;
+  private getCount = 0;
+  private headCount = 0;
+  private listCount = 0;
+  private memBytes = 0;
+
+  constructor(opts: MockR2Options | MockR2Faults = {}) {
+    if ("failPutEvery" in opts || "putDelayMs" in opts) {
+      this.faults = opts as MockR2Faults;
+      this.maxInMemoryBytes = Number.POSITIVE_INFINITY;
+    } else {
+      const o = opts as MockR2Options;
+      this.faults = o.faults ?? {};
+      this.maxInMemoryBytes = o.maxInMemoryBytes ?? Number.POSITIVE_INFINITY;
+      this.spillDir = o.spillDir;
+    }
+  }
+
+  private mkEtag(bytes: Uint8Array): string {
+    return createHash("sha256").update(bytes).digest("hex");
+  }
+
+  private async hashFile(path: string): Promise<string> {
+    const hash = createHash("sha256");
+    await new Promise<void>((resolve, reject) => {
+      const stream = createReadStream(path);
+      stream.on("data", (chunk) => hash.update(chunk));
+      stream.on("error", (err) => reject(err));
+      stream.on("end", () => resolve());
+    });
+    return hash.digest("hex");
+  }
+
+  private mkPath(key: string): string {
+    const hex = createHash("sha256").update(key).digest("hex");
+    return this.spillDir ? join(this.spillDir, `${hex}.bin`) : hex;
+  }
+
+  private shouldSpill(len: number): boolean {
+    return this.spillDir != null && len > this.maxInMemoryBytes;
+  }
+
+  private maybeFail(count: number, every?: number, msg?: string): void {
+    if (every && count % every === 0) throw dsError(msg ?? "MockR2: injected failure");
+  }
+
+  private maybeTimeout(count: number, every?: number, msg?: string): void {
+    if (every && count % every === 0) throw dsError(msg ?? "MockR2: injected timeout");
+  }
+
+  async put(key: string, bytes: Uint8Array): Promise<PutResult> {
+    this.putCount++;
+    if (this.faults.failPutPrefix && key.startsWith(this.faults.failPutPrefix)) {
+      throw dsError(`MockR2: injected PUT failure for ${key}`);
+    }
+    this.maybeFail(this.putCount, this.faults.failPutEvery, `MockR2: injected PUT failure for ${key}`);
+    this.maybeTimeout(this.putCount, this.faults.timeoutPutEvery, `MockR2: injected PUT timeout for ${key}`);
+    await sleep(this.faults.putDelayMs ?? 0);
+
+    const copy = new Uint8Array(bytes);
+    const etag = this.mkEtag(copy);
+    const size = copy.byteLength;
+
+    const existing = this.data.get(key);
+    if (existing?.bytes) this.memBytes -= existing.bytes.byteLength;
+    if (existing?.path) {
+      try { unlinkSync(existing.path); } catch { /* ignore */ }
+    }
+
+    if (this.shouldSpill(size)) {
+      const path = this.mkPath(key);
+      mkdirSync(dirname(path), { recursive: true });
+      writeFileSync(path, copy);
+      this.data.set(key, { etag, size, path });
+    } else {
+      this.memBytes += size;
+      this.data.set(key, { etag, size, bytes: copy });
+    }
+
+    return { etag };
+  }
+
+  async putFile(key: string, path: string, size: number): Promise<PutResult> {
+    this.putCount++;
+    if (this.faults.failPutPrefix && key.startsWith(this.faults.failPutPrefix)) {
+      throw dsError(`MockR2: injected PUT failure for ${key}`);
+    }
+    this.maybeFail(this.putCount, this.faults.failPutEvery, `MockR2: injected PUT failure for ${key}`);
+    this.maybeTimeout(this.putCount, this.faults.timeoutPutEvery, `MockR2: injected PUT timeout for ${key}`);
+    await sleep(this.faults.putDelayMs ?? 0);
+
+    const etag = await this.hashFile(path);
+
+    const existing = this.data.get(key);
+    if (existing?.bytes) this.memBytes -= existing.bytes.byteLength;
+    if (existing?.path) {
+      try { unlinkSync(existing.path); } catch { /* ignore */ }
+    }
+
+    if (this.shouldSpill(size)) {
+      const dest = this.mkPath(key);
+      mkdirSync(dirname(dest), { recursive: true });
+      copyFileSync(path, dest);
+      this.data.set(key, { etag, size, path: dest });
+    } else {
+      const bytes = new Uint8Array(await Bun.file(path).arrayBuffer());
+      this.memBytes += bytes.byteLength;
+      this.data.set(key, { etag, size, bytes });
+    }
+
+    return { etag };
+  }
+
+  async get(key: string, opts: GetOptions = {}): Promise<Uint8Array | null> {
+    this.getCount++;
+    this.maybeFail(this.getCount, this.faults.failGetEvery, `MockR2: injected GET failure for ${key}`);
+    this.maybeTimeout(this.getCount, this.faults.timeoutGetEvery, `MockR2: injected GET timeout for ${key}`);
+    await sleep(this.faults.getDelayMs ?? 0);
+
+    const entry = this.data.get(key);
+    if (!entry) return null;
+
+    const range = opts.range;
+    const total = entry.size;
+    const start = range?.start ?? 0;
+    const end = range?.end ?? total - 1;
+    if (start >= total) return new Uint8Array(0);
+    const clampEnd = Math.min(end, total - 1);
+    const length = Math.max(0, clampEnd - start + 1);
+
+    let out: Uint8Array;
+    if (entry.bytes) {
+      out = entry.bytes.slice(start, start + length);
+    } else if (entry.path) {
+      if (length === total) {
+        out = readFileSync(entry.path);
+      } else {
+        const fd = openSync(entry.path, "r");
+        try {
+          const buf = new Uint8Array(length);
+          readSync(fd, buf, 0, length, start);
+          out = buf;
+        } finally {
+          closeSync(fd);
+        }
+      }
+    } else {
+      return null;
+    }
+
+    if (this.faults.partialGetEvery && this.getCount % this.faults.partialGetEvery === 0) {
+      const half = Math.max(0, Math.floor(out.byteLength / 2));
+      return out.slice(0, half);
+    }
+
+    return out;
+  }
+
+  async head(key: string): Promise<{ etag: string; size: number } | null> {
+    this.headCount++;
+    this.maybeFail(this.headCount, this.faults.failHeadEvery, `MockR2: injected HEAD failure for ${key}`);
+    this.maybeTimeout(this.headCount, this.faults.timeoutHeadEvery, `MockR2: injected HEAD timeout for ${key}`);
+    await sleep(this.faults.headDelayMs ?? 0);
+
+    const v = this.data.get(key);
+    if (!v) return null;
+    return { etag: v.etag, size: v.size };
+  }
+
+  async delete(key: string): Promise<void> {
+    const v = this.data.get(key);
+    if (v?.bytes) this.memBytes -= v.bytes.byteLength;
+    if (v?.path) {
+      try { unlinkSync(v.path); } catch { /* ignore */ }
+    }
+    this.data.delete(key);
+  }
+
+  async list(prefix: string): Promise<string[]> {
+    this.listCount++;
+    this.maybeFail(this.listCount, this.faults.failListEvery, "MockR2: injected LIST failure");
+    this.maybeTimeout(this.listCount, this.faults.timeoutListEvery, "MockR2: injected LIST timeout");
+    await sleep(this.faults.listDelayMs ?? 0);
+
+    const out: string[] = [];
+    for (const k of this.data.keys()) {
+      if (k.startsWith(prefix)) out.push(k);
+    }
+    out.sort();
+    return out;
+  }
+
+  // Helpers for tests
+  has(key: string): boolean {
+    return this.data.has(key);
+  }
+
+  size(): number {
+    return this.data.size;
+  }
+
+  memoryBytes(): number {
+    return this.memBytes;
+  }
+
+  stats(): { puts: number; gets: number; heads: number; lists: number; memoryBytes: number } {
+    return {
+      puts: this.putCount,
+      gets: this.getCount,
+      heads: this.headCount,
+      lists: this.listCount,
+      memoryBytes: this.memBytes,
+    };
+  }
+
+  resetStats(): void {
+    this.putCount = 0;
+    this.getCount = 0;
+    this.headCount = 0;
+    this.listCount = 0;
+  }
+}

package/src/objectstore/null.ts

@@ -0,0 +1,32 @@
+import type { GetOptions, ObjectStore, PutResult } from "./interface";
+import { dsError } from "../util/ds_error.ts";
+
+function disabled(op: string, key?: string): never {
+  throw dsError(`object store disabled in local mode (${op}${key ? `: ${key}` : ""})`);
+}
+
+export class NullObjectStore implements ObjectStore {
+  async put(key: string, _data: Uint8Array, _opts?: { contentType?: string; contentLength?: number }): Promise<PutResult> {
+    return disabled("put", key);
+  }
+
+  async putFile(key: string, _path: string, _size: number, _opts?: { contentType?: string }): Promise<PutResult> {
+    return disabled("putFile", key);
+  }
+
+  async get(key: string, _opts?: GetOptions): Promise<Uint8Array | null> {
+    return disabled("get", key);
+  }
+
+  async head(key: string): Promise<{ etag: string; size: number } | null> {
+    return disabled("head", key);
+  }
+
+  async delete(key: string): Promise<void> {
+    return disabled("delete", key);
+  }
+
+  async list(prefix: string): Promise<string[]> {
+    return disabled("list", prefix);
+  }
+}

package/src/objectstore/r2.ts

@@ -0,0 +1,318 @@
+import { createHash, createHmac } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { Readable } from "node:stream";
+import type { GetOptions, ObjectStore, PutResult } from "./interface";
+import { dsError } from "../util/ds_error.ts";
+
+export type R2Config = {
+  accountId: string;
+  bucket: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+  region?: string;
+  endpoint?: string;
+};
+
+const EMPTY_SHA256 = sha256Hex(new Uint8Array(0));
+const XML_DECODER = new TextDecoder();
+
+function sha256Hex(data: Uint8Array | string): string {
+  return createHash("sha256").update(data).digest("hex");
+}
+
+async function readResponseBytes(res: Response): Promise<Uint8Array> {
+  if (!res.body) return new Uint8Array(0);
+  const reader = res.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  try {
+    for (;;) {
+      const next = await reader.read();
+      if (next.done) break;
+      const chunk = next.value;
+      chunks.push(chunk);
+      total += chunk.byteLength;
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  if (chunks.length === 1) return chunks[0]!;
+  const out = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return out;
+}
+
+async function readResponseText(res: Response): Promise<string> {
+  return XML_DECODER.decode(await readResponseBytes(res));
+}
+
+function fileStreamBody(path: string): BodyInit {
+  return Readable.toWeb(createReadStream(path)) as unknown as BodyInit;
+}
+
+async function sha256FileHex(path: string): Promise<string> {
+  const hash = createHash("sha256");
+  await new Promise<void>((resolve, reject) => {
+    const stream = createReadStream(path);
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("error", (err) => reject(err));
+    stream.on("end", () => resolve());
+  });
+  return hash.digest("hex");
+}
+
+function stripQuotes(value: string | null): string {
+  if (!value) return "";
+  return value.replace(/^\"|\"$/g, "");
+}
+
+function encodePathPart(part: string): string {
+  return encodeURIComponent(part).replace(/[!'()*]/g, (ch) => `%${ch.charCodeAt(0).toString(16).toUpperCase()}`);
+}
+
+function encodeKeyPath(key: string): string {
+  return key.split("/").map(encodePathPart).join("/");
+}
+
+function encodeQueryPart(part: string): string {
+  return encodePathPart(part).replace(/%7E/g, "~");
+}
+
+function hmac(key: string | Buffer, data: string): Buffer {
+  return createHmac("sha256", key).update(data).digest();
+}
+
+function xmlDecode(value: string): string {
+  return value
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, "\"")
+    .replace(/&apos;/g, "'")
+    .replace(/&amp;/g, "&");
+}
+
+function firstXmlTag(xml: string, tag: string): string | null {
+  const match = xml.match(new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`));
+  return match ? xmlDecode(match[1] ?? "") : null;
+}
+
+function allXmlTags(xml: string, tag: string): string[] {
+  const out: string[] = [];
+  const re = new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`, "g");
+  for (;;) {
+    const match = re.exec(xml);
+    if (!match) break;
+    out.push(xmlDecode(match[1] ?? ""));
+  }
+  return out;
+}
+
+export class R2ObjectStore implements ObjectStore {
+  private readonly endpoint: URL;
+  private readonly bucket: string;
+  private readonly accessKeyId: string;
+  private readonly secretAccessKey: string;
+  private readonly region: string;
+
+  constructor(cfg: R2Config) {
+    this.endpoint = new URL(cfg.endpoint ?? `https://${cfg.accountId}.r2.cloudflarestorage.com`);
+    this.bucket = cfg.bucket;
+    this.accessKeyId = cfg.accessKeyId;
+    this.secretAccessKey = cfg.secretAccessKey;
+    this.region = cfg.region ?? "auto";
+  }
+
+  private objectPath(key: string): string {
+    return `/${encodePathPart(this.bucket)}/${encodeKeyPath(key)}`;
+  }
+
+  private requestUrl(path: string, query: Array<[string, string]> = []): URL {
+    const url = new URL(this.endpoint.href);
+    const basePath = url.pathname.replace(/\/+$/, "");
+    url.pathname = `${basePath}${path}`;
+    url.search = "";
+    for (const [key, value] of query) url.searchParams.append(key, value);
+    return url;
+  }
+
+  private authorization(method: string, url: URL, headers: Headers, payloadHash: string, amzDate: string): string {
+    const date = amzDate.slice(0, 8);
+    const host = url.host;
+    headers.set("host", host);
+    headers.set("x-amz-content-sha256", payloadHash);
+    headers.set("x-amz-date", amzDate);
+
+    const signedHeaderNames = [...headers.keys()].map((h) => h.toLowerCase()).sort();
+    const canonicalHeaders = signedHeaderNames
+      .map((name) => `${name}:${headers.get(name)?.trim().replace(/\s+/g, " ") ?? ""}\n`)
+      .join("");
+    const signedHeaders = signedHeaderNames.join(";");
+    const queryEntries = [...url.searchParams.entries()].sort(([ak, av], [bk, bv]) => {
+      if (ak === bk) return av < bv ? -1 : av > bv ? 1 : 0;
+      return ak < bk ? -1 : 1;
+    });
+    const canonicalQuery = queryEntries
+      .map(([key, value]) => `${encodeQueryPart(key)}=${encodeQueryPart(value)}`)
+      .join("&");
+    const canonicalRequest = [
+      method,
+      url.pathname,
+      canonicalQuery,
+      canonicalHeaders,
+      signedHeaders,
+      payloadHash,
+    ].join("\n");
+    const scope = `${date}/${this.region}/s3/aws4_request`;
+    const stringToSign = [
+      "AWS4-HMAC-SHA256",
+      amzDate,
+      scope,
+      sha256Hex(canonicalRequest),
+    ].join("\n");
+    const kDate = hmac(`AWS4${this.secretAccessKey}`, date);
+    const kRegion = hmac(kDate, this.region);
+    const kService = hmac(kRegion, "s3");
+    const kSigning = hmac(kService, "aws4_request");
+    const signature = createHmac("sha256", kSigning).update(stringToSign).digest("hex");
+    return `AWS4-HMAC-SHA256 Credential=${this.accessKeyId}/${scope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
+  }
+
+  private async request(
+    method: string,
+    path: string,
+    opts: {
+      query?: Array<[string, string]>;
+      headers?: HeadersInit;
+      body?: BodyInit;
+      payloadHash?: string;
+    } = {}
+  ): Promise<Response> {
+    const url = this.requestUrl(path, opts.query ?? []);
+    const headers = new Headers(opts.headers);
+    const now = new Date();
+    const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, "");
+    const payloadHash = opts.payloadHash ?? EMPTY_SHA256;
+    headers.set("authorization", this.authorization(method, url, headers, payloadHash, amzDate));
+    return fetch(url, {
+      method,
+      headers,
+      body: opts.body,
+    });
+  }
+
+  private wrapStatus(op: string, key: string, res: Response): never {
+    throw dsError(`R2 ${op} failed for ${key}: HTTP ${res.status} ${res.statusText}`);
+  }
+
+  private wrapError(op: string, key: string, err: unknown): never {
+    const message = String((err as any)?.message ?? err);
+    throw dsError(`R2 ${op} failed for ${key}: ${message}`);
+  }
+
+  async put(key: string, data: Uint8Array, opts: { contentType?: string; contentLength?: number } = {}): Promise<PutResult> {
+    const payloadHash = sha256Hex(data);
+    try {
+      const headers: Record<string, string> = {
+        "content-length": String(opts.contentLength ?? data.byteLength),
+      };
+      if (opts.contentType) headers["content-type"] = opts.contentType;
+      const res = await this.request("PUT", this.objectPath(key), {
+        headers,
+        body: data as unknown as BodyInit,
+        payloadHash,
+      });
+      if (!res.ok) this.wrapStatus("PUT", key, res);
+      return { etag: payloadHash };
+    } catch (err) {
+      this.wrapError("PUT", key, err);
+    }
+  }
+
+  async putFile(key: string, path: string, size: number, opts: { contentType?: string } = {}): Promise<PutResult> {
+    const payloadHash = await sha256FileHex(path);
+    try {
+      const headers: Record<string, string> = {
+        "content-length": String(size),
+      };
+      if (opts.contentType) headers["content-type"] = opts.contentType;
+      const res = await this.request("PUT", this.objectPath(key), {
+        headers,
+        body: fileStreamBody(path),
+        payloadHash,
+      });
+      if (!res.ok) this.wrapStatus("PUT", key, res);
+      return { etag: payloadHash };
+    } catch (err) {
+      this.wrapError("PUT", key, err);
+    }
+  }
+
+  async get(key: string, opts: GetOptions = {}): Promise<Uint8Array | null> {
+    try {
+      const headers: Record<string, string> = {};
+      if (opts.range) {
+        const end = opts.range.end == null ? "" : String(opts.range.end);
+        headers.range = `bytes=${opts.range.start}-${end}`;
+      }
+      const res = await this.request("GET", this.objectPath(key), { headers });
+      if (res.status === 404) return null;
+      if (!res.ok && res.status !== 206) this.wrapStatus("GET", key, res);
+      return readResponseBytes(res);
+    } catch (err) {
+      this.wrapError("GET", key, err);
+    }
+  }
+
+  async head(key: string): Promise<{ etag: string; size: number } | null> {
+    try {
+      const res = await this.request("HEAD", this.objectPath(key));
+      if (res.status === 404) return null;
+      if (!res.ok) this.wrapStatus("HEAD", key, res);
+      return {
+        etag: stripQuotes(res.headers.get("etag")),
+        size: Number(res.headers.get("content-length") ?? "0"),
+      };
+    } catch (err) {
+      this.wrapError("HEAD", key, err);
+    }
+  }
+
+  async delete(key: string): Promise<void> {
+    try {
+      const res = await this.request("DELETE", this.objectPath(key));
+      if (res.status === 404) return;
+      if (!res.ok && res.status !== 204) this.wrapStatus("DELETE", key, res);
+    } catch (err) {
+      this.wrapError("DELETE", key, err);
+    }
+  }
+
+  async list(prefix: string): Promise<string[]> {
+    try {
+      const keys: string[] = [];
+      let continuationToken: string | null = null;
+      for (;;) {
+        const query: Array<[string, string]> = [
+          ["list-type", "2"],
+          ["prefix", prefix],
+        ];
+        if (continuationToken) query.push(["continuation-token", continuationToken]);
+        const res = await this.request("GET", `/${encodePathPart(this.bucket)}`, { query });
+        if (!res.ok) this.wrapStatus("LIST", prefix, res);
+        const xml = await readResponseText(res);
+        keys.push(...allXmlTags(xml, "Key"));
+        const truncated = firstXmlTag(xml, "IsTruncated") === "true";
+        if (!truncated) break;
+        continuationToken = firstXmlTag(xml, "NextContinuationToken");
+        if (!continuationToken) break;
+      }
+      return keys;
+    } catch (err) {
+      this.wrapError("LIST", prefix, err);
+    }
+  }
+}

package/src/observe/pairing.ts

@@ -0,0 +1,61 @@
+import type { StreamProfileSpec } from "../profiles";
+
+export type RequestObservabilityPairingDescriptor = {
+  request: {
+    events_stream: string;
+    traces_stream: string;
+  };
+};
+
+function readRequestPairing(
+  profile: StreamProfileSpec
+): Record<string, unknown> | null {
+  const observability = profile.observability;
+  if (
+    !observability ||
+    typeof observability !== "object" ||
+    Array.isArray(observability)
+  ) {
+    return null;
+  }
+  const request = (observability as Record<string, unknown>).request;
+  return request && typeof request === "object" && !Array.isArray(request)
+    ? (request as Record<string, unknown>)
+    : null;
+}
+
+function nonEmptyString(value: unknown): string | null {
+  return typeof value === "string" && value.trim() !== "" ? value.trim() : null;
+}
+
+export function buildRequestObservabilityPairingDescriptor(
+  stream: string,
+  profile: StreamProfileSpec
+): RequestObservabilityPairingDescriptor | null {
+  const request = readRequestPairing(profile);
+  if (!request) return null;
+
+  if (profile.kind === "evlog") {
+    const tracesStream = nonEmptyString(request.tracesStream);
+    if (!tracesStream) return null;
+    return {
+      request: {
+        events_stream: stream,
+        traces_stream: tracesStream,
+      },
+    };
+  }
+
+  if (profile.kind === "otel-traces") {
+    const eventsStream = nonEmptyString(request.eventsStream);
+    if (!eventsStream) return null;
+    return {
+      request: {
+        events_stream: eventsStream,
+        traces_stream: stream,
+      },
+    };
+  }
+
+  return null;
+}