@qlever-llc/trellis 0.8.3 → 0.9.0-rc.10

package/src/store.ts

@@ -0,0 +1,658 @@
+import {
+  type ObjectInfo,
+  type ObjectResult,
+  type ObjectStore,
+  type ObjectStoreStatus,
+  Objm,
+} from "@nats-io/obj";
+import type { NatsConnection } from "@nats-io/nats-core/internal";
+import {
+  AsyncResult,
+  Result,
+  type Result as ResultType,
+} from "@qlever-llc/result";
+import { StoreError } from "./errors/index.js";
+import type { PageResponse } from "./models/trellis/Page.js";
+
+const INTERNAL_CONTENT_TYPE_METADATA_KEY = "__trellis_content_type";
+const DEFAULT_STORE_WAIT_POLL_INTERVAL_MS = 250;
+const MAX_STORE_LIST_LIMIT = 500;
+
+export type StoreBody =
+  | Uint8Array
+  | ReadableStream<Uint8Array>
+  | AsyncIterable<Uint8Array>;
+
+export type StoreWaitOptions = {
+  timeoutMs?: number;
+  pollIntervalMs?: number;
+  signal?: AbortSignal;
+};
+
+export type StoreOpenOptions = {
+  ttlMs?: number;
+  maxObjectBytes?: number;
+  maxTotalBytes?: number;
+  bindOnly?: boolean;
+};
+
+export type StorePutOptions = {
+  contentType?: string;
+  metadata?: Record<string, string>;
+};
+
+/** Explicit bounded query for listing object metadata in a typed store. */
+export type StoreListOptions = {
+  prefix?: string;
+  offset?: number;
+  limit: number;
+};
+
+export type StoreInfo = {
+  key: string;
+  size: number;
+  updatedAt: string;
+  digest?: string;
+  contentType?: string;
+  metadata: Record<string, string>;
+};
+
+export type StoreStatus = {
+  size: number;
+  sealed: boolean;
+  ttlMs: number;
+  maxObjectBytes?: number;
+  maxTotalBytes?: number;
+};
+
+function metadataWithContentType(
+  options?: StorePutOptions,
+): Record<string, string> | undefined {
+  if (!options?.metadata && !options?.contentType) {
+    return undefined;
+  }
+
+  return {
+    ...(options?.metadata ?? {}),
+    ...(options?.contentType
+      ? { [INTERNAL_CONTENT_TYPE_METADATA_KEY]: options.contentType }
+      : {}),
+  };
+}
+
+function storeInfoFromObjectInfo(info: ObjectInfo): StoreInfo {
+  const { [INTERNAL_CONTENT_TYPE_METADATA_KEY]: contentType, ...metadata } =
+    info.metadata ?? {};
+  return {
+    key: info.name,
+    size: info.size,
+    updatedAt: info.mtime,
+    ...(info.digest ? { digest: info.digest } : {}),
+    ...(contentType ? { contentType } : {}),
+    metadata,
+  };
+}
+
+function streamFromBytes(data: Uint8Array): ReadableStream<Uint8Array> {
+  return new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.enqueue(data);
+      controller.close();
+    },
+  });
+}
+
+function streamFromAsyncIterable(
+  iterable: AsyncIterable<Uint8Array>,
+): ReadableStream<Uint8Array> {
+  const iterator = iterable[Symbol.asyncIterator]();
+  return new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      const next = await iterator.next();
+      if (next.done) {
+        controller.close();
+        return;
+      }
+      controller.enqueue(next.value);
+    },
+    async cancel(reason) {
+      await iterator.return?.(reason);
+    },
+  });
+}
+
+function validateStoreListOptions(
+  opts: StoreListOptions,
+): ResultType<Required<StoreListOptions>, StoreError> {
+  if (!Number.isInteger(opts.limit) || opts.limit < 0) {
+    return Result.err(
+      new StoreError({
+        operation: "list",
+        context: { reason: "invalid_limit", limit: opts.limit },
+      }),
+    );
+  }
+  if (opts.limit > MAX_STORE_LIST_LIMIT) {
+    return Result.err(
+      new StoreError({
+        operation: "list",
+        context: {
+          reason: "limit_exceeded",
+          limit: opts.limit,
+          maxLimit: MAX_STORE_LIST_LIMIT,
+        },
+      }),
+    );
+  }
+
+  const offset = opts.offset ?? 0;
+  if (!Number.isInteger(offset) || offset < 0) {
+    return Result.err(
+      new StoreError({
+        operation: "list",
+        context: { reason: "invalid_offset", offset },
+      }),
+    );
+  }
+
+  return Result.ok({ prefix: opts.prefix ?? "", offset, limit: opts.limit });
+}
+
+function enforceMaxObjectBytes(
+  stream: ReadableStream<Uint8Array>,
+  maxObjectBytes?: number,
+): ReadableStream<Uint8Array> {
+  if (maxObjectBytes === undefined) {
+    return stream;
+  }
+
+  const reader = stream.getReader();
+  let totalBytes = 0;
+
+  return new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      const next = await reader.read();
+      if (next.done) {
+        controller.close();
+        return;
+      }
+
+      totalBytes += next.value.length;
+      if (totalBytes > maxObjectBytes) {
+        controller.error(
+          new StoreError({
+            operation: "put",
+            context: {
+              reason: "max_object_bytes_exceeded",
+              maxObjectBytes,
+              attemptedBytes: totalBytes,
+            },
+          }),
+        );
+        await reader.cancel();
+        return;
+      }
+
+      controller.enqueue(next.value);
+    },
+    async cancel(reason) {
+      await reader.cancel(reason);
+    },
+  });
+}
+
+async function bytesFromStream(
+  stream: ReadableStream<Uint8Array>,
+): Promise<Uint8Array> {
+  const reader = stream.getReader();
+  const chunks: Uint8Array[] = [];
+  let totalLength = 0;
+
+  while (true) {
+    const next = await reader.read();
+    if (next.done) {
+      break;
+    }
+    chunks.push(next.value);
+    totalLength += next.value.length;
+  }
+
+  const merged = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    merged.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return merged;
+}
+
+function isNotFoundStoreError(error: StoreError): boolean {
+  return error.getContext().reason === "not_found";
+}
+
+function abortedStoreError(key: string, cause: unknown): StoreError {
+  return new StoreError({
+    operation: "waitFor",
+    cause,
+    context: { key, reason: "aborted" },
+  });
+}
+
+async function sleepWithSignal(
+  ms: number,
+  signal?: AbortSignal,
+): Promise<void> {
+  if (signal?.aborted) {
+    throw signal.reason ??
+      new DOMException("The operation was aborted", "AbortError");
+  }
+
+  await new Promise<void>((resolve, reject) => {
+    const timeoutId = setTimeout(() => {
+      cleanup();
+      resolve();
+    }, ms);
+
+    const onAbort = () => {
+      cleanup();
+      reject(
+        signal?.reason ??
+          new DOMException("The operation was aborted", "AbortError"),
+      );
+    };
+
+    const cleanup = () => {
+      clearTimeout(timeoutId);
+      signal?.removeEventListener("abort", onAbort);
+    };
+
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+function streamFromBody(
+  body: Exclude<StoreBody, Uint8Array>,
+): ReadableStream<Uint8Array> {
+  return body instanceof ReadableStream ? body : streamFromAsyncIterable(body);
+}
+
+async function unwrapObjectInfo(
+  store: ObjectStore,
+  key: string,
+): Promise<ResultType<ObjectInfo, StoreError>> {
+  try {
+    const info = await store.info(key);
+    if (info === null || info.deleted) {
+      return Result.err(
+        new StoreError({
+          operation: "get",
+          context: { key, reason: "not_found" },
+        }),
+      );
+    }
+    return Result.ok(info);
+  } catch (cause) {
+    return Result.err(
+      new StoreError({ operation: "get", cause, context: { key } }),
+    );
+  }
+}
+
+export class TypedStore {
+  readonly #store: ObjectStore;
+  readonly #options:
+    & Required<Pick<StoreOpenOptions, "ttlMs">>
+    & Omit<StoreOpenOptions, "ttlMs">;
+
+  private constructor(store: ObjectStore, options: StoreOpenOptions) {
+    this.#store = store;
+    this.#options = {
+      ttlMs: options.ttlMs ?? 0,
+      ...(options.maxObjectBytes !== undefined
+        ? { maxObjectBytes: options.maxObjectBytes }
+        : {}),
+      ...(options.maxTotalBytes !== undefined
+        ? { maxTotalBytes: options.maxTotalBytes }
+        : {}),
+      ...(options.bindOnly !== undefined ? { bindOnly: options.bindOnly } : {}),
+    };
+  }
+
+  static open(
+    nats: NatsConnection,
+    name: string,
+    options: StoreOpenOptions = {},
+  ): AsyncResult<TypedStore, StoreError> {
+    return AsyncResult.from((async () => {
+      try {
+        const objm = new Objm(nats);
+        const store = options.bindOnly
+          ? await objm.open(name)
+          : await objm.create(name, {
+            ...(options.ttlMs && options.ttlMs > 0
+              ? { ttl: options.ttlMs * 1_000_000 }
+              : {}),
+            ...(options.maxTotalBytes !== undefined
+              ? { max_bytes: options.maxTotalBytes }
+              : {}),
+          });
+        return Result.ok(new TypedStore(store, options));
+      } catch (cause) {
+        return Result.err(
+          new StoreError({ operation: "open", cause, context: { name } }),
+        );
+      }
+    })());
+  }
+
+  create(
+    key: string,
+    body: StoreBody,
+    options?: StorePutOptions,
+  ): AsyncResult<void, StoreError> {
+    return AsyncResult.from((async () => {
+      const existing = await unwrapObjectInfo(this.#store, key);
+      if (existing.isOk()) {
+        return Result.err(
+          new StoreError({
+            operation: "create",
+            context: { key, reason: "already_exists" },
+          }),
+        );
+      }
+
+      return await this.#putInternal("create", key, body, options);
+    })());
+  }
+
+  put(
+    key: string,
+    body: StoreBody,
+    options?: StorePutOptions,
+  ): AsyncResult<void, StoreError> {
+    return AsyncResult.from(this.#putInternal("put", key, body, options));
+  }
+
+  get(key: string): AsyncResult<TypedStoreEntry, StoreError> {
+    return AsyncResult.from((async () => {
+      const info = await unwrapObjectInfo(this.#store, key);
+      return info.map((objectInfo) =>
+        new TypedStoreEntry(this.#store, storeInfoFromObjectInfo(objectInfo))
+      );
+    })());
+  }
+
+  /**
+   * Waits for an object key to appear in the store and returns the resulting entry.
+   */
+  waitFor(
+    key: string,
+    options: StoreWaitOptions = {},
+  ): AsyncResult<TypedStoreEntry, StoreError> {
+    return AsyncResult.from((async () => {
+      const startedAt = Date.now();
+      const pollIntervalMs = options.pollIntervalMs ??
+        DEFAULT_STORE_WAIT_POLL_INTERVAL_MS;
+
+      while (true) {
+        if (options.signal?.aborted) {
+          return Result.err(abortedStoreError(key, options.signal.reason));
+        }
+
+        const entry = await this.get(key);
+        if (entry.isOk()) {
+          return entry;
+        }
+        if (!isNotFoundStoreError(entry.error)) {
+          return entry;
+        }
+
+        const remainingTimeoutMs = options.timeoutMs === undefined
+          ? undefined
+          : options.timeoutMs - (Date.now() - startedAt);
+        if (remainingTimeoutMs !== undefined && remainingTimeoutMs <= 0) {
+          return Result.err(
+            new StoreError({
+              operation: "waitFor",
+              context: { key, reason: "timeout", timeoutMs: options.timeoutMs },
+            }),
+          );
+        }
+
+        try {
+          await sleepWithSignal(
+            remainingTimeoutMs === undefined
+              ? pollIntervalMs
+              : Math.min(pollIntervalMs, remainingTimeoutMs),
+            options.signal,
+          );
+        } catch (cause) {
+          return Result.err(abortedStoreError(key, cause));
+        }
+      }
+    })());
+  }
+
+  delete(key: string): AsyncResult<void, StoreError> {
+    return AsyncResult.from((async () => {
+      try {
+        await this.#store.delete(key);
+        return Result.ok(undefined);
+      } catch (cause) {
+        return Result.err(
+          new StoreError({ operation: "delete", cause, context: { key } }),
+        );
+      }
+    })());
+  }
+
+  list(
+    opts: StoreListOptions,
+  ): AsyncResult<PageResponse<StoreInfo>, StoreError> {
+    return AsyncResult.from((async () => {
+      const query = validateStoreListOptions(opts);
+      if (query.isErr()) return Result.err(query.error);
+
+      const { prefix, offset, limit } = query.unwrapOrElse(() => {
+        throw new Error("unreachable");
+      });
+      try {
+        const objects = await this.#store.list();
+        const filtered = objects
+          .filter((info) => !info.deleted && info.name.startsWith(prefix))
+          .map(storeInfoFromObjectInfo)
+          .sort((left, right) => left.key.localeCompare(right.key));
+        const entries = filtered.slice(offset, offset + limit);
+
+        return Result.ok({
+          entries,
+          count: filtered.length,
+          offset,
+          limit,
+          nextOffset: limit <= 0 || offset + limit >= filtered.length
+            ? undefined
+            : offset + limit,
+        });
+      } catch (cause) {
+        return Result.err(
+          new StoreError({ operation: "list", cause, context: { prefix } }),
+        );
+      }
+    })());
+  }
+
+  status(): AsyncResult<StoreStatus, StoreError> {
+    return AsyncResult.from((async () => {
+      try {
+        const status = await this.#store.status();
+        return Result.ok(
+          storeStatusFromObjectStoreStatus(status, this.#options),
+        );
+      } catch (cause) {
+        return Result.err(new StoreError({ operation: "status", cause }));
+      }
+    })());
+  }
+
+  async #putInternal(
+    operation: "create" | "put",
+    key: string,
+    body: StoreBody,
+    options?: StorePutOptions,
+  ): Promise<ResultType<void, StoreError>> {
+    try {
+      const metadata = metadataWithContentType(options);
+      if (body instanceof Uint8Array) {
+        if (
+          this.#options.maxObjectBytes !== undefined &&
+          body.length > this.#options.maxObjectBytes
+        ) {
+          return Result.err(
+            new StoreError({
+              operation,
+              context: {
+                key,
+                reason: "max_object_bytes_exceeded",
+                maxObjectBytes: this.#options.maxObjectBytes,
+                attemptedBytes: body.length,
+              },
+            }),
+          );
+        }
+
+        await this.#store.putBlob({
+          name: key,
+          ...(metadata ? { metadata } : {}),
+        }, body);
+        return Result.ok(undefined);
+      }
+
+      const limitedStream = enforceMaxObjectBytes(
+        streamFromBody(body),
+        this.#options.maxObjectBytes,
+      );
+
+      await this.#store.put(
+        { name: key, ...(metadata ? { metadata } : {}) },
+        limitedStream,
+      );
+      return Result.ok(undefined);
+    } catch (cause) {
+      return Result.err(new StoreError({ operation, cause, context: { key } }));
+    }
+  }
+}
+
+function storeStatusFromObjectStoreStatus(
+  status: ObjectStoreStatus,
+  options:
+    & Required<Pick<StoreOpenOptions, "ttlMs">>
+    & Omit<StoreOpenOptions, "ttlMs">,
+): StoreStatus {
+  return {
+    size: status.size,
+    sealed: status.sealed,
+    ttlMs: status.ttl > 0 ? Math.floor(status.ttl / 1_000_000) : options.ttlMs,
+    ...(options.maxObjectBytes !== undefined
+      ? { maxObjectBytes: options.maxObjectBytes }
+      : {}),
+    ...(options.maxTotalBytes !== undefined
+      ? { maxTotalBytes: options.maxTotalBytes }
+      : {}),
+  };
+}
+
+export class TypedStoreEntry {
+  readonly key: string;
+  readonly info: StoreInfo;
+  readonly #store: ObjectStore;
+
+  constructor(store: ObjectStore, info: StoreInfo) {
+    this.#store = store;
+    this.key = info.key;
+    this.info = info;
+  }
+
+  stream(): AsyncResult<ReadableStream<Uint8Array>, StoreError> {
+    return AsyncResult.from((async () => {
+      try {
+        const result = await this.#store.get(this.key);
+        if (result === null) {
+          return Result.err(
+            new StoreError({
+              operation: "stream",
+              context: { key: this.key, reason: "not_found" },
+            }),
+          );
+        }
+
+        return Result.ok(streamWithErrorCheck(result));
+      } catch (cause) {
+        return Result.err(
+          new StoreError({
+            operation: "stream",
+            cause,
+            context: { key: this.key },
+          }),
+        );
+      }
+    })());
+  }
+
+  bytes(): AsyncResult<Uint8Array, StoreError> {
+    return AsyncResult.from((async () => {
+      try {
+        const bytes = await this.#store.getBlob(this.key);
+        if (bytes === null) {
+          return Result.err(
+            new StoreError({
+              operation: "bytes",
+              context: { key: this.key, reason: "not_found" },
+            }),
+          );
+        }
+        return Result.ok(bytes);
+      } catch (cause) {
+        return Result.err(
+          new StoreError({
+            operation: "bytes",
+            cause,
+            context: { key: this.key },
+          }),
+        );
+      }
+    })());
+  }
+}
+
+function streamWithErrorCheck(
+  result: ObjectResult,
+): ReadableStream<Uint8Array> {
+  const reader = result.data.getReader();
+
+  return new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      const next = await reader.read();
+      if (next.done) {
+        const error = await result.error;
+        if (error) {
+          controller.error(error);
+          return;
+        }
+        controller.close();
+        return;
+      }
+
+      controller.enqueue(next.value);
+    },
+    async cancel(reason) {
+      await reader.cancel(reason);
+    },
+  });
+}
+
+export async function bytesFromStoreStream(
+  stream: ReadableStream<Uint8Array>,
+): Promise<Uint8Array> {
+  return bytesFromStream(stream);
+}

package/src/tasks.ts

@@ -0,0 +1,34 @@
+import { AsyncResult, type BaseError, Result } from "@qlever-llc/result";
+import { logger, type LoggerLike } from "./globals.js";
+
+export class TrellisTasks {
+  #tasks: Record<string, Promise<Result<void, BaseError>>> = {};
+
+  #log: LoggerLike;
+
+  constructor(opts: { log?: LoggerLike }) {
+    this.#log = opts.log ?? logger;
+  }
+
+  /*
+   * Add a task to the task queue
+   */
+  add<E extends BaseError>(name: string, task: AsyncResult<void, E>) {
+    if (name in this.#tasks) {
+      this.#log.error({ name }, "Task already running?");
+      throw new Error(`Task ${name} already running`);
+    }
+    this.#log.debug({ name }, "Added task");
+    this.#tasks[name] = task.then((r: Result<void, E>) => {
+      if (Result.isErr(r)) {
+        this.#log.error(r, "Task encountered a runtime error");
+      }
+
+      return r;
+    });
+  }
+
+  wait(): AsyncResult<void, BaseError> {
+    return AsyncResult.any(Object.values(this.#tasks));
+  }
+}

package/src/telemetry/carrier.ts

@@ -0,0 +1,35 @@
+import {
+  type Context,
+  context,
+  propagation,
+  type Span,
+  trace,
+} from "@opentelemetry/api";
+
+export interface HeaderCarrier {
+  get(key: string): string | undefined;
+  set(key: string, value: string): void;
+}
+
+export function injectTraceContext(carrier: HeaderCarrier, span?: Span): void {
+  const ctx = span ? trace.setSpan(context.active(), span) : context.active();
+
+  propagation.inject(ctx, carrier, {
+    set: (c, k, v) => c.set(k, v),
+  });
+}
+
+export function extractTraceContext(carrier: HeaderCarrier): Context {
+  return propagation.extract(context.active(), carrier, {
+    get: (c, k) => c.get(k),
+    keys: () => [],
+  });
+}
+
+export function createMapCarrier(): HeaderCarrier {
+  const map = new Map<string, string>();
+  return {
+    get: (key: string) => map.get(key),
+    set: (key: string, value: string) => map.set(key, value),
+  };
+}