@oatscenter/trellis 0.100.0-rc.1

package/service/runtime/health.d.ts

@@ -0,0 +1,97 @@
+/**
+ * Health check types and utility functions for Trellis services.
+ *
+ * This module provides a standardized way to implement health checks
+ * for services, with support for individual check results and
+ * aggregated health status (healthy, degraded, unhealthy).
+ *
+ * @module
+ */
+import type { JsonValue } from "../../participant.ts";
+import { type HealthHeartbeatSample } from "../../internal_sdk/generated/types/_internal/p0.js";
+type MaybePromise<T> = T | Promise<T>;
+/**
+ * Result of a single health check.
+ */
+export type HealthCheckResult = {
+    /** Name of the health check */
+    name: string;
+    /** Status of the check: "ok" if passed, "failed" if not */
+    status: "ok" | "failed";
+    /** Error message if the check failed with an error */
+    error?: string;
+    /** Optional short human-readable summary */
+    summary?: string;
+    /** Optional structured metadata for the check */
+    info?: Record<string, JsonValue>;
+    /** Time in milliseconds the check took to execute */
+    latencyMs: number;
+};
+export type ServiceHealthInfo = {
+    version?: string;
+    info?: Record<string, JsonValue>;
+};
+export type ServiceHealthCheck = {
+    status: "ok" | "failed";
+    summary?: string;
+    info?: Record<string, JsonValue>;
+    error?: string;
+};
+export type ServiceHealthCheckFn = () => MaybePromise<ServiceHealthCheck>;
+export type ServiceHealthInfoFn = () => MaybePromise<ServiceHealthInfo | undefined>;
+/**
+ * Aggregated health response for a service.
+ */
+export type HealthResponse = {
+    /** Overall status: "healthy" if all pass, "unhealthy" if all fail, "degraded" if mixed */
+    status: "healthy" | "unhealthy" | "degraded";
+    /** Name of the service */
+    service: string;
+    /** ISO timestamp of when the health check was performed */
+    timestamp: string;
+    /** Individual health check results */
+    checks: HealthCheckResult[];
+};
+export declare function runServiceHealthCheck(name: string, check: ServiceHealthCheckFn): Promise<HealthCheckResult>;
+export declare function runAllServiceHealthChecks(service: string, checks: Record<string, ServiceHealthCheckFn>): Promise<HealthResponse>;
+export declare function createHealthHeartbeatSample(args: {
+    serviceName: string;
+    kind?: HealthHeartbeatSample["participant"]["kind"];
+    instanceId: string;
+    contractId: string;
+    contractDigest: string;
+    startedAt: string;
+    publishIntervalMs: number;
+    checks: HealthCheckResult[];
+    info?: ServiceHealthInfo;
+}): HealthHeartbeatSample;
+/** Public health enrichment surface exposed by connected participants. */
+export type ServiceHealth = Readonly<{
+    setInfo(info: ServiceHealthInfo | ServiceHealthInfoFn): void;
+    add(name: string, check: ServiceHealthCheckFn): () => void;
+}>;
+/** Internal health runtime used by automatic heartbeat publishers. */
+export declare class ServiceHealthRuntime implements ServiceHealth {
+    #private;
+    readonly serviceName: string;
+    readonly kind: HealthHeartbeatSample["participant"]["kind"];
+    readonly instanceId: string;
+    readonly contractId: string;
+    readonly contractDigest: string;
+    readonly startedAt: string;
+    readonly publishIntervalMs: number;
+    constructor(args: {
+        serviceName: string;
+        kind?: HealthHeartbeatSample["participant"]["kind"];
+        instanceId?: string;
+        contractId: string;
+        contractDigest: string;
+        publishIntervalMs: number;
+    });
+    setInfo(info: ServiceHealthInfo | ServiceHealthInfoFn): void;
+    add(name: string, check: ServiceHealthCheckFn): () => void;
+    checks(): Promise<HealthCheckResult[]>;
+    response(): Promise<HealthResponse>;
+    sample(): Promise<HealthHeartbeatSample>;
+}
+export {};

package/service/runtime/health.js

@@ -0,0 +1,153 @@
+/**
+ * Health check types and utility functions for Trellis services.
+ *
+ * This module provides a standardized way to implement health checks
+ * for services, with support for individual check results and
+ * aggregated health status (healthy, degraded, unhealthy).
+ *
+ * @module
+ */
+import { ulid } from "ulid";
+import { HealthHeartbeatSampleCodec, } from "../../internal_sdk/generated/types/_internal/p0.js";
+function summarizeHealthStatus(results) {
+    const allOk = results.every((r) => r.status === "ok");
+    const anyOk = results.some((r) => r.status === "ok");
+    return allOk ? "healthy" : anyOk ? "degraded" : "unhealthy";
+}
+function summarizeHealthChecks(results) {
+    const failedCount = results.filter((r) => r.status === "failed").length;
+    if (failedCount === 0) {
+        return undefined;
+    }
+    return `${failedCount} check${failedCount === 1 ? "" : "s"} failing`;
+}
+function detectRuntime() {
+    const maybeDeno = Reflect.get(globalThis, "Deno");
+    if (maybeDeno?.version?.deno) {
+        return { runtime: "deno", runtimeVersion: maybeDeno.version.deno };
+    }
+    const maybeProcess = Reflect.get(globalThis, "process");
+    if (typeof maybeProcess?.version === "string") {
+        return { runtime: "node", runtimeVersion: maybeProcess.version };
+    }
+    return { runtime: "unknown" };
+}
+export async function runServiceHealthCheck(name, check) {
+    const start = performance.now();
+    try {
+        const result = await check();
+        const latencyMs = performance.now() - start;
+        return {
+            name,
+            status: result.status,
+            error: result.error,
+            summary: result.summary,
+            info: result.info,
+            latencyMs,
+        };
+    }
+    catch (error) {
+        const latencyMs = performance.now() - start;
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+            name,
+            status: "failed",
+            error: message,
+            summary: message,
+            latencyMs,
+        };
+    }
+}
+export async function runAllServiceHealthChecks(service, checks) {
+    const results = await Promise.all(Object.entries(checks).map(([name, fn]) => runServiceHealthCheck(name, fn)));
+    return {
+        status: summarizeHealthStatus(results),
+        service,
+        timestamp: new Date().toISOString(),
+        checks: results,
+    };
+}
+export function createHealthHeartbeatSample(args) {
+    const runtime = detectRuntime();
+    const summary = summarizeHealthChecks(args.checks);
+    return HealthHeartbeatSampleCodec.decode({
+        sample: {
+            id: ulid(),
+            time: new Date().toISOString(),
+        },
+        participant: {
+            name: args.serviceName,
+            kind: args.kind ?? "service",
+            instanceId: args.instanceId,
+            contractId: args.contractId,
+            contractDigest: args.contractDigest,
+            startedAt: args.startedAt,
+            publishIntervalMs: String(args.publishIntervalMs),
+            runtime: runtime.runtime,
+            ...(runtime.runtimeVersion
+                ? { runtimeVersion: runtime.runtimeVersion }
+                : {}),
+            ...(args.info?.version ? { version: args.info.version } : {}),
+            ...(args.info?.info ? { info: args.info.info } : {}),
+        },
+        reportedStatus: summarizeHealthStatus(args.checks),
+        ...(summary ? { summary } : {}),
+        checks: args.checks,
+    });
+}
+/** Internal health runtime used by automatic heartbeat publishers. */
+export class ServiceHealthRuntime {
+    serviceName;
+    kind;
+    instanceId;
+    contractId;
+    contractDigest;
+    startedAt;
+    publishIntervalMs;
+    #checks = new Map();
+    #info;
+    constructor(args) {
+        this.serviceName = args.serviceName;
+        this.kind = args.kind ?? "service";
+        this.instanceId = args.instanceId ?? ulid();
+        this.contractId = args.contractId;
+        this.contractDigest = args.contractDigest;
+        this.startedAt = new Date().toISOString();
+        this.publishIntervalMs = args.publishIntervalMs;
+    }
+    setInfo(info) {
+        if (typeof info === "function") {
+            this.#info = info;
+            return;
+        }
+        this.#info = () => info;
+    }
+    add(name, check) {
+        this.#checks.set(name, check);
+        return () => {
+            this.#checks.delete(name);
+        };
+    }
+    async checks() {
+        return await Promise.all(Array.from(this.#checks.entries()).map(([name, check]) => runServiceHealthCheck(name, check)));
+    }
+    async response() {
+        const checks = Object.fromEntries(this.#checks.entries());
+        return await runAllServiceHealthChecks(this.serviceName, checks);
+    }
+    async sample() {
+        const checks = await this.checks();
+        const info = this.#info ? await this.#info() : undefined;
+        return createHealthHeartbeatSample({
+            serviceName: this.serviceName,
+            kind: this.kind,
+            instanceId: this.instanceId,
+            contractId: this.contractId,
+            contractDigest: this.contractDigest,
+            startedAt: this.startedAt,
+            publishIntervalMs: this.publishIntervalMs,
+            checks,
+            info,
+        });
+    }
+}

package/service/runtime/health_schemas.d.ts

@@ -0,0 +1,23 @@
+import { Type } from "typebox";
+export declare const HealthInfoSchema: Type.TRecord<"^.*$", Type.TUnknown>;
+export declare const HealthCheckResultSchema: Type.TObject<{
+    name: Type.TString;
+    status: Type.TUnion<[Type.TLiteral<"ok">, Type.TLiteral<"failed">]>;
+    error: Type.TOptional<Type.TString>;
+    summary: Type.TOptional<Type.TString>;
+    info: Type.TOptional<Type.TRecord<"^.*$", Type.TUnknown>>;
+    latencyMs: Type.TNumber;
+}>;
+export declare const HealthResponseSchema: Type.TObject<{
+    status: Type.TUnion<[Type.TLiteral<"healthy">, Type.TLiteral<"unhealthy">, Type.TLiteral<"degraded">]>;
+    service: Type.TString;
+    timestamp: Type.TString;
+    checks: Type.TArray<Type.TObject<{
+        name: Type.TString;
+        status: Type.TUnion<[Type.TLiteral<"ok">, Type.TLiteral<"failed">]>;
+        error: Type.TOptional<Type.TString>;
+        summary: Type.TOptional<Type.TString>;
+        info: Type.TOptional<Type.TRecord<"^.*$", Type.TUnknown>>;
+        latencyMs: Type.TNumber;
+    }>>;
+}>;

package/service/runtime/health_schemas.js

@@ -0,0 +1,20 @@
+import { Type } from "typebox";
+export const HealthInfoSchema = Type.Record(Type.String(), Type.Unknown());
+export const HealthCheckResultSchema = Type.Object({
+    name: Type.String(),
+    status: Type.Union([Type.Literal("ok"), Type.Literal("failed")]),
+    error: Type.Optional(Type.String()),
+    summary: Type.Optional(Type.String()),
+    info: Type.Optional(HealthInfoSchema),
+    latencyMs: Type.Number(),
+});
+export const HealthResponseSchema = Type.Object({
+    status: Type.Union([
+        Type.Literal("healthy"),
+        Type.Literal("unhealthy"),
+        Type.Literal("degraded"),
+    ]),
+    service: Type.String(),
+    timestamp: Type.String({ format: "date-time" }),
+    checks: Type.Array(HealthCheckResultSchema),
+});

package/service/runtime/internal_jobs/active-job.d.ts

@@ -0,0 +1,40 @@
+import type { Job, JobContext, JobLogEntry, JobProgress, JobWaitTarget } from "./types.ts";
+export declare class ActiveJobRuntimeError extends Error {
+    constructor(message: string);
+}
+export type ActiveJobRuntimeMetadata = {
+    redeliveryCount: number;
+};
+export declare class JobCancellationToken {
+    #private;
+    cancel(): void;
+    cancelForShutdown(): void;
+    cancelForLeaseLoss(): void;
+    get signal(): AbortSignal;
+    isCancelled(): boolean;
+    isJobCancelled(): boolean;
+    isHostShutdown(): boolean;
+    isLeaseLost(): boolean;
+}
+type ActiveJobHooks = {
+    updateProgress: (progress: JobProgress) => Promise<void>;
+    log: (entry: JobLogEntry) => Promise<void>;
+    heartbeat: () => Promise<void>;
+    waitFor: <T>(target: JobWaitTarget, fn: () => Promise<T>) => Promise<T>;
+};
+export declare class ActiveJob<TPayload = unknown, TResult = unknown> {
+    #private;
+    constructor(job: Job<TPayload, TResult>, cancellation: JobCancellationToken, hooks: ActiveJobHooks, metadata: ActiveJobRuntimeMetadata);
+    job(): Readonly<Job<TPayload, TResult>>;
+    context(): Readonly<JobContext>;
+    get signal(): AbortSignal;
+    isCancelled(): boolean;
+    cancellationToken(): JobCancellationToken;
+    redeliveryCount(): number;
+    isRedelivery(): boolean;
+    heartbeat(): Promise<void>;
+    updateProgress(progress: JobProgress): Promise<void>;
+    log(level: JobLogEntry["level"], message: string): Promise<void>;
+    waitFor<T>(target: JobWaitTarget, fn: () => Promise<T> | T): Promise<T>;
+}
+export {};

package/service/runtime/internal_jobs/active-job.js

@@ -0,0 +1,91 @@
+export class ActiveJobRuntimeError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "ActiveJobRuntimeError";
+    }
+}
+export class JobCancellationToken {
+    #controller = new AbortController();
+    #reason = "none";
+    cancel() {
+        if (this.#reason === "shutdown") {
+            return;
+        }
+        this.#reason = "job";
+        this.#controller.abort("job");
+    }
+    cancelForShutdown() {
+        this.#reason = "shutdown";
+        this.#controller.abort("shutdown");
+    }
+    cancelForLeaseLoss() {
+        if (this.#reason !== "none")
+            return;
+        this.#reason = "lease-lost";
+        this.#controller.abort("lease-lost");
+    }
+    get signal() {
+        return this.#controller.signal;
+    }
+    isCancelled() {
+        return this.#reason !== "none";
+    }
+    isJobCancelled() {
+        return this.#reason === "job";
+    }
+    isHostShutdown() {
+        return this.#reason === "shutdown";
+    }
+    isLeaseLost() {
+        return this.#reason === "lease-lost";
+    }
+}
+export class ActiveJob {
+    #job;
+    #cancellation;
+    #hooks;
+    #metadata;
+    constructor(job, cancellation, hooks, metadata) {
+        this.#job = job;
+        this.#cancellation = cancellation;
+        this.#hooks = hooks;
+        this.#metadata = metadata;
+    }
+    job() {
+        return this.#job;
+    }
+    context() {
+        return this.#job.context;
+    }
+    get signal() {
+        return this.#cancellation.signal;
+    }
+    isCancelled() {
+        return this.#cancellation.isCancelled();
+    }
+    cancellationToken() {
+        return this.#cancellation;
+    }
+    redeliveryCount() {
+        return this.#metadata.redeliveryCount;
+    }
+    isRedelivery() {
+        return this.#metadata.redeliveryCount > 0;
+    }
+    heartbeat() {
+        return this.#hooks.heartbeat();
+    }
+    updateProgress(progress) {
+        return this.#hooks.updateProgress(progress);
+    }
+    log(level, message) {
+        return this.#hooks.log({
+            timestamp: new Date().toISOString(),
+            level,
+            message,
+        });
+    }
+    waitFor(target, fn) {
+        return this.#hooks.waitFor(target, async () => await fn());
+    }
+}

package/service/runtime/internal_jobs/bindings.d.ts

@@ -0,0 +1,40 @@
+export type JobsQueueBinding = {
+    queueType: string;
+    publishPrefix: string;
+    workSubject: string;
+    consumerName: string;
+    payload: {
+        schema: string;
+    };
+    update?: {
+        schema: string;
+    };
+    updatesPrefix?: string;
+    result?: {
+        schema: string;
+    };
+    maxDeliver: number;
+    backoffMs: number[];
+    ackWaitMs: number;
+    defaultDeadlineMs?: number;
+    keyConcurrency?: {
+        key: string[];
+        maxActive: number;
+        heartbeatIntervalMs: number;
+        heartbeatTtlMs: number;
+        stalePolicy: "fail-stale" | "block";
+    };
+    queue?: {
+        maxQueuedPerKey: number;
+        whenFull: "reject" | "coalesce" | "replace-oldest";
+    };
+};
+export type JobsBinding = {
+    serviceName: string;
+    namespace: string;
+    queues: Record<string, JobsQueueBinding>;
+};
+export type JobsRuntimeBinding = {
+    jobs: JobsBinding;
+    workStream: string;
+};

package/service/runtime/internal_jobs/bindings.js

@@ -0,0 +1 @@
+export {};

package/service/runtime/internal_jobs/cancellation-registry.d.ts

@@ -0,0 +1,13 @@
+import type { JobCancellationToken } from "./active-job.ts";
+export declare class ActiveJobCancellationGuard {
+    #private;
+    constructor(key: string, token: JobCancellationToken, registry: ActiveJobCancellationRegistry);
+    dispose(): void;
+}
+export declare class ActiveJobCancellationRegistry {
+    #private;
+    register(key: string, token: JobCancellationToken): ActiveJobCancellationGuard;
+    cancel(key: string): boolean;
+    clearPending(key: string): void;
+    unregister(key: string, token: JobCancellationToken): void;
+}

package/service/runtime/internal_jobs/cancellation-registry.js

@@ -0,0 +1,55 @@
+export class ActiveJobCancellationGuard {
+    #key;
+    #token;
+    #registry;
+    #disposed = false;
+    constructor(key, token, registry) {
+        this.#key = key;
+        this.#token = token;
+        this.#registry = registry;
+    }
+    dispose() {
+        if (this.#disposed) {
+            return;
+        }
+        this.#disposed = true;
+        this.#registry.unregister(this.#key, this.#token);
+    }
+}
+export class ActiveJobCancellationRegistry {
+    #tokens = new Map();
+    #pending = new Set();
+    register(key, token) {
+        const tokens = this.#tokens.get(key) ?? new Set();
+        tokens.add(token);
+        this.#tokens.set(key, tokens);
+        if (this.#pending.delete(key)) {
+            token.cancel();
+        }
+        return new ActiveJobCancellationGuard(key, token, this);
+    }
+    cancel(key) {
+        const tokens = this.#tokens.get(key);
+        if (!tokens || tokens.size === 0) {
+            this.#pending.add(key);
+            return false;
+        }
+        for (const token of tokens) {
+            token.cancel();
+        }
+        return true;
+    }
+    clearPending(key) {
+        this.#pending.delete(key);
+    }
+    unregister(key, token) {
+        const tokens = this.#tokens.get(key);
+        if (!tokens) {
+            return;
+        }
+        tokens.delete(token);
+        if (tokens.size === 0) {
+            this.#tokens.delete(key);
+        }
+    }
+}

package/service/runtime/internal_jobs/heartbeat.d.ts

@@ -0,0 +1,33 @@
+import type { WorkerHeartbeat } from "./types.ts";
+type HeartbeatPublisher = {
+    publish(subject: string, payload: Uint8Array): void | Promise<void>;
+};
+type WorkerHeartbeatOptions = {
+    service: string;
+    jobType: string;
+    instanceId: string;
+    concurrency?: number;
+    version?: string;
+    timestamp: string;
+};
+type WorkerHeartbeatLoopOptions = {
+    publisher: HeartbeatPublisher;
+    service: string;
+    subjectService?: string;
+    jobType: string;
+    instanceId: string;
+    concurrency?: number;
+    version?: string;
+    intervalMs?: number;
+    nowIso?: () => string;
+};
+export declare class WorkerHeartbeatLoopError extends AggregateError {
+    constructor(errors: unknown[]);
+}
+export declare function workerHeartbeatSubject(service: string, jobType: string, instanceId: string): string;
+export declare function newWorkerHeartbeat(options: WorkerHeartbeatOptions): WorkerHeartbeat;
+export declare function publishWorkerHeartbeat(publisher: HeartbeatPublisher, heartbeat: WorkerHeartbeat, subjectService?: string): Promise<void>;
+export declare function startWorkerHeartbeatLoop(options: WorkerHeartbeatLoopOptions): Promise<{
+    stop(): Promise<void>;
+}>;
+export {};

package/service/runtime/internal_jobs/heartbeat.js

@@ -0,0 +1,65 @@
+export class WorkerHeartbeatLoopError extends AggregateError {
+    constructor(errors) {
+        super(errors, "worker heartbeat loop failed");
+        this.name = "WorkerHeartbeatLoopError";
+    }
+}
+export function workerHeartbeatSubject(service, jobType, instanceId) {
+    return `trellis.jobs.workers.${service}.${jobType}.${instanceId}.heartbeat`;
+}
+export function newWorkerHeartbeat(options) {
+    return {
+        service: options.service,
+        jobType: options.jobType,
+        instanceId: options.instanceId,
+        ...(options.concurrency !== undefined
+            ? { concurrency: options.concurrency }
+            : {}),
+        ...(options.version !== undefined ? { version: options.version } : {}),
+        timestamp: options.timestamp,
+    };
+}
+export async function publishWorkerHeartbeat(publisher, heartbeat, subjectService = heartbeat.service) {
+    await publisher.publish(workerHeartbeatSubject(subjectService, heartbeat.jobType, heartbeat.instanceId), new TextEncoder().encode(JSON.stringify(heartbeat)));
+}
+export async function startWorkerHeartbeatLoop(options) {
+    const nowIso = options.nowIso ?? (() => new Date().toISOString());
+    const intervalMs = options.intervalMs ?? 30000;
+    const errors = [];
+    const inFlight = new Set();
+    const publish = async () => {
+        await publishWorkerHeartbeat(options.publisher, newWorkerHeartbeat({
+            service: options.service,
+            jobType: options.jobType,
+            instanceId: options.instanceId,
+            ...(options.concurrency !== undefined
+                ? { concurrency: options.concurrency }
+                : {}),
+            ...(options.version !== undefined ? { version: options.version } : {}),
+            timestamp: nowIso(),
+        }), options.subjectService);
+    };
+    const publishSafely = () => {
+        const task = publish()
+            .catch((error) => {
+            errors.push(error);
+        })
+            .finally(() => {
+            inFlight.delete(task);
+        });
+        inFlight.add(task);
+    };
+    await publish();
+    const timer = setInterval(() => {
+        publishSafely();
+    }, intervalMs);
+    return {
+        async stop() {
+            clearInterval(timer);
+            await Promise.all(inFlight);
+            if (errors.length > 0) {
+                throw new WorkerHeartbeatLoopError(errors);
+            }
+        },
+    };
+}