ocel 0.0.1-alpha.0 → 0.0.1

package/dist/topic/topic.js

@@ -0,0 +1,117 @@
+import { createClient } from "@connectrpc/connect";
+import { refuseUnbound } from "../binding/binding.js";
+import { unprovisioned, unprovisionedPhase } from "../binding/unprovisioned.js";
+import { declarationSite } from "../declaration/callsite.js";
+import { defer } from "../declaration/defer.js";
+import { encodeDueAt, encodeDuration } from "../delivery/duration.js";
+import { encodeLane } from "../delivery/lane.js";
+import { encodePayload } from "../delivery/payload.js";
+import { encodeRetryPolicy } from "../delivery/retry.js";
+import { encodeJsonSchema } from "../delivery/schema.js";
+import { ResourceType } from "../gen/proto/app/resources/v1/resources_pb.js";
+import { TopicService } from "../gen/proto/app/topic/v1/topic_pb.js";
+import { rpc } from "../runtime/rpc.js";
+import { createRuntimeTransport } from "../runtime/transport.js";
+import { DEFAULT_WORKER, registerRun } from "../worker/registry.js";
+import { DeadLetters } from "./dead-letters.js";
+/** A declared topic, and the handle its messages are sent through. */
+export class Topic {
+    /** The name this topic was declared under. */
+    name;
+    schema;
+    client;
+    constructor(name, options) {
+        this.name = name;
+        this.schema = options.schema;
+        if (process.env.OCEL_PHASE === "discovery") {
+            defer(rpc.resource.declare({
+                resource: { name, type: ResourceType.TOPIC },
+                config: {
+                    case: "topic",
+                    value: {
+                        schema: encodeJsonSchema(options.schema),
+                        ordered: options.ordered ?? false,
+                        retry: encodeRetryPolicy(options.retry),
+                    },
+                },
+                source: declarationSite(),
+            }));
+        }
+    }
+    ensureClient(operation) {
+        if (unprovisionedPhase()) {
+            throw unprovisioned(`topic("${this.name}")`, operation);
+        }
+        const refusal = refuseUnbound(this.name, "topic");
+        if (refusal) {
+            throw refusal;
+        }
+        return (this.client ??= createClient(TopicService, createRuntimeTransport()));
+    }
+    /** Sends `payload` to every consumer of this topic, and answers the message's id. */
+    async send(payload, options = {}) {
+        const client = this.ensureClient("send");
+        const { messageId } = await client.send({
+            topic: this.name,
+            payload: encodePayload(payload),
+            dueAt: encodeDueAt(options.delay),
+            idempotencyKey: options.idempotencyKey ?? "",
+            key: options.key ?? "",
+            lane: encodeLane(options.lane),
+        });
+        return messageId;
+    }
+    declareConsumer(name, run, options, batch) {
+        registerRun({
+            kind: "consumer",
+            topic: this.name,
+            name,
+            worker: options.worker?.name ?? DEFAULT_WORKER,
+            batch: batch !== undefined,
+            schema: this.schema,
+            run,
+            hooks: {},
+        });
+        if (process.env.OCEL_PHASE === "discovery") {
+            defer(rpc.resource.declare({
+                resource: { name, type: ResourceType.CONSUMER },
+                config: {
+                    case: "consumer",
+                    value: {
+                        topic: this.name,
+                        worker: options.worker?.name ?? "",
+                        retry: encodeRetryPolicy(options.retry),
+                        concurrency: options.concurrency ?? 0,
+                        maxDuration: encodeDuration(options.maxDuration),
+                        lanes: (options.lanes ?? []).map(encodeLane),
+                        batch: batch && { size: batch.size, timeout: encodeDuration(batch.timeout) },
+                    },
+                },
+                source: declarationSite(),
+            }));
+        }
+        return { name, topic: this.name };
+    }
+    /** Declares a consumer that receives every message sent to this topic, one at a time. */
+    consumer(name, run, options = {}) {
+        return this.declareConsumer(name, run, options, undefined);
+    }
+    /** Declares a consumer that receives the messages sent to this topic in batches. */
+    batchConsumer(name, run, options) {
+        const { batchSize, batchTimeout, ...rest } = options;
+        return this.declareConsumer(name, run, rest, {
+            size: batchSize,
+            timeout: batchTimeout,
+        });
+    }
+    /** The messages a consumer of this topic gave up on. */
+    deadLetter(consumer) {
+        return new DeadLetters(typeof consumer === "string" ? consumer : consumer.name, (operation) => ({
+            client: this.ensureClient(operation),
+            topic: this.name,
+        }));
+    }
+}
+export function topic(name, options = {}) {
+    return new Topic(name, options);
+}

package/dist/version.d.ts

@@ -0,0 +1,2 @@
+/** The version of this SDK, stamped at release. */
+export declare const SDK_VERSION = "0.0.1";

package/dist/version.js

@@ -0,0 +1,2 @@
+/** The version of this SDK, stamped at release. */
+export const SDK_VERSION = "0.0.1";

package/dist/worker/context.d.ts

@@ -0,0 +1,36 @@
+/** What a run, its hooks and every middleware know about the attempt in progress. */
+export interface RunContext {
+    /** Whether a task's run or a topic consumer is being served. */
+    kind: "task" | "consumer";
+    /** The task's name, or the consumer's. */
+    name: string;
+    /** The topic the message was sent to; a task's topic is the task's own name. */
+    topic: string;
+    /** The run's id for a task, the execution's for a consumer; a batch's first message's. */
+    id: string;
+    /** Which attempt this is. */
+    attempt: {
+        /** This attempt's number, the first being 1. */
+        number: number;
+        /** How many attempts are made at most. */
+        of: number;
+        /** When the first attempt started. */
+        firstAttemptedAt: Date | undefined;
+    };
+    /** The message being served; a batch's first message. */
+    message: {
+        /** The message's id. */
+        id: string;
+        /** When the message was sent. */
+        publishedAt: Date | undefined;
+    };
+    /** Aborted when the run is canceled while this attempt is in progress. */
+    signal: AbortSignal;
+}
+/** The second argument a run or a consumer receives. */
+export interface RunOptions {
+    /** The attempt in progress. */
+    ctx: RunContext;
+    /** Aborted when the run is canceled while this attempt is in progress. */
+    signal: AbortSignal;
+}

package/dist/worker/context.js

@@ -0,0 +1 @@
+export {};

package/dist/worker/deliver.d.ts

@@ -0,0 +1,13 @@
+/** The HTTP answer `deliver` gives the message's sender. */
+export interface DeliveryAnswer {
+    /** 200 on success, 422 to fail without retry, 500 to retry, 400 or 404 for a bad delivery. */
+    status: number;
+    /** The run's output as JSON, the abort as JSON, or a plain-text reason. */
+    body: string;
+}
+/**
+ * Serves one message the worker named `workerName` received: `body` is the envelope POSTed
+ * to it, and `signal` aborts when the run is canceled. Answers what to send back: 200 with
+ * the output, 422 to fail without retry, 500 to retry, 400 or 404 for a bad delivery.
+ */
+export declare function deliver(workerName: string, body: string, signal?: AbortSignal): Promise<DeliveryAnswer>;

package/dist/worker/deliver.js

@@ -0,0 +1,258 @@
+import { fromJson } from "@bufbuild/protobuf";
+import { timestampDate } from "@bufbuild/protobuf/wkt";
+import { validatePayload } from "../delivery/schema.js";
+import { EnvelopeSchema } from "../gen/proto/app/topic/v1/topic_pb.js";
+import { AbortTaskRunError } from "../task/errors.js";
+import { findRegistration, findWorker } from "./registry.js";
+import { Semaphore } from "./semaphore.js";
+const describeError = (error) => (error instanceof Error ? error.message : String(error));
+const createAbortAnswer = (reason) => ({
+    status: 422,
+    body: JSON.stringify({ abort: { reason } }),
+});
+const decodeTimestamp = (timestamp) => timestamp ? timestampDate(timestamp) : undefined;
+function parseEnvelope(body) {
+    const parsed = JSON.parse(body);
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+        throw new Error("the envelope is not a JSON object");
+    }
+    const { payload, messages, ...fields } = parsed;
+    const deliveries = Array.isArray(messages) ? messages : [];
+    const bare = Array.isArray(messages)
+        ? deliveries.map(({ payload: _payload, ...delivery }) => delivery)
+        : messages;
+    const envelope = fromJson(EnvelopeSchema, (bare === undefined ? fields : { ...fields, messages: bare }), { ignoreUnknownFields: true });
+    const payloads = deliveries.length > 0
+        ? deliveries.map((delivery) => delivery.payload ?? null)
+        : [payload ?? null];
+    return { envelope, payloads };
+}
+const starts = new Map();
+const semaphores = new Map();
+function ensureStarted(workerName) {
+    const onStart = findWorker(workerName)?.onStart;
+    if (!onStart)
+        return Promise.resolve();
+    let start = starts.get(workerName);
+    if (!start) {
+        start = Promise.resolve()
+            .then(onStart)
+            .catch((error) => {
+            starts.delete(workerName);
+            throw error;
+        });
+        starts.set(workerName, start);
+    }
+    return start;
+}
+function ensureSemaphore(workerName) {
+    const concurrency = findWorker(workerName)?.concurrency;
+    if (!concurrency || concurrency < 1)
+        return undefined;
+    let semaphore = semaphores.get(workerName);
+    if (!semaphore) {
+        semaphore = new Semaphore(concurrency);
+        semaphores.set(workerName, semaphore);
+    }
+    return semaphore;
+}
+function reportHookError(hook, registration, error) {
+    console.error(`ocel: ${registration.kind} "${registration.name}" ${hook} threw`, error);
+}
+async function callHook(hook, registration, call) {
+    try {
+        await call();
+    }
+    catch (error) {
+        reportHookError(hook, registration, error);
+    }
+}
+function createRunContext(registration, envelope, signal) {
+    const first = envelope.messages[0];
+    const message = first?.message ?? envelope.message;
+    const attempt = first?.attempt ?? envelope.attempt;
+    return {
+        kind: registration.kind,
+        name: registration.name,
+        topic: registration.topic,
+        id: first?.execution ?? envelope.execution,
+        attempt: {
+            number: attempt?.number ?? 1,
+            of: attempt?.of ?? 1,
+            firstAttemptedAt: decodeTimestamp(attempt?.firstAttemptedAt),
+        },
+        message: { id: message?.id ?? "", publishedAt: decodeTimestamp(message?.publishedAt) },
+        signal,
+    };
+}
+async function decodePayload(registration, { envelope, payloads: values }) {
+    const isBatchEnvelope = envelope.messages.length > 0;
+    if (!registration.batch && isBatchEnvelope) {
+        return {
+            ok: false,
+            answer: {
+                status: 400,
+                body: `${registration.kind} "${registration.name}" takes one message at a time, and this envelope carries a batch`,
+            },
+        };
+    }
+    const { schema } = registration;
+    if (!schema)
+        return { ok: true, payload: registration.batch ? values : values[0] };
+    const validated = [];
+    for (const value of values) {
+        const result = await validatePayload(schema, value);
+        if (!result.ok) {
+            return {
+                ok: false,
+                answer: createAbortAnswer(`the payload does not match the schema: ${result.message}`),
+            };
+        }
+        validated.push(result.value);
+    }
+    return { ok: true, payload: registration.batch ? validated : validated[0] };
+}
+async function serveAttempt(registration, workerName, delivered, signal) {
+    const parsed = await decodePayload(registration, delivered);
+    if (!parsed.ok)
+        return parsed.answer;
+    const { payload } = parsed;
+    const ctx = createRunContext(registration, delivered.envelope, signal);
+    const { hooks } = registration;
+    const workerMiddleware = findWorker(workerName)?.middleware;
+    let canceled = false;
+    const onAbort = () => {
+        canceled = true;
+        if (hooks.onCancel) {
+            void callHook("onCancel", registration, () => hooks.onCancel?.({ payload, ctx }));
+        }
+    };
+    if (signal.aborted)
+        onAbort();
+    else
+        signal.addEventListener("abort", onAbort, { once: true });
+    let output;
+    const runAttempt = async () => {
+        await hooks.onStartAttempt?.({ payload, ctx });
+        const run = async () => {
+            output = await registration.run(payload, { ctx, signal });
+        };
+        if (hooks.middleware)
+            await hooks.middleware({ payload, ctx, next: run });
+        else
+            await run();
+    };
+    let failure;
+    try {
+        if (workerMiddleware)
+            await workerMiddleware({ ctx, next: runAttempt });
+        else
+            await runAttempt();
+    }
+    catch (error) {
+        failure = { error };
+    }
+    finally {
+        signal.removeEventListener("abort", onAbort);
+    }
+    let body = "null";
+    if (!failure) {
+        try {
+            body = JSON.stringify(output ?? null) ?? "null";
+        }
+        catch (error) {
+            failure = {
+                error: new AbortTaskRunError(`the output does not encode as JSON: ${describeError(error)}`),
+            };
+        }
+    }
+    if (!failure) {
+        if (!canceled)
+            await callCompletionHooks(registration, { payload, ctx, result: { ok: true, output } });
+        return { status: 200, body };
+    }
+    const { error } = failure;
+    let abort = error instanceof AbortTaskRunError;
+    if (!abort && hooks.catchError) {
+        try {
+            const answer = (await hooks.catchError({ payload, error, ctx }));
+            abort = answer?.skipRetrying === true;
+        }
+        catch (hookError) {
+            reportHookError("catchError", registration, hookError);
+        }
+    }
+    if (!canceled && (abort || ctx.attempt.number >= ctx.attempt.of)) {
+        await callCompletionHooks(registration, { payload, ctx, result: { ok: false, error } });
+    }
+    return abort
+        ? createAbortAnswer(describeError(error))
+        : { status: 500, body: describeError(error) };
+}
+async function callCompletionHooks(registration, args) {
+    const { hooks } = registration;
+    const { payload, ctx, result } = args;
+    if (result.ok) {
+        if (hooks.onSuccess) {
+            await callHook("onSuccess", registration, () => hooks.onSuccess?.({ payload, output: result.output, ctx }));
+        }
+    }
+    else if (hooks.onFailure) {
+        await callHook("onFailure", registration, () => hooks.onFailure?.({ payload, error: result.error, ctx }));
+    }
+    if (hooks.onComplete) {
+        await callHook("onComplete", registration, () => hooks.onComplete?.({ payload, ctx, result }));
+    }
+}
+/**
+ * Serves one message the worker named `workerName` received: `body` is the envelope POSTed
+ * to it, and `signal` aborts when the run is canceled. Answers what to send back: 200 with
+ * the output, 422 to fail without retry, 500 to retry, 400 or 404 for a bad delivery.
+ */
+export async function deliver(workerName, body, signal = new AbortController().signal) {
+    let delivered;
+    try {
+        delivered = parseEnvelope(body);
+    }
+    catch (error) {
+        return { status: 400, body: `the body is not a message envelope: ${describeError(error)}` };
+    }
+    const { envelope } = delivered;
+    const registration = findRegistration(envelope.topic, envelope.consumer);
+    if (!registration) {
+        return {
+            status: 404,
+            body: `no consumer "${envelope.consumer}" of topic "${envelope.topic}" is declared in this app`,
+        };
+    }
+    if (registration.worker !== workerName) {
+        return {
+            status: 404,
+            body: `consumer "${envelope.consumer}" of topic "${envelope.topic}" runs on worker "${registration.worker}", not on worker "${workerName}"`,
+        };
+    }
+    try {
+        await ensureStarted(workerName);
+    }
+    catch (error) {
+        return { status: 500, body: `worker "${workerName}" failed to start: ${describeError(error)}` };
+    }
+    const semaphore = ensureSemaphore(workerName);
+    if (!semaphore)
+        return serveAttempt(registration, workerName, delivered, signal);
+    try {
+        await semaphore.acquire(signal);
+    }
+    catch {
+        return {
+            status: 500,
+            body: `the delivery was canceled while it waited for a free slot on worker "${workerName}"`,
+        };
+    }
+    try {
+        return await serveAttempt(registration, workerName, delivered, signal);
+    }
+    finally {
+        semaphore.release();
+    }
+}

package/dist/worker/index.d.ts

@@ -0,0 +1,4 @@
+export type { RunContext, RunOptions } from "./context.js";
+export { type DeliveryAnswer, deliver } from "./deliver.js";
+export type { WorkerMiddleware } from "./registry.js";
+export { Worker, type WorkerOptions, worker } from "./worker.js";

package/dist/worker/index.js

@@ -0,0 +1,2 @@
+export { deliver } from "./deliver.js";
+export { Worker, worker } from "./worker.js";

package/dist/worker/registry.d.ts

@@ -0,0 +1,28 @@
+import type { StandardSchemaV1 } from "@standard-schema/spec";
+import type { TaskHooks } from "../task/hooks.js";
+import type { RunContext, RunOptions } from "./context.js";
+export declare const DEFAULT_WORKER = "worker";
+export interface Registration {
+    kind: "task" | "consumer";
+    topic: string;
+    name: string;
+    worker: string;
+    batch: boolean;
+    schema: StandardSchemaV1 | undefined;
+    run: (payload: unknown, options: RunOptions) => unknown;
+    hooks: TaskHooks<unknown, unknown>;
+}
+/** Wraps every run a worker serves, of its tasks and its consumers alike; `next` runs it. */
+export type WorkerMiddleware = (args: {
+    ctx: RunContext;
+    next: () => Promise<void>;
+}) => unknown;
+export interface WorkerRegistration {
+    concurrency: number | undefined;
+    onStart: (() => unknown) | undefined;
+    middleware: WorkerMiddleware | undefined;
+}
+export declare function registerRun(registration: Registration): void;
+export declare function findRegistration(topic: string, consumer: string): Registration | undefined;
+export declare function registerWorker(name: string, worker: WorkerRegistration): void;
+export declare function findWorker(name: string): WorkerRegistration | undefined;

package/dist/worker/registry.js

@@ -0,0 +1,22 @@
+export const DEFAULT_WORKER = "worker";
+const registryKey = Symbol.for("ocel.worker.registry");
+function ensureRegistry() {
+    const global = globalThis;
+    global[registryKey] ??= { runs: new Map(), workers: new Map() };
+    return global[registryKey];
+}
+function formatRunKey(topic, consumer) {
+    return `${topic}\u0000${consumer}`;
+}
+export function registerRun(registration) {
+    ensureRegistry().runs.set(formatRunKey(registration.topic, registration.name), registration);
+}
+export function findRegistration(topic, consumer) {
+    return ensureRegistry().runs.get(formatRunKey(topic, consumer));
+}
+export function registerWorker(name, worker) {
+    ensureRegistry().workers.set(name, worker);
+}
+export function findWorker(name) {
+    return ensureRegistry().workers.get(name);
+}

package/dist/worker/semaphore.d.ts

@@ -0,0 +1,7 @@
+export declare class Semaphore {
+    private available;
+    private waiting;
+    constructor(permits: number);
+    acquire(signal: AbortSignal): Promise<void>;
+    release(): void;
+}

package/dist/worker/semaphore.js

@@ -0,0 +1,36 @@
+export class Semaphore {
+    available;
+    waiting = [];
+    constructor(permits) {
+        this.available = permits;
+    }
+    acquire(signal) {
+        if (this.available > 0) {
+            this.available--;
+            return Promise.resolve();
+        }
+        if (signal.aborted)
+            return Promise.reject(signal.reason);
+        return new Promise((resolve, reject) => {
+            const takePermit = () => {
+                signal.removeEventListener("abort", stopWaiting);
+                resolve();
+            };
+            const stopWaiting = () => {
+                const index = this.waiting.indexOf(takePermit);
+                if (index !== -1)
+                    this.waiting.splice(index, 1);
+                reject(signal.reason);
+            };
+            this.waiting.push(takePermit);
+            signal.addEventListener("abort", stopWaiting, { once: true });
+        });
+    }
+    release() {
+        const next = this.waiting.shift();
+        if (next)
+            next();
+        else
+            this.available++;
+    }
+}

package/dist/worker/worker.d.ts

@@ -0,0 +1,21 @@
+import { type WorkerMiddleware } from "./registry.js";
+/** How a worker is declared. */
+export interface WorkerOptions {
+    /** The most runs this worker serves at once, across all its tasks and consumers. */
+    concurrency?: number;
+    /** Runs once per process, before the first run it serves. */
+    onStart?: () => unknown;
+    /** Wraps every run this worker serves. */
+    middleware?: WorkerMiddleware;
+}
+/** A declared worker: the app that serves the tasks and consumers placed on it. */
+export declare class Worker {
+    /** The name this worker was declared under, and the `ocel.json` app it joins. */
+    readonly name: string;
+    constructor(name: string, options: WorkerOptions);
+}
+/**
+ * Declares a worker. A task or consumer runs on it when declared with `worker: <this>`;
+ * without one, it runs on the worker named `worker`.
+ */
+export declare function worker(name: string, options?: WorkerOptions): Worker;

package/dist/worker/worker.js

@@ -0,0 +1,32 @@
+import { declarationSite } from "../declaration/callsite.js";
+import { defer } from "../declaration/defer.js";
+import { ResourceType } from "../gen/proto/app/resources/v1/resources_pb.js";
+import { rpc } from "../runtime/rpc.js";
+import { registerWorker } from "./registry.js";
+/** A declared worker: the app that serves the tasks and consumers placed on it. */
+export class Worker {
+    /** The name this worker was declared under, and the `ocel.json` app it joins. */
+    name;
+    constructor(name, options) {
+        this.name = name;
+        registerWorker(name, {
+            concurrency: options.concurrency,
+            onStart: options.onStart,
+            middleware: options.middleware,
+        });
+        if (process.env.OCEL_PHASE === "discovery") {
+            defer(rpc.resource.declare({
+                resource: { name, type: ResourceType.WORKER },
+                config: { case: "worker", value: { concurrency: options.concurrency ?? 0 } },
+                source: declarationSite(),
+            }));
+        }
+    }
+}
+/**
+ * Declares a worker. A task or consumer runs on it when declared with `worker: <this>`;
+ * without one, it runs on the worker named `worker`.
+ */
+export function worker(name, options = {}) {
+    return new Worker(name, options);
+}