@jondotsoy/pika 0.1.1

package/lib/esm/flow/configs/env-configs.d.ts

@@ -0,0 +1,37 @@
+import { z } from "zod";
+import { RunMode } from "../constants/run-mode.js";
+export declare class EnvConfigs {
+    private constructor();
+    static configParser: z.ZodObject<{
+        contextPath: z.ZodCatch<z.ZodString>;
+        runMode: z.ZodCatch<z.ZodUnion<readonly [z.ZodPipe<z.ZodLiteral<"IN_PROCESS">, z.ZodTransform<RunMode, "IN_PROCESS">>, z.ZodPipe<z.ZodLiteral<"CLUSTER">, z.ZodTransform<RunMode, "CLUSTER">>, z.ZodPipe<z.ZodLiteral<"CLUSTER_PRIMARY">, z.ZodTransform<RunMode, "CLUSTER_PRIMARY">>, z.ZodPipe<z.ZodLiteral<"CLUSTER_WORKER">, z.ZodTransform<RunMode, "CLUSTER_WORKER">>]>>;
+        concurrentWorkers: z.ZodCatch<z.ZodUnion<readonly [z.ZodNumber, z.ZodPipe<z.ZodString, z.ZodTransform<number, string>>]>>;
+        disableLoadJob: z.ZodCatch<z.ZodUnion<readonly [z.ZodBoolean, z.ZodPipe<z.ZodNumber, z.ZodTransform<boolean, number>>, z.ZodPipe<z.ZodString, z.ZodTransform<boolean, string>>]>>;
+        disableContextFile: z.ZodCatch<z.ZodUnion<readonly [z.ZodBoolean, z.ZodPipe<z.ZodNumber, z.ZodTransform<boolean, number>>, z.ZodPipe<z.ZodString, z.ZodTransform<boolean, string>>]>>;
+        cliInspectorUrl: z.ZodCatch<z.ZodNullable<z.ZodString>>;
+        dsn: z.ZodCatch<z.ZodNullable<z.ZodString>>;
+    }, z.core.$strip>;
+    static loadEnvironmentSettings(): Promise<{
+        contextPath: string;
+        runMode: RunMode;
+        concurrentWorkers: number;
+        disableLoadJob: boolean;
+        disableContextFile: boolean;
+        cliInspectorUrl: string | null;
+        dsn: string | null;
+    }>;
+    private static __values;
+    /**
+     * Returns the parsed environment configuration, loading it once and caching the result.
+     * @returns The parsed environment configuration object.
+     */
+    static configs(): Promise<{
+        contextPath: string;
+        runMode: RunMode;
+        concurrentWorkers: number;
+        disableLoadJob: boolean;
+        disableContextFile: boolean;
+        cliInspectorUrl: string | null;
+        dsn: string | null;
+    }>;
+}

package/lib/esm/flow/configs/env-configs.js

@@ -0,0 +1,59 @@
+import { ulid } from "ulid";
+import { z } from "zod";
+import { RunMode } from "../constants/run-mode.js";
+const booleable = z.union([
+    z.boolean(),
+    z.number().transform((num) => !!num),
+    z
+        .string()
+        .transform((str) => ["1", "true", "yes"].includes(str.toLowerCase())),
+]);
+const numeric = z.union([
+    z.number(),
+    z
+        .string()
+        .regex(/^\d+$/)
+        .transform((str) => parseInt(str, 10)),
+]);
+export class EnvConfigs {
+    constructor() { }
+    static configParser = z.object({
+        contextPath: z
+            .string()
+            .min(1)
+            .catch(() => `${process.cwd()}/.session/${ulid()}.yaml`),
+        runMode: z
+            .union([
+            z.literal("IN_PROCESS").transform(() => RunMode.IN_PROCESS),
+            z.literal("CLUSTER").transform(() => RunMode.CLUSTER_PRIMARY),
+            z.literal("CLUSTER_PRIMARY").transform(() => RunMode.CLUSTER_PRIMARY),
+            z.literal("CLUSTER_WORKER").transform(() => RunMode.CLUSTER_WORKER),
+        ])
+            .catch(RunMode.IN_PROCESS),
+        concurrentWorkers: numeric.catch(1),
+        disableLoadJob: booleable.catch(false),
+        disableContextFile: booleable.catch(false),
+        cliInspectorUrl: z.string().url().nullable().catch(null),
+        dsn: z.string().url().nullable().catch(null),
+    });
+    static async loadEnvironmentSettings() {
+        return this.configParser.parse({
+            contextPath: process.env.PIKA_CONTEXT_PATH,
+            runMode: process.env.PIKA_RUN_MODE,
+            concurrentWorkers: process.env.PIKA_CONCURRENT_WORKERS,
+            disableLoadJob: process.env.PIKA_DISABLE_LOAD_JOB,
+            disableContextFile: process.env.PIKA_DISABLE_CONTEXT_FILE,
+            cliInspectorUrl: process.env.PIKA_CLI_INSPECTOR_URL,
+            dsn: process.env.PIKA_DSN,
+        });
+    }
+    static __values = null;
+    /**
+     * Returns the parsed environment configuration, loading it once and caching the result.
+     * @returns The parsed environment configuration object.
+     */
+    static async configs() {
+        EnvConfigs.__values ??= await EnvConfigs.loadEnvironmentSettings();
+        return EnvConfigs.__values;
+    }
+}

package/lib/esm/flow/constants/run-mode.d.ts

@@ -0,0 +1,5 @@
+export declare enum RunMode {
+    IN_PROCESS = "IN_PROCESS",
+    CLUSTER_PRIMARY = "CLUSTER_PRIMARY",
+    CLUSTER_WORKER = "CLUSTER_WORKER"
+}

package/lib/esm/flow/constants/run-mode.js

@@ -0,0 +1,6 @@
+export var RunMode;
+(function (RunMode) {
+    RunMode["IN_PROCESS"] = "IN_PROCESS";
+    RunMode["CLUSTER_PRIMARY"] = "CLUSTER_PRIMARY";
+    RunMode["CLUSTER_WORKER"] = "CLUSTER_WORKER";
+})(RunMode || (RunMode = {}));

package/lib/esm/flow/context-reflects/cli-http-reflect.d.ts

@@ -0,0 +1,9 @@
+import { Context } from "../context.js";
+export declare class CliHttpReflect {
+    readonly context: Context;
+    readonly inspectorUrl: string;
+    constructor(context: Context, inspectorUrl: string);
+    private post;
+    private attach;
+    static attach(context: Context, inspectorUrl: string): CliHttpReflect;
+}

package/lib/esm/flow/context-reflects/cli-http-reflect.js

@@ -0,0 +1,55 @@
+export class CliHttpReflect {
+    context;
+    inspectorUrl;
+    constructor(context, inspectorUrl) {
+        this.context = context;
+        this.inspectorUrl = inspectorUrl;
+        this.attach();
+    }
+    post(payload) {
+        fetch(this.inspectorUrl, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify(payload),
+        }).catch(() => { });
+    }
+    attach() {
+        const { context } = this;
+        context.events.addEventListener("job_created", (jobId) => {
+            const job = context.jobs.get(jobId);
+            if (job)
+                this.post({ event: "job_created", jobId, job });
+        });
+        context.events.addEventListener("job_started", (jobId) => {
+            const job = context.jobs.get(jobId);
+            if (job)
+                this.post({ event: "job_started", jobId, job });
+        });
+        context.events.addEventListener("job_done", (jobId) => {
+            const job = context.jobs.get(jobId);
+            if (job)
+                this.post({ event: "job_done", jobId, job });
+        });
+        context.events.addEventListener("job_error", (jobId, err) => {
+            const job = context.jobs.get(jobId);
+            if (job)
+                this.post({
+                    event: "job_error",
+                    jobId,
+                    job,
+                    error: {
+                        name: err?.name ?? "Error",
+                        message: err?.message ?? String(err),
+                    },
+                });
+        });
+        context.events.addEventListener("job_retry", (jobId, retryCount) => {
+            const job = context.jobs.get(jobId);
+            if (job)
+                this.post({ event: "job_retry", jobId, job, retryCount });
+        });
+    }
+    static attach(context, inspectorUrl) {
+        return new CliHttpReflect(context, inspectorUrl);
+    }
+}

package/lib/esm/flow/context-reflects/dsn-reflect.d.ts

@@ -0,0 +1,29 @@
+import { Context } from "../context.js";
+import { EventManager } from "../utils/event-manager.js";
+import { Bytes } from "../utils/bytes.js";
+export type DsnReflectOptions = {
+    dsn: string;
+    maxEventsPerRequest?: number;
+    waitTime?: number;
+    maxRequestSize?: Bytes;
+    maxRetries?: number;
+};
+export declare class DsnReflect {
+    readonly context: Context;
+    private messages;
+    private messageResolver;
+    private options;
+    readonly events: EventManager<{
+        push_message: (message: {
+            eventId: string;
+            body: Uint8Array;
+        }) => void;
+    }>;
+    constructor(context: Context, options: DsnReflectOptions);
+    static defaultOptions(partial?: DsnReflectOptions): Required<DsnReflectOptions>;
+    private post;
+    private startSendLoop;
+    private sendRequest;
+    private attach;
+    static attach(context: Context, options: DsnReflectOptions): DsnReflect;
+}

package/lib/esm/flow/context-reflects/dsn-reflect.js

@@ -0,0 +1,181 @@
+import { EventManager } from "../utils/event-manager.js";
+import { Bytes } from "../utils/bytes.js";
+export class DsnReflect {
+    context;
+    messages = new Set();
+    messageResolver = Promise.withResolvers();
+    options;
+    events = new EventManager();
+    constructor(context, options) {
+        this.context = context;
+        this.options = DsnReflect.defaultOptions(options);
+        this.events.addEventListener("push_message", () => {
+            this.messageResolver.resolve();
+        });
+        this.startSendLoop();
+        this.attach();
+    }
+    static defaultOptions(partial) {
+        return {
+            dsn: partial?.dsn ?? "",
+            maxEventsPerRequest: partial?.maxEventsPerRequest ?? 100,
+            waitTime: partial?.waitTime ?? 0,
+            maxRequestSize: partial?.maxRequestSize ?? Bytes.from({ mb: 1 }),
+            maxRetries: partial?.maxRetries ?? 3,
+        };
+    }
+    post(notification) {
+        const body = new TextEncoder().encode(JSON.stringify(notification));
+        const message = { eventId: notification.params.eventId, body };
+        this.messages.add(message);
+        this.events.emit("push_message", message);
+    }
+    async startSendLoop() {
+        const { maxEventsPerRequest, waitTime, maxRequestSize } = this.options;
+        const maxBytes = maxRequestSize.bytes;
+        while (true) {
+            await this.messageResolver.promise;
+            this.messageResolver = Promise.withResolvers();
+            const requests = [];
+            while (this.messages.size > 0) {
+                const batch = [];
+                let totalSize = 0n;
+                for (const msg of this.messages) {
+                    if (batch.length >= maxEventsPerRequest)
+                        break;
+                    if (batch.length > 0 &&
+                        totalSize + BigInt(msg.body.length) > maxBytes)
+                        break;
+                    batch.push(msg);
+                    totalSize += BigInt(msg.body.length);
+                }
+                if (batch.length === 0)
+                    break;
+                for (const msg of batch)
+                    this.messages.delete(msg);
+                const newline = new TextEncoder().encode("\n");
+                const parts = [];
+                for (let i = 0; i < batch.length; i++) {
+                    parts.push(batch[i].body);
+                    if (i < batch.length - 1)
+                        parts.push(newline);
+                }
+                const combined = new Uint8Array(parts.reduce((s, p) => s + p.length, 0));
+                let offset = 0;
+                for (const p of parts) {
+                    combined.set(p, offset);
+                    offset += p.length;
+                }
+                requests.push({
+                    url: this.options.dsn,
+                    init: {
+                        method: "POST",
+                        headers: {
+                            "Content-Type": "application/vnd.pika.jsonrpc+jsonl; charset=utf-8",
+                            "X-Pika-Event-Count": String(batch.length),
+                        },
+                        body: combined,
+                    },
+                });
+            }
+            for (const request of requests) {
+                await this.sendRequest(request);
+            }
+            if (waitTime > 0)
+                await new Promise((r) => setTimeout(r, waitTime));
+        }
+    }
+    async sendRequest(request) {
+        const { maxRetries } = this.options;
+        let attempt = 0;
+        while (attempt <= maxRetries) {
+            let response = null;
+            try {
+                response = await fetch(request.url, request.init);
+            }
+            catch {
+                return;
+            }
+            if (response.status === 429) {
+                attempt++;
+                if (attempt > maxRetries)
+                    return;
+                const retryAfter = response.headers.get("Retry-After");
+                const waitMs = retryAfter ? parseFloat(retryAfter) * 1000 : 1000;
+                await new Promise((r) => setTimeout(r, waitMs));
+                continue;
+            }
+            return;
+        }
+    }
+    attach() {
+        const { context } = this;
+        this.post({
+            jsonrpc: "2.0",
+            method: "system.ready",
+            params: { eventId: crypto.randomUUID(), timestamp: Date.now() },
+        });
+        context.events.addEventListener("job_created", (jobId) => {
+            const job = context.jobs.get(jobId);
+            if (job)
+                this.post({
+                    jsonrpc: "2.0",
+                    method: "job.create",
+                    params: { eventId: crypto.randomUUID(), timestamp: Date.now(), job },
+                });
+        });
+        context.events.addEventListener("job_started", (jobId) => {
+            const job = context.jobs.get(jobId);
+            if (job)
+                this.post({
+                    jsonrpc: "2.0",
+                    method: "job.start",
+                    params: { eventId: crypto.randomUUID(), timestamp: Date.now(), job },
+                });
+        });
+        context.events.addEventListener("job_done", (jobId) => {
+            const job = context.jobs.get(jobId);
+            if (job)
+                this.post({
+                    jsonrpc: "2.0",
+                    method: "job.done",
+                    params: { eventId: crypto.randomUUID(), timestamp: Date.now(), job },
+                });
+        });
+        context.events.addEventListener("job_error", (jobId, err) => {
+            const job = context.jobs.get(jobId);
+            if (!job)
+                return;
+            const error = err instanceof Error
+                ? { name: err.name, message: err.message, stack: err.stack ?? null }
+                : { name: "Error", message: String(err), stack: null };
+            this.post({
+                jsonrpc: "2.0",
+                method: "job.error",
+                params: {
+                    eventId: crypto.randomUUID(),
+                    timestamp: Date.now(),
+                    job,
+                    error,
+                },
+            });
+        });
+        context.events.addEventListener("job_retry", (jobId, retryCount) => {
+            const job = context.jobs.get(jobId);
+            if (job)
+                this.post({
+                    jsonrpc: "2.0",
+                    method: "job.retry",
+                    params: {
+                        eventId: crypto.randomUUID(),
+                        timestamp: Date.now(),
+                        job,
+                        retryCount,
+                    },
+                });
+        });
+    }
+    static attach(context, options) {
+        return new DsnReflect(context, options);
+    }
+}

package/lib/esm/flow/context-reflects/file-context-reflect.d.ts

@@ -0,0 +1,22 @@
+import { Context } from "../context";
+export declare class FileContextReflect {
+    readonly context: Context;
+    readonly path: URL | string;
+    private _writePending;
+    constructor(context: Context, path: URL | string);
+    write(): Promise<void>;
+    static read(path: URL | string): Promise<{
+        context: Context;
+        reflector: FileContextReflect;
+    }>;
+    static create(path: URL | string): Promise<{
+        context: Context;
+        reflector: FileContextReflect;
+    }>;
+    static load(path: URL | string, options?: {
+        disabled?: boolean;
+    }): Promise<{
+        context: Context;
+        reflector: FileContextReflect | null;
+    }>;
+}

package/lib/esm/flow/context-reflects/file-context-reflect.js

@@ -0,0 +1,47 @@
+import * as YAML from "yaml";
+import { Context } from "../context";
+import { File } from "../utils/file";
+export class FileContextReflect {
+    context;
+    path;
+    _writePending = Promise.resolve();
+    constructor(context, path) {
+        this.context = context;
+        this.path = path;
+        context.events.addEventListener("manifest_updated", async () => {
+            await this.write();
+        });
+    }
+    async write() {
+        this._writePending = this._writePending
+            .catch(() => { })
+            .then(async () => {
+            const yaml = YAML.stringify(this.context.manifest);
+            await new File(this.path).write(yaml);
+            this.context.events.emit("manifest_stored", this.path.toString());
+        });
+        await this._writePending;
+    }
+    static async read(path) {
+        const content = await new File(path).text();
+        const manifest = YAML.parse(content);
+        const context = new Context(manifest);
+        const reflector = new FileContextReflect(context, path);
+        return { context, reflector };
+    }
+    static async create(path) {
+        const context = new Context();
+        const reflector = new FileContextReflect(context, path);
+        await reflector.write();
+        return { context, reflector };
+    }
+    static async load(path, options) {
+        if (options?.disabled) {
+            return { context: new Context(), reflector: null };
+        }
+        const exists = await new File(path).exists();
+        return exists
+            ? await FileContextReflect.read(path)
+            : await FileContextReflect.create(path);
+    }
+}

package/lib/esm/flow/context.d.ts

@@ -0,0 +1,43 @@
+import { EventManager } from "./utils/event-manager.js";
+import { JobId } from "./dto/job-id.js";
+import { Job } from "./dto/job.js";
+import { Manifest } from "./dto/manifest.js";
+import { CBApp } from "./dto/cb-app.js";
+export declare class Context {
+    readonly manifest: Manifest;
+    readonly events: EventManager<{
+        job_created: (jobId: string) => void;
+        job_loaded: (jobId: string) => void;
+        job_started: (jobId: string) => void;
+        job_done: (jobId: string) => void;
+        job_error: (jobId: string, error: any) => void;
+        job_retry: (jobId: string, retryCount: number) => void;
+        app_registered: (appId: string) => void;
+        manifest_stored: (path: string | URL) => void;
+        manifest_updated: () => void;
+    }>;
+    private appsDefinitions;
+    constructor(manifest?: Manifest);
+    configs: {
+        enableLoadJob: boolean;
+    };
+    jobs: {
+        get: (jobId: JobId) => Job;
+        load: (job: Job) => Job;
+        error: (jobOrId: JobId | Job, err: any) => void;
+        start: (jobOrId: string | Job, startedAt?: Date) => void;
+        done: (jobOrId: string | Job, output: Record<string, any>, completedAt?: Date) => void;
+    };
+    apps: {
+        register: (appId: string, inputDef: Record<string, any>, cb: CBApp, credentials?: {
+            name: string;
+        }[]) => void;
+        get: (appId: string) => Promise<{
+            inputDef: Record<string, any>;
+            cb: CBApp;
+            credentials?: {
+                name: string;
+            }[];
+        }>;
+    };
+}

package/lib/esm/flow/context.js

@@ -0,0 +1,104 @@
+import { processingJobContext } from "./worker_ctx.js";
+import { EventManager } from "./utils/event-manager.js";
+const DEFAULT_MAX_RETRIES = 3;
+export class Context {
+    manifest;
+    events = new EventManager();
+    appsDefinitions = new Map();
+    constructor(manifest = {
+        jobs: {},
+    }) {
+        this.manifest = manifest;
+    }
+    configs = {
+        enableLoadJob: true,
+    };
+    jobs = {
+        get: (jobId) => {
+            return this.manifest.jobs[jobId] ?? null;
+        },
+        load: (job) => {
+            const prevJob = this.jobs.get(job.jobId);
+            const storedJob = prevJob ?? job;
+            const isNew = !prevJob;
+            this.manifest.jobs[job.jobId] = storedJob;
+            if (isNew) {
+                this.events.emit("job_created", job.jobId);
+                this.events.emit("manifest_updated");
+            }
+            this.events.emit("job_loaded", job.jobId);
+            return storedJob;
+        },
+        error: (jobOrId, err) => {
+            const jobId = typeof jobOrId === "string" ? jobOrId : jobOrId.jobId;
+            if (!this.manifest.jobs[jobId]) {
+                if (typeof jobOrId === "string")
+                    throw new Error(`Job ${jobId} not found`);
+                this.manifest.jobs[jobId] = jobOrId;
+            }
+            const job = this.manifest.jobs[jobId];
+            const maxRetries = job?.maxRetries ?? DEFAULT_MAX_RETRIES;
+            job.error =
+                err instanceof Error
+                    ? { name: err.name, message: err.message, stack: err.stack }
+                    : err;
+            job.retries ??= 0;
+            this.events.emit("job_error", jobId, err);
+            if (job.retries < maxRetries) {
+                job.retries += 1;
+                this.events.emit("job_retry", jobId, job.retries);
+            }
+            this.events.emit("manifest_updated");
+        },
+        start: (jobOrId, startedAt = new Date()) => {
+            const jobId = typeof jobOrId === "string" ? jobOrId : jobOrId.jobId;
+            if (!this.manifest.jobs[jobId]) {
+                if (typeof jobOrId === "string")
+                    throw new Error(`Job ${jobId} not found`);
+                this.manifest.jobs[jobId] = jobOrId;
+            }
+            const job = this.manifest.jobs[jobId];
+            job.startedAt = startedAt;
+            job.workerId = processingJobContext.getStore()?.processingId ?? null;
+            this.events.emit("job_started", jobId);
+            this.events.emit("manifest_updated");
+        },
+        done: (jobOrId, output, completedAt = new Date()) => {
+            const jobId = typeof jobOrId === "string" ? jobOrId : jobOrId.jobId;
+            if (!this.manifest.jobs[jobId]) {
+                if (typeof jobOrId === "string")
+                    throw new Error(`Job ${jobId} not found`);
+                this.manifest.jobs[jobId] = jobOrId;
+            }
+            const job = this.manifest.jobs[jobId];
+            job.completedAt = completedAt;
+            job.output = output;
+            this.events.emit("job_done", jobId);
+            this.events.emit("manifest_updated");
+        },
+    };
+    apps = {
+        register: (appId, inputDef, cb, credentials) => {
+            this.appsDefinitions.set(appId, { inputDef, cb, credentials });
+            this.events.emit("app_registered", appId);
+        },
+        get: async (appId) => {
+            const app = this.appsDefinitions.get(appId);
+            if (app)
+                return app;
+            const waitRegister = Promise.withResolvers();
+            const listener = (registeredAppId) => {
+                if (registeredAppId === appId) {
+                    waitRegister.resolve(true);
+                    this.events.removeEventListener("app_registered", listener);
+                }
+            };
+            this.events.addEventListener("app_registered", listener);
+            await waitRegister.promise;
+            const registeredApp = this.appsDefinitions.get(appId);
+            if (!registeredApp)
+                throw new Error(`App ${appId} not found after waiting`);
+            return registeredApp;
+        },
+    };
+}

package/lib/esm/flow/current.d.ts

@@ -0,0 +1,4 @@
+import { Context } from "./context.js";
+export declare const currentContext: {
+    current: Context | null;
+};

package/lib/esm/flow/current.js

@@ -0,0 +1 @@
+export const currentContext = { current: null };

package/lib/esm/flow/dto/cb-app.d.ts

@@ -0,0 +1,2 @@
+import { Job } from "./job.js";
+export type CBApp = (params: Record<string, any>, job: Job) => Promise<any>;

package/lib/esm/flow/dto/cb-app.js

@@ -0,0 +1 @@
+export {};

package/lib/esm/flow/dto/job-id.d.ts

@@ -0,0 +1 @@
+export type JobId = string;

package/lib/esm/flow/dto/job-id.js

@@ -0,0 +1 @@
+export {};

package/lib/esm/flow/dto/job.d.ts

@@ -0,0 +1,18 @@
+import { JobId } from "./job-id.js";
+export type Job = {
+    parentJobId: JobId | null;
+    jobId: JobId;
+    appId: string;
+    credentials?: {
+        name: string;
+    }[];
+    startedAt: Date | null;
+    completedAt: Date | null;
+    input: Record<string, any>;
+    output: null | Record<string, any>;
+    error?: any;
+    retries?: number;
+    maxRetries?: number;
+    workerId: string | null;
+};
+export declare const isJob: (value: any) => value is Job;

package/lib/esm/flow/dto/job.js

@@ -0,0 +1,9 @@
+export const isJob = (value) => typeof value === "object" &&
+    value !== null &&
+    "jobId" in value &&
+    "appId" in value &&
+    "startedAt" in value &&
+    "completedAt" in value &&
+    "input" in value &&
+    "output" in value &&
+    "workerId" in value;

package/lib/esm/flow/dto/manifest.d.ts

@@ -0,0 +1,5 @@
+import { Job } from "./job";
+import { JobId } from "./job-id";
+export type Manifest = {
+    jobs: Record<JobId, Job>;
+};

package/lib/esm/flow/dto/manifest.js

@@ -0,0 +1 @@
+export {};