alepha 0.13.7 → 0.14.0

package/src/orm/providers/drivers/CloudflareD1Provider.ts

@@ -0,0 +1,164 @@
+import { $env, $hook, $inject, AlephaError, t } from "alepha";
+import { $logger } from "alepha/logger";
+import type { DrizzleD1Database } from "drizzle-orm/d1";
+import type { PgDatabase } from "drizzle-orm/pg-core";
+import { SqliteModelBuilder } from "../../services/SqliteModelBuilder.ts";
+import { DrizzleKitProvider } from "../DrizzleKitProvider.ts";
+import { DatabaseProvider, type SQLLike } from "./DatabaseProvider.ts";
+
+// ---------------------------------------------------------------------------------------------------------------------
+
+/**
+ * D1Database interface matching Cloudflare's D1 API.
+ */
+export interface D1Database {
+  prepare(query: string): D1PreparedStatement;
+  batch<T = unknown>(statements: D1PreparedStatement[]): Promise<T[]>;
+  exec(query: string): Promise<D1ExecResult>;
+  dump(): Promise<ArrayBuffer>;
+}
+
+export interface D1PreparedStatement {
+  bind(...values: unknown[]): D1PreparedStatement;
+  first<T = unknown>(colName?: string): Promise<T | null>;
+  run(): Promise<D1Result>;
+  all<T = unknown>(): Promise<D1Result<T>>;
+  raw<T = unknown>(): Promise<T[]>;
+}
+
+export interface D1Result<T = unknown> {
+  results: T[];
+  success: boolean;
+  meta: {
+    duration: number;
+    changes: number;
+    last_row_id: number;
+    served_by: string;
+    internal_stats: unknown;
+  };
+}
+
+export interface D1ExecResult {
+  count: number;
+  duration: number;
+}
+
+// ---------------------------------------------------------------------------------------------------------------------
+
+/**
+ * Cloudflare D1 SQLite provider using Drizzle ORM.
+ *
+ * This provider requires a D1 binding to be set via `cloudflareD1Options` before starting.
+ * The binding is typically obtained from the Cloudflare Workers environment.
+ *
+ * @example
+ * ```ts
+ * // In your Cloudflare Worker
+ * alepha.set(cloudflareD1Options, { binding: env.DB });
+ * ```
+ */
+export class CloudflareD1Provider extends DatabaseProvider {
+  protected readonly kit = $inject(DrizzleKitProvider);
+  protected readonly log = $logger();
+  protected readonly builder = $inject(SqliteModelBuilder);
+  protected readonly env = $env(
+    t.object({
+      DATABASE_URL: t.string({
+        description: "Expect to be 'cloudflare-d1://name:id'",
+      }),
+    }),
+  );
+
+  protected d1?: D1Database;
+  protected drizzleDb?: DrizzleD1Database;
+
+  public get name() {
+    return "d1";
+  }
+
+  public override readonly dialect = "sqlite";
+
+  public override get url(): string {
+    return this.env.DATABASE_URL;
+  }
+
+  public override get db(): PgDatabase<any> {
+    if (!this.drizzleDb) {
+      throw new AlephaError("D1 database not initialized");
+    }
+
+    return this.drizzleDb as unknown as PgDatabase<any>;
+  }
+
+  public override async execute(
+    query: SQLLike,
+  ): Promise<Array<Record<string, unknown>>> {
+    const { rows } = await (this.db as any).run(query);
+    return rows;
+  }
+
+  protected readonly onStart = $hook({
+    on: "start",
+    handler: async () => {
+      const [bindingName] = this.env.DATABASE_URL.replace(
+        "cloudflare-d1://",
+        "",
+      ).split(":");
+      const cloudflareEnv = this.alepha.store.get("cloudflare.env" as any);
+      if (!cloudflareEnv) {
+        throw new AlephaError(
+          "Cloudflare Workers environment not found in Alepha store under 'cloudflare.env'.",
+        );
+      }
+
+      const binding = cloudflareEnv[bindingName] as D1Database;
+      if (!binding) {
+        throw new AlephaError(
+          `D1 binding '${bindingName}' not found in Cloudflare Workers environment.`,
+        );
+      }
+
+      this.d1 = binding;
+
+      // Dynamic import to avoid crashes when not on Cloudflare
+      const { drizzle } = await import("drizzle-orm/d1");
+
+      this.drizzleDb = drizzle(this.d1) as DrizzleD1Database;
+
+      await this.migrateDatabase();
+
+      this.log.info("Using Cloudflare D1 database");
+    },
+  });
+
+  protected async executeMigrations(migrationsFolder: string): Promise<void> {
+    // Dynamic import for D1 migrator
+    const { migrate } = await import("drizzle-orm/d1/migrator");
+    await migrate(this.db as any, { migrationsFolder });
+  }
+
+  /**
+   * Override development migration to skip sync (not supported on D1).
+   * D1 requires proper migrations to be applied.
+   */
+  protected override async runDevelopmentMigration(
+    migrationsFolder: string,
+  ): Promise<void> {
+    await this.executeMigrations(migrationsFolder);
+  }
+
+  /**
+   * Override test migration to run migrations instead of sync.
+   * D1 doesn't support schema synchronization.
+   */
+  protected override async runTestMigration(): Promise<void> {
+    const migrationsFolder = this.getMigrationsFolder();
+    try {
+      await this.executeMigrations(migrationsFolder);
+    } catch {
+      this.log.warn(
+        "D1 migrations failed in test environment - ensure migrations exist",
+      );
+    }
+  }
+}

package/src/orm/providers/drivers/NodeSqliteProvider.ts

@@ -145,7 +145,9 @@ export class NodeSqliteProvider extends DatabaseProvider {
     on: "start",
     handler: async () => {
       const { DatabaseSync } = await import("node:sqlite");
-      const filepath = this.url.replace("sqlite://", "");
+
+      const filepath = this.url.replace("sqlite://", "").replace("sqlite:", "");
+
       if (filepath !== ":memory:" && filepath !== "") {
         const dirname = filepath.split("/").slice(0, -1).join("/");
         if (dirname) {

package/src/queue/{index.ts → core/index.ts}

@@ -7,7 +7,6 @@ import { WorkerProvider } from "./providers/WorkerProvider.ts";
 
 // ---------------------------------------------------------------------------------------------------------------------
 
-export * from "./interfaces/QueueJob.ts";
 export * from "./primitives/$consumer.ts";
 export * from "./primitives/$queue.ts";
 export * from "./providers/MemoryQueueProvider.ts";
@@ -17,9 +16,9 @@ export * from "./providers/WorkerProvider.ts";
 // ---------------------------------------------------------------------------------------------------------------------
 
 /**
- * Provides asynchronous message queuing and processing capabilities through declarative queue primitives.
+ * Provides asynchronous message queuing and processing capabilities through declarative queue descriptors.
  *
- * The queue module enables reliable background job processing and message passing using the `$queue` primitive
+ * The queue module enables reliable background job processing and message passing using the `$queue` descriptor
  * on class properties. It supports schema validation, automatic retries, and multiple queue backends for
  * building scalable, decoupled applications with robust error handling.
  *

package/src/queue/{primitives → core/primitives}/$queue.ts

@@ -1,4 +1,5 @@
 import {
+  $inject,
   createPrimitive,
   KIND,
   Primitive,
@@ -7,12 +8,9 @@ import {
   type TSchema,
 } from "alepha";
 import { $logger } from "alepha/logger";
-import type {
-  QueueAddJobOptions,
-  QueueJobBackoff,
-} from "../interfaces/QueueJob.ts";
 import { MemoryQueueProvider } from "../providers/MemoryQueueProvider.ts";
 import { QueueProvider } from "../providers/QueueProvider.ts";
+import { WorkerProvider } from "../providers/WorkerProvider.ts";
 
 /**
  * Creates a queue primitive for asynchronous message processing with background workers.
@@ -256,83 +254,6 @@ export interface QueuePrimitiveOptions<T extends TSchema> {
    * ```
    */
   handler?: (message: QueueMessage<T>) => Promise<void>;
-
-  // ===========================================
-  // Job Options (for crash recovery and retries)
-  // ===========================================
-
-  /**
-   * Maximum number of processing attempts before the job is marked as failed.
-   * Includes the initial attempt.
-   *
-   * Set this to enable automatic retries on failure.
-   *
-   * @default 1 (no retries)
-   * @example 3 // Allows 2 retries after initial failure
-   */
-  maxAttempts?: number;
-
-  /**
-   * Backoff configuration for retries.
-   * Controls the delay between retry attempts.
-   *
-   * @example
-   * ```ts
-   * backoff: {
-   *   type: "exponential",
-   *   delay: 1000,      // Initial delay: 1 second
-   *   maxDelay: 60000   // Maximum delay: 1 minute
-   * }
-   * ```
-   */
-  backoff?: QueueJobBackoff;
-
-  /**
-   * Maximum time in milliseconds a job can be processed before it's considered stalled.
-   * If the worker doesn't complete or extend the lock within this time, the job
-   * can be picked up by another worker.
-   *
-   * Increase this for long-running jobs.
-   *
-   * @default 30000 (30 seconds)
-   */
-  lockDuration?: number;
-
-  /**
-   * Automatically remove jobs when they complete successfully.
-   * - `true`: Remove immediately after completion
-   * - `false`: Keep in completed list (default)
-   * - `number`: Keep this many most recent completed jobs, remove older ones
-   *
-   * @default false
-   * @example
-   * ```ts
-   * // Remove immediately after completion
-   * removeOnComplete: true
-   *
-   * // Keep only the last 100 completed jobs
-   * removeOnComplete: 100
-   * ```
-   */
-  removeOnComplete?: boolean | number;
-
-  /**
-   * Automatically remove jobs when they fail permanently (after all retries exhausted).
-   * - `true`: Remove immediately after failure
-   * - `false`: Keep in failed list (default)
-   * - `number`: Keep this many most recent failed jobs, remove older ones
-   *
-   * @default false
-   * @example
-   * ```ts
-   * // Remove immediately after failure
-   * removeOnFail: true
-   *
-   * // Keep only the last 50 failed jobs for debugging
-   * removeOnFail: 50
-   * ```
-   */
-  removeOnFail?: boolean | number;
 }
 
 // ---------------------------------------------------------------------------------------------------------------------
@@ -341,90 +262,24 @@ export class QueuePrimitive<T extends TSchema> extends Primitive<
   QueuePrimitiveOptions<T>
 > {
   protected readonly log = $logger();
+  protected readonly workerProvider = $inject(WorkerProvider);
   public readonly provider = this.$provider();
 
-  /**
-   * Push one or more payloads to the queue for background processing.
-   *
-   * Jobs will be processed with crash recovery, retries (if configured),
-   * and proper lifecycle management.
-   *
-   * @param payloads - One or more payloads to queue
-   */
-  public async push(...payloads: Array<Static<T>>): Promise<void>;
-  /**
-   * Push a payload to the queue with specific options.
-   *
-   * @param payload - The payload to queue
-   * @param options - Job options (priority, delay)
-   */
-  public async push(
-    payload: Static<T>,
-    options: QueueAddJobOptions,
-  ): Promise<void>;
-
-  public async push(
-    payloadOrFirst: Static<T>,
-    optionsOrSecond?: QueueAddJobOptions | Static<T>,
-    ...rest: Array<Static<T>>
-  ): Promise<void> {
-    // Check if second argument is options object
-    const isOptions =
-      optionsOrSecond != null &&
-      typeof optionsOrSecond === "object" &&
-      ("priority" in optionsOrSecond || "delay" in optionsOrSecond);
-
-    if (isOptions) {
-      // Single payload with options
-      const payload = this.alepha.codec.decode(
-        this.options.schema,
-        payloadOrFirst,
-      );
-      await this.provider.addJob(this.name, payload, {
-        ...this.getDefaultJobOptions(),
-        priority: (optionsOrSecond as QueueAddJobOptions).priority,
-        delay: (optionsOrSecond as QueueAddJobOptions).delay,
-      });
-      this.log.debug(`Pushed job to queue ${this.name}`, {
-        payload,
-        options: optionsOrSecond,
-      });
-    } else {
-      // Multiple payloads without per-job options
-      const payloads =
-        optionsOrSecond != null
-          ? [payloadOrFirst, optionsOrSecond as Static<T>, ...rest]
-          : [payloadOrFirst, ...rest];
+  public async push(...payloads: Array<Static<T>>) {
+    await Promise.all(
+      payloads.map((payload) =>
+        this.provider.push(
+          this.name,
+          JSON.stringify({
+            headers: {},
+            payload: this.alepha.codec.decode(this.options.schema, payload),
+          }),
+        ),
+      ),
+    );
 
-      await Promise.all(
-        payloads.map((p) => {
-          const payload = this.alepha.codec.decode(this.options.schema, p);
-          return this.provider.addJob(
-            this.name,
-            payload,
-            this.getDefaultJobOptions(),
-          );
-        }),
-      );
-
-      this.log.debug(
-        `Pushed ${payloads.length} job(s) to queue ${this.name}`,
-        payloads,
-      );
-    }
-  }
-
-  /**
-   * Get default job options from primitive configuration.
-   */
-  protected getDefaultJobOptions() {
-    return {
-      maxAttempts: this.options.maxAttempts,
-      backoff: this.options.backoff,
-      lockDuration: this.options.lockDuration,
-      removeOnComplete: this.options.removeOnComplete,
-      removeOnFail: this.options.removeOnFail,
-    };
+    this.log.debug(`Pushed to queue ${this.name}`, payloads);
+    this.workerProvider.wakeUp();
   }
 
   public get name() {

package/src/queue/core/providers/MemoryQueueProvider.ts

@@ -0,0 +1,19 @@
+import { $logger } from "alepha/logger";
+import type { QueueProvider } from "./QueueProvider.ts";
+
+export class MemoryQueueProvider implements QueueProvider {
+  protected readonly log = $logger();
+  protected queueList: Record<string, string[]> = {};
+
+  public async push(queue: string, ...messages: string[]): Promise<void> {
+    if (this.queueList[queue] == null) {
+      this.queueList[queue] = [];
+    }
+
+    this.queueList[queue].push(...messages);
+  }
+
+  public async pop(queue: string): Promise<string | undefined> {
+    return this.queueList[queue]?.shift();
+  }
+}

package/src/queue/core/providers/QueueProvider.ts

@@ -0,0 +1,23 @@
+/**
+ * Minimalist Queue interface.
+ *
+ * Will be probably enhanced in the future to support more advanced features. But for now, it's enough!
+ */
+export abstract class QueueProvider {
+  /**
+   * Push a message to the queue.
+   *
+   * @param queue Name of the queue to push the message to.
+   * @param message String message to be pushed to the queue. Buffer messages are not supported for now.
+   */
+  public abstract push(queue: string, message: string): Promise<void>;
+
+  /**
+   * Pop a message from the queue.
+   *
+   * @param queue Name of the queue to pop the message from.
+   *
+   * @returns The message popped or `undefined` if the queue is empty.
+   */
+  public abstract pop(queue: string): Promise<string | undefined>;
+}

package/src/queue/core/providers/WorkerProvider.ts

@@ -0,0 +1,244 @@
+import {
+  $env,
+  $hook,
+  $inject,
+  Alepha,
+  type Static,
+  type TSchema,
+  t,
+} from "alepha";
+import { DateTimeProvider } from "alepha/datetime";
+import { $logger } from "alepha/logger";
+import { $consumer } from "../primitives/$consumer.ts";
+import {
+  $queue,
+  type QueueMessage,
+  type QueuePrimitive,
+} from "../primitives/$queue.ts";
+import { QueueProvider } from "./QueueProvider.ts";
+
+const envSchema = t.object({
+  /**
+   * The interval in milliseconds to wait before checking for new messages.
+   */
+  QUEUE_WORKER_INTERVAL: t.integer({
+    default: 1000,
+  }),
+  /**
+   * The maximum interval in milliseconds to wait before checking for new messages.
+   */
+  QUEUE_WORKER_MAX_INTERVAL: t.integer({
+    default: 32000,
+  }),
+  /**
+   * The number of workers to run concurrently. Defaults to 1.
+   * Useful only if you are doing a lot of I/O.
+   */
+  QUEUE_WORKER_CONCURRENCY: t.integer({
+    default: 1,
+  }),
+});
+
+declare module "alepha" {
+  interface Env extends Partial<Static<typeof envSchema>> {}
+}
+
+export class WorkerProvider {
+  protected readonly log = $logger();
+  protected readonly env = $env(envSchema);
+  protected readonly alepha = $inject(Alepha);
+  protected readonly queueProvider = $inject(QueueProvider);
+  protected readonly dateTimeProvider = $inject(DateTimeProvider);
+
+  protected workerPromises: Array<Promise<void>> = [];
+  protected workersRunning = 0;
+  protected abortController = new AbortController();
+  protected workerIntervals: Record<number, number> = {};
+  protected consumers: Array<Consumer> = [];
+
+  public get isRunning(): boolean {
+    return this.workersRunning > 0;
+  }
+
+  protected readonly start = $hook({
+    on: "start",
+    priority: "last",
+    handler: () => {
+      for (const queue of this.alepha.primitives($queue)) {
+        const handler = queue.options.handler;
+        if (handler) {
+          this.consumers.push({
+            handler,
+            queue,
+          });
+        }
+      }
+
+      for (const consumer of this.alepha.primitives($consumer)) {
+        this.consumers.push(consumer.options);
+      }
+
+      if (this.consumers.length > 0) {
+        this.startWorkers();
+        this.log.debug(
+          `Watching for ${this.consumers.length} queue${this.consumers.length > 1 ? "s" : ""} with ${this.env.QUEUE_WORKER_CONCURRENCY} worker${
+            this.env.QUEUE_WORKER_CONCURRENCY > 1 ? "s" : ""
+          }.`,
+        );
+      }
+    },
+  });
+
+  // -------------------------------------------------------------------------------------------------------------------
+
+  // Engine part - this is the part that will run the workers and process the messages
+
+  /**
+   * Start the workers.
+   * This method will create an endless loop that will check for new messages!
+   */
+  protected startWorkers(): void {
+    const workerToStart =
+      this.env.QUEUE_WORKER_CONCURRENCY - this.workersRunning;
+
+    for (let i = 0; i < workerToStart; i++) {
+      this.workersRunning += 1;
+      this.log.debug(`Starting worker n-${i}`);
+
+      const workerLoop = async () => {
+        while (this.workersRunning > 0) {
+          this.log.trace(`Worker n-${i} is checking for new messages`);
+          const next = await this.getNextMessage();
+          if (next) {
+            this.workerIntervals[i] = 0;
+            await this.processMessage(next);
+          } else {
+            await this.waitForNextMessage(i);
+          }
+        }
+        this.log.info(`Worker n-${i} has stopped`);
+        // Only decrement if we're not already at 0 (shutdown case)
+        if (this.workersRunning > 0) {
+          this.workersRunning -= 1;
+        }
+      };
+
+      this.workerPromises.push(
+        workerLoop().catch((e) => {
+          this.log.error(`Worker n-${i} has crashed`, e);
+          // Always decrement on crash, regardless of shutdown state
+          this.workersRunning -= 1;
+        }),
+      );
+    }
+  }
+
+  protected readonly stop = $hook({
+    on: "stop",
+    handler: async () => {
+      if (this.consumers.length > 0) {
+        await this.stopWorkers();
+      }
+    },
+  });
+
+  /**
+   * Wait for the next message, where `n` is the worker number.
+   *
+   * This method will wait for a certain amount of time, increasing the wait time again if no message is found.
+   */
+  protected async waitForNextMessage(n: number): Promise<void> {
+    const intervals = this.workerIntervals;
+    const milliseconds = intervals[n] || this.env.QUEUE_WORKER_INTERVAL;
+
+    this.log.trace(`Worker n-${n} is waiting for ${milliseconds}ms.`);
+
+    if (this.abortController.signal.aborted) {
+      this.log.warn(`Worker n-${n} aborted.`);
+      return;
+    }
+
+    await this.dateTimeProvider.wait(milliseconds, {
+      signal: this.abortController.signal,
+    });
+
+    if (intervals[n]) {
+      if (intervals[n] < this.env.QUEUE_WORKER_MAX_INTERVAL) {
+        intervals[n] = intervals[n] * 2;
+      }
+    } else {
+      intervals[n] = milliseconds;
+    }
+  }
+
+  /**
+   * Get the next message.
+   */
+  protected async getNextMessage(): Promise<undefined | NextMessage> {
+    for (const consumer of this.consumers) {
+      const provider = consumer.queue.provider;
+      const message = await provider.pop(consumer.queue.name);
+      if (message) {
+        return { message, consumer };
+      }
+    }
+  }
+
+  /**
+   * Process a message from a queue.
+   */
+  protected async processMessage(response: {
+    message: any;
+    consumer: Consumer;
+  }) {
+    const { message, consumer } = response;
+
+    try {
+      const json = JSON.parse(message);
+      const payload = this.alepha.codec.decode(
+        consumer.queue.options.schema,
+        json.payload,
+      );
+      await this.alepha.context.run(() => consumer.handler({ payload }));
+    } catch (e) {
+      this.log.error("Failed to process message", e);
+    }
+  }
+
+  /**
+   * Stop the workers.
+   *
+   * This method will stop the workers and wait for them to finish processing.
+   */
+  protected async stopWorkers() {
+    this.workersRunning = 0;
+
+    this.log.trace("Stopping workers...");
+    this.abortController.abort();
+
+    this.log.trace("Waiting for workers to finish...");
+    await Promise.all(this.workerPromises);
+  }
+
+  /**
+   * Force the workers to get back to work.
+   */
+  public wakeUp(): void {
+    this.log.debug("Waking up workers...");
+    this.abortController.abort();
+    this.abortController = new AbortController();
+
+    // if no workers are running, start them, (should not happen, but just in case)
+    this.startWorkers();
+  }
+}
+
+export interface Consumer<T extends TSchema = TSchema> {
+  queue: QueuePrimitive<T>;
+  handler: (message: QueueMessage<T>) => Promise<void>;
+}
+
+export interface NextMessage {
+  consumer: Consumer;
+  message: string;
+}