akanjs 0.0.1 → 2.0.0-beta.0

package/service/predefinedAdaptor/solidQueue.adaptor.ts

@@ -0,0 +1,203 @@
+import type { Database, Statement } from "bun:sqlite";
+import { adapt } from "../adapt";
+import { type AkanJob, type AkanJobOptions, type AkanWorker, sendAkanIpc } from "../ipcTypes";
+import type { QueueAdaptor } from "./queue.adaptor";
+import { getSolidConfig, openSolidDatabase, type SolidConfig, type SolidEnv } from "./solidSqlite";
+
+type JobRow = {
+  id: string;
+  name: string;
+  payload: string;
+  attempts: number;
+  maxAttempts: number;
+  lockedUntil: number | null;
+};
+
+class SolidWorker implements AkanWorker {
+  #timer: Timer | null = null;
+  #running = false;
+  #closed = false;
+
+  constructor(
+    private readonly queue: SolidQueue,
+    private readonly name: string,
+    private readonly handler: (job: AkanJob) => Promise<void>,
+    pollIntervalMs: number,
+  ) {
+    this.#timer = setInterval(() => void this.pump(), pollIntervalMs);
+    void this.pump();
+  }
+
+  wake() {
+    void this.pump();
+  }
+
+  async pump() {
+    if (this.#closed || this.#running) return;
+    this.#running = true;
+    try {
+      while (!this.#closed) {
+        const job = this.queue.claimJob(this.name);
+        if (!job) return;
+        await this.queue.runJob(job, this.handler);
+      }
+    } finally {
+      this.#running = false;
+    }
+  }
+
+  async close() {
+    this.#closed = true;
+    if (this.#timer) clearInterval(this.#timer);
+    while (this.#running) await new Promise((resolve) => setTimeout(resolve, 25));
+  }
+}
+
+export class SolidQueue
+  extends adapt("solidQueue", ({ env }) => ({
+    config: env((env: SolidEnv) => getSolidConfig(env)),
+    queueName: env((env: SolidEnv) => `queue-${env.repoName}-${env.appName}-${env.environment}-${env.operationMode}`),
+    workerId: env((env: SolidEnv) => `${env.appName}-${process.env.AKAN_REPLICA_IDX ?? "0"}-${process.pid}`),
+  }))
+  implements QueueAdaptor
+{
+  #db!: Database;
+  #workers = new Map<string, SolidWorker>();
+  #claimStmt!: Statement;
+  readonly #messageHandler = (message: unknown) => {
+    if (!message || typeof message !== "object") return;
+    if ((message as { type?: string }).type === "queue.wake") this.wake((message as { name?: string }).name);
+  };
+
+  override async onInit() {
+    this.#db = await openSolidDatabase(this.config as Required<SolidConfig>);
+    this.#db.run(
+      `CREATE TABLE IF NOT EXISTS "_akan_solid_jobs" (
+        "id" TEXT PRIMARY KEY NOT NULL,
+        "queue" TEXT NOT NULL,
+        "name" TEXT NOT NULL,
+        "payload" TEXT NOT NULL,
+        "status" TEXT NOT NULL,
+        "priority" INTEGER NOT NULL,
+        "runAt" INTEGER NOT NULL,
+        "attempts" INTEGER NOT NULL,
+        "maxAttempts" INTEGER NOT NULL,
+        "lockedBy" TEXT,
+        "lockedUntil" INTEGER,
+        "lastError" TEXT,
+        "createdAt" INTEGER NOT NULL,
+        "updatedAt" INTEGER NOT NULL
+      )`,
+    );
+    this.#db.run(
+      `CREATE INDEX IF NOT EXISTS "_akan_solid_jobs_claim_idx"
+       ON "_akan_solid_jobs" ("queue", "name", "status", "runAt", "priority", "createdAt")`,
+    );
+    this.#claimStmt = this.#db.query(
+      `SELECT "id", "name", "payload", "attempts", "maxAttempts", "lockedUntil"
+       FROM "_akan_solid_jobs"
+       WHERE "queue" = ?
+         AND "name" = ?
+         AND "runAt" <= ?
+         AND (
+          "status" = 'pending'
+          OR ("status" = 'running' AND "lockedUntil" IS NOT NULL AND "lockedUntil" <= ?)
+         )
+       ORDER BY "priority" DESC, "createdAt" ASC
+       LIMIT 1`,
+    );
+    process.on("message", this.#messageHandler);
+  }
+
+  override async onDestroy() {
+    process.off("message", this.#messageHandler);
+    await Promise.all([...this.#workers.values()].map((worker) => worker.close()));
+    this.#db?.run("PRAGMA wal_checkpoint(TRUNCATE)");
+    this.#db?.close();
+  }
+
+  registerProcessWorker(key: string, handler: (job: AkanJob) => Promise<void>): AkanWorker {
+    const worker = new SolidWorker(this, key, handler, this.config.queuePollIntervalMs);
+    this.#workers.set(key, worker);
+    return worker;
+  }
+
+  async registerProcessQueue(key: string, args: unknown[], jobOptions: AkanJobOptions = {}): Promise<AkanJob> {
+    const now = Date.now();
+    const id = Bun.randomUUIDv7();
+    const runAt = now + Number(jobOptions.delay ?? 0);
+    const maxAttempts = Math.max(1, Number(jobOptions.attempts ?? 1));
+    const payload = JSON.stringify({ args, opts: jobOptions });
+    this.#db
+      .query(
+        `INSERT INTO "_akan_solid_jobs" ("id", "queue", "name", "payload", "status", "priority", "runAt", "attempts", "maxAttempts", "createdAt", "updatedAt")
+         VALUES (?, ?, ?, ?, 'pending', ?, ?, 0, ?, ?, ?)`,
+      )
+      .run(id, this.queueName, key, payload, Number(jobOptions.priority ?? 0), runAt, maxAttempts, now, now);
+    sendAkanIpc({ type: "queue.enqueued", queue: this.queueName, name: key, jobId: id });
+    this.wake(key);
+    return { id, name: key, data: args, attemptsMade: 0, opts: jobOptions };
+  }
+
+  wake(name?: string) {
+    if (name) this.#workers.get(name)?.wake();
+    else for (const worker of this.#workers.values()) worker.wake();
+  }
+
+  claimJob(name: string): AkanJob | null {
+    const now = Date.now();
+    this.#db.run("BEGIN IMMEDIATE");
+    try {
+      const row = this.#claimStmt.get(this.queueName, name, now, now) as JobRow | null;
+      if (!row) {
+        this.#db.run("COMMIT");
+        return null;
+      }
+      const payload = JSON.parse(row.payload) as { args: unknown[]; opts?: AkanJobOptions };
+      this.#db
+        .query(
+          `UPDATE "_akan_solid_jobs"
+           SET "status" = 'running', "attempts" = "attempts" + 1, "lockedBy" = ?, "lockedUntil" = ?, "updatedAt" = ?
+           WHERE "id" = ?`,
+        )
+        .run(this.workerId, now + this.config.queueLeaseMs, now, row.id);
+      this.#db.run("COMMIT");
+      return { id: row.id, name: row.name, data: payload.args, attemptsMade: row.attempts + 1, opts: payload.opts };
+    } catch (error) {
+      this.#db.run("ROLLBACK");
+      throw error;
+    }
+  }
+
+  async runJob(job: AkanJob, handler: (job: AkanJob) => Promise<void>) {
+    try {
+      await handler({ ...job, data: job.data });
+      this.#db
+        .query(
+          `UPDATE "_akan_solid_jobs" SET "status" = 'completed', "lockedBy" = NULL, "lockedUntil" = NULL, "updatedAt" = ? WHERE "id" = ?`,
+        )
+        .run(Date.now(), job.id);
+    } catch (error) {
+      const message = error instanceof Error ? (error.stack ?? error.message) : String(error);
+      const row = this.#db
+        .query(`SELECT "attempts", "maxAttempts" FROM "_akan_solid_jobs" WHERE "id" = ?`)
+        .get(job.id) as { attempts: number; maxAttempts: number } | null;
+      const failed = !row || row.attempts >= row.maxAttempts;
+      const delay = this.#getBackoffDelay(job.opts?.backoff);
+      this.#db
+        .query(
+          `UPDATE "_akan_solid_jobs"
+           SET "status" = ?, "runAt" = ?, "lockedBy" = NULL, "lockedUntil" = NULL, "lastError" = ?, "updatedAt" = ?
+           WHERE "id" = ?`,
+        )
+        .run(failed ? "failed" : "pending", Date.now() + delay, message, Date.now(), job.id);
+      if (failed) this.logger.error(`Solid job failed ${job.name}/${job.id}: ${message}`);
+    }
+  }
+
+  #getBackoffDelay(backoff: AkanJobOptions["backoff"]) {
+    if (!backoff) return 0;
+    if (typeof backoff === "number") return backoff;
+    return Number(backoff.delay ?? 0);
+  }
+}

package/service/predefinedAdaptor/solidSqlite.ts

@@ -0,0 +1,85 @@
+import { Database } from "bun:sqlite";
+import { mkdir } from "node:fs/promises";
+import path from "node:path";
+import type { BaseEnv, Dayjs } from "akanjs/base";
+import { resolveDefaultSqliteFile } from "./sqlitePath";
+
+export interface SolidConfig {
+  filePath?: string;
+  journalMode?: "WAL" | "DELETE" | "TRUNCATE" | "PERSIST" | "MEMORY" | "OFF";
+  busyTimeoutMs?: number;
+  synchronous?: "OFF" | "NORMAL" | "FULL" | "EXTRA";
+  cleanupIntervalMs?: number;
+  queuePollIntervalMs?: number;
+  queueLeaseMs?: number;
+}
+
+export interface SolidEnv extends BaseEnv {
+  workspaceRoot?: string;
+  solid?: SolidConfig;
+}
+
+export type SolidValueType = "string" | "number" | "buffer";
+
+const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+export const getSolidConfig = (env: SolidEnv): Required<SolidConfig> => {
+  const appName = env.appName ?? "akan";
+  const environment = env.environment ?? "local";
+  const defaultFile = resolveDefaultSqliteFile({
+    appName,
+    fileName: `${appName}-${environment}_solid.db`,
+    isProduction: process.env.NODE_ENV === "production",
+    workspaceRoot: env.workspaceRoot,
+  });
+  return {
+    filePath: env.solid?.filePath ?? process.env.AKAN_SOLID_DB_PATH ?? defaultFile,
+    journalMode: env.solid?.journalMode ?? "WAL",
+    busyTimeoutMs: env.solid?.busyTimeoutMs ?? 5000,
+    synchronous: env.solid?.synchronous ?? "NORMAL",
+    cleanupIntervalMs: env.solid?.cleanupIntervalMs ?? 60_000,
+    queuePollIntervalMs: env.solid?.queuePollIntervalMs ?? 2000,
+    queueLeaseMs: env.solid?.queueLeaseMs ?? 30_000,
+  };
+};
+
+export const openSolidDatabase = async (config: Required<SolidConfig>) => {
+  await mkdir(path.dirname(config.filePath), { recursive: true });
+  let lastError: unknown;
+  for (let attempt = 0; attempt < 8; attempt++) {
+    try {
+      const db = new Database(config.filePath, { strict: true, create: true });
+      db.run(`PRAGMA busy_timeout = ${config.busyTimeoutMs}`);
+      db.run(`PRAGMA journal_mode = ${config.journalMode}`);
+      db.run(`PRAGMA synchronous = ${config.synchronous}`);
+      return db;
+    } catch (error) {
+      lastError = error;
+      if (!String(error instanceof Error ? error.message : error).includes("locked")) throw error;
+      await wait(50 * (attempt + 1));
+    }
+  }
+  throw lastError instanceof Error ? lastError : new Error(String(lastError));
+};
+
+export const encodeSolidValue = (value: string | number | Buffer): { type: SolidValueType; value: string | Buffer } => {
+  if (Buffer.isBuffer(value)) return { type: "buffer", value };
+  if (typeof value === "number") return { type: "number", value: String(value) };
+  return { type: "string", value };
+};
+
+export const decodeSolidValue = <T extends string | number | Buffer>(
+  type: SolidValueType,
+  value: string | Buffer | null,
+): T | undefined => {
+  if (value === null) return undefined;
+  if (type === "buffer") return Buffer.isBuffer(value) ? (value as T) : (Buffer.from(value) as T);
+  if (type === "number") return Number(value) as T;
+  return String(value) as T;
+};
+
+export const toEpochMs = (value?: Dayjs | number | null) => {
+  if (value == null) return null;
+  if (typeof value === "number") return value;
+  return value.toDate().getTime();
+};

package/service/predefinedAdaptor/sqlitePath.ts

@@ -0,0 +1,20 @@
+import path from "node:path";
+
+interface ResolveDefaultSqliteFileOptions {
+  appName: string;
+  fileName: string;
+  isProduction: boolean;
+  workspaceRoot?: string;
+}
+
+export const resolveDefaultSqliteFile = ({
+  appName,
+  fileName,
+  isProduction,
+  workspaceRoot,
+}: ResolveDefaultSqliteFileOptions) => {
+  const sqliteDir = process.env.AKAN_SQLITE_DIR;
+  if (sqliteDir) return path.join(sqliteDir, fileName);
+  if (isProduction) return path.join(process.cwd(), "sqlite", fileName);
+  return path.join(workspaceRoot ?? process.cwd(), "local", "apps", appName, fileName);
+};

package/service/predefinedAdaptor/storage.adaptor.ts

@@ -0,0 +1,121 @@
+import { renameSync } from "node:fs";
+import type { BaseEnv } from "akanjs/base";
+import { adapt } from "../adapt";
+
+export interface DownloadRequest {
+  path: string;
+  localPath: string;
+  renamePath?: string;
+}
+export interface LocalFilePath {
+  localPath: string;
+}
+export interface UploadRequest {
+  path: string;
+  localPath: string;
+  meta?: { [key: string]: string };
+  rename?: string;
+  host?: string;
+}
+export interface CopyRequest {
+  bucket: string;
+  copyPath: string;
+  pastePath: string;
+  filename: string;
+  host?: string;
+}
+export interface UploadFromStreamRequest {
+  path: string;
+  body: ReadableStream;
+  mimetype: string;
+  root?: string;
+  updateProgress: (progress: { loaded?: number; total?: number; part?: number }) => void;
+  uploadSuccess: (url: string) => void;
+}
+export interface UploadProgress {
+  loaded?: number;
+  total?: number;
+  part?: number;
+}
+
+export interface StorageAdaptor {
+  readData(path: string): Promise<ReadableStream>;
+  readDataAsJson<T>(path: string): Promise<T>;
+  getDataList(prefix?: string): Promise<string[]>;
+  uploadDataFromLocal(request: UploadRequest): Promise<string>;
+  uploadDataFromStream(request: UploadFromStreamRequest): void;
+  saveData(request: DownloadRequest): Promise<LocalFilePath>;
+  copyData(request: CopyRequest): Promise<string>;
+  deleteData(url: string): Promise<boolean>;
+}
+
+export interface BlobStorageOptions extends BaseEnv {
+  blobStorage?: { baseDir?: string; urlPrefix?: string };
+}
+
+export class BlobStorage
+  extends adapt("blobStorage", ({ env }) => ({
+    root: env(
+      ({ appName, blobStorage = { baseDir: "local", urlPrefix: "/backend/localFile/getBlob" } }: BlobStorageOptions) =>
+        `${process.env.AKAN_WORKSPACE_ROOT ?? "."}/${blobStorage.baseDir}/${appName}/backend`,
+    ),
+    urlPrefix: env(
+      ({ blobStorage = { urlPrefix: "/backend/localFile/getBlob" } }: BlobStorageOptions) => blobStorage.urlPrefix,
+    ),
+  }))
+  implements StorageAdaptor
+{
+  #localPathToUrl(path: string) {
+    return `${this.urlPrefix}/${path}`;
+  }
+  async readData(path: string): Promise<ReadableStream> {
+    const filePath = `${this.root}/${path}`;
+    return Bun.file(filePath).stream();
+  }
+  async readDataAsJson<T>(path: string) {
+    const filePath = `${this.root}/${path}`;
+    return Bun.file(filePath).json() as T;
+  }
+  async getDataList(prefix?: string) {
+    const dir = `${this.root}${prefix ? `/${prefix}` : ""}`;
+    const paths = Array.from(new Bun.Glob("*").scanSync({ cwd: dir, onlyFiles: false }));
+    return paths.map((path) => this.#localPathToUrl(path));
+  }
+  async uploadDataFromLocal({ path, localPath, meta }: UploadRequest) {
+    const filePath = `${this.root}/${path}`;
+    await Bun.write(filePath, Bun.file(localPath));
+    if (meta) await Bun.write(`${filePath}.meta`, JSON.stringify(meta));
+    return this.#localPathToUrl(path);
+  }
+  async uploadDataFromStream({ path, body, mimetype, updateProgress, uploadSuccess }: UploadFromStreamRequest) {
+    const filePath = `${this.root}/${path}`;
+    try {
+      await Bun.write(filePath, new Response(body));
+      uploadSuccess(this.#localPathToUrl(path));
+    } catch (error) {
+      this.logger.error(error instanceof Error ? error.message : String(error));
+    }
+  }
+  async saveData({ path, localPath, renamePath }: DownloadRequest): Promise<LocalFilePath> {
+    const data = await this.readData(path);
+    await Bun.write(localPath, new Response(data));
+    if (renamePath) renameSync(localPath, renamePath);
+    return { localPath: renamePath ?? localPath };
+  }
+  async copyData({ copyPath, pastePath, host }: CopyRequest) {
+    await Bun.write(`${this.root}/${pastePath}`, Bun.file(`${this.root}/${copyPath}`));
+    return pastePath;
+  }
+  async deleteData(url: string) {
+    try {
+      const basePath = this.#localPathToUrl("");
+      if (!url.startsWith(basePath)) throw new Error("Invalid Base URL, Unable to delete data");
+      const path = url.replace(basePath, "");
+      await Bun.file(`${this.root}/${path}`).delete();
+      return true;
+    } catch (error) {
+      this.logger.error(error instanceof Error ? error.message : "Unknown error");
+      return false;
+    }
+  }
+}