@fragno-dev/db 0.2.2 → 0.3.0

package/src/fragments/internal-fragment.schema.ts

@@ -0,0 +1,51 @@
+import { schema, idColumn, column } from "../schema/create";
+
+// Constants for Fragno's internal settings table
+export const SETTINGS_TABLE_NAME = "fragno_db_settings" as const;
+// FIXME: In some places we simply use empty string "" as namespace, which is not correct.
+export const SETTINGS_NAMESPACE = "fragno-db-settings" as const;
+
+export const internalSchema = schema("fragno_internal", (s) => {
+  return s
+    .addTable(SETTINGS_TABLE_NAME, (t) => {
+      return t
+        .addColumn("id", idColumn())
+        .addColumn("key", column("string"))
+        .addColumn("value", column("string"))
+        .createIndex("unique_key", ["key"], { unique: true });
+    })
+    .addTable("fragno_hooks", (t) => {
+      return t
+        .addColumn("id", idColumn())
+        .addColumn("namespace", column("string"))
+        .addColumn("hookName", column("string"))
+        .addColumn("payload", column("json"))
+        .addColumn("status", column("string")) // "pending" | "processing" | "completed" | "failed"
+        .addColumn("attempts", column("integer").defaultTo(0))
+        .addColumn("maxAttempts", column("integer").defaultTo(5))
+        .addColumn("lastAttemptAt", column("timestamp").nullable())
+        .addColumn("nextRetryAt", column("timestamp").nullable())
+        .addColumn("error", column("string").nullable())
+        .addColumn(
+          "createdAt",
+          column("timestamp").defaultTo((b) => b.now()),
+        )
+        .addColumn("nonce", column("string"))
+        .createIndex("idx_namespace_status_retry", ["namespace", "status", "nextRetryAt"])
+        .createIndex("idx_nonce", ["nonce"]);
+    })
+    .addTable("fragno_db_outbox", (t) => {
+      return t
+        .addColumn("id", idColumn())
+        .addColumn("versionstamp", column("string"))
+        .addColumn("uowId", column("string"))
+        .addColumn("payload", column("json"))
+        .addColumn("refMap", column("json").nullable())
+        .addColumn(
+          "createdAt",
+          column("timestamp").defaultTo((b) => b.now()),
+        )
+        .createIndex("idx_outbox_versionstamp", ["versionstamp"], { unique: true })
+        .createIndex("idx_outbox_uow", ["uowId"]);
+    });
+});

package/src/fragments/internal-fragment.test.ts

@@ -2,16 +2,22 @@ import SQLite from "better-sqlite3";
 import { SqliteDialect } from "kysely";
 import { beforeAll, describe, expect, it } from "vitest";
 import { instantiate } from "@fragno-dev/core";
-import { internalFragmentDef, internalSchema, SETTINGS_NAMESPACE } from "./internal-fragment";
+import {
+  internalFragmentDef,
+  internalSchema,
+  SETTINGS_NAMESPACE,
+  getSchemaVersionFromDatabase,
+} from "./internal-fragment";
 import type { FragnoPublicConfigWithDatabase } from "../db-fragment-definition-builder";
-import { DrizzleAdapter } from "../adapters/drizzle/drizzle-adapter";
+import { SqlAdapter } from "../adapters/generic-sql/generic-sql-adapter";
 import { BetterSQLite3DriverConfig } from "../adapters/generic-sql/driver-config";
 import { ExponentialBackoffRetryPolicy, NoRetryPolicy } from "../query/unit-of-work/retry-policy";
+import { ConcurrencyConflictError } from "../query/unit-of-work/execute-unit-of-work";
 import type { FragnoId } from "../schema/create";
 
 describe("Internal Fragment", () => {
   let sqliteDatabase: SQLite.Database;
-  let adapter: DrizzleAdapter;
+  let adapter: SqlAdapter;
   let fragment: ReturnType<typeof instantiateFragment>;
 
   function instantiateFragment(options: FragnoPublicConfigWithDatabase) {
@@ -25,19 +31,20 @@ describe("Internal Fragment", () => {
       database: sqliteDatabase,
     });
 
-    adapter = new DrizzleAdapter({
+    adapter = new SqlAdapter({
       dialect,
       driverConfig: new BetterSQLite3DriverConfig(),
     });
 
     {
-      const migrations = adapter.prepareMigrations(internalSchema, "");
+      const migrations = adapter.prepareMigrations(internalSchema, null);
       await migrations.executeWithDriver(adapter.driver, 0);
     }
 
     // Instantiate fragment with shared database adapter
     const options: FragnoPublicConfigWithDatabase = {
       databaseAdapter: adapter,
+      databaseNamespace: null,
     };
 
     fragment = instantiateFragment(options);
@@ -151,7 +158,7 @@ describe("Internal Fragment", () => {
 
 describe("Hook Service", () => {
   let sqliteDatabase: SQLite.Database;
-  let adapter: DrizzleAdapter;
+  let adapter: SqlAdapter;
   let fragment: ReturnType<typeof instantiateFragment>;
 
   function instantiateFragment(options: FragnoPublicConfigWithDatabase) {
@@ -165,18 +172,19 @@ describe("Hook Service", () => {
       database: sqliteDatabase,
     });
 
-    adapter = new DrizzleAdapter({
+    adapter = new SqlAdapter({
       dialect,
       driverConfig: new BetterSQLite3DriverConfig(),
     });
 
     {
-      const migrations = adapter.prepareMigrations(internalSchema, "");
+      const migrations = adapter.prepareMigrations(internalSchema, null);
       await migrations.executeWithDriver(adapter.driver, 0);
     }
 
     const options: FragnoPublicConfigWithDatabase = {
       databaseAdapter: adapter,
+      databaseNamespace: null,
     };
 
     fragment = instantiateFragment(options);
@@ -282,6 +290,51 @@ describe("Hook Service", () => {
     expect(result?.lastAttemptAt).toBeInstanceOf(Date);
   });
 
+  it("should reject marking completed with a stale id", async () => {
+    const namespace = "complete-stale";
+    const nonce = "test-nonce-complete-stale";
+    let staleId!: FragnoId;
+
+    await fragment.inContext(async function () {
+      const createdId = await this.handlerTx()
+        .mutate(({ forSchema }) => {
+          const uow = forSchema(internalSchema);
+          return uow.create("fragno_hooks", {
+            namespace,
+            hookName: "onCompleteStale",
+            payload: { test: "data" },
+            status: "pending",
+            attempts: 0,
+            maxAttempts: 5,
+            lastAttemptAt: null,
+            nextRetryAt: null,
+            error: null,
+            nonce,
+          });
+        })
+        .execute();
+      staleId = createdId;
+    });
+
+    await fragment.inContext(async function () {
+      await this.handlerTx()
+        .withServiceCalls(
+          () => [fragment.services.hookService.claimPendingHookEvents(namespace)] as const,
+        )
+        .execute();
+    });
+
+    await expect(
+      fragment.inContext(async function () {
+        await this.handlerTx()
+          .withServiceCalls(
+            () => [fragment.services.hookService.markHookCompleted(staleId)] as const,
+          )
+          .execute();
+      }),
+    ).rejects.toThrow(ConcurrencyConflictError);
+  });
+
   it("should mark a hook event as processing", async () => {
     const nonce = "test-nonce-3";
     let eventId: FragnoId;
@@ -477,6 +530,49 @@ describe("Hook Service", () => {
     expect(staleEvent?.attempts).toBe(1);
   });
 
+  it("should detect conflicts when requeueing after another update in the same transaction", async () => {
+    const namespace = "requeue-conflict";
+    const nonce = "test-nonce-requeue-conflict";
+    let eventId!: FragnoId;
+
+    const staleBefore = new Date(Date.now() - 60_000);
+    const lastAttemptAt = new Date(Date.now() - 120_000);
+
+    await fragment.inContext(async function () {
+      eventId = await this.handlerTx()
+        .mutate(({ forSchema }) => {
+          const uow = forSchema(internalSchema);
+          return uow.create("fragno_hooks", {
+            namespace,
+            hookName: "onRequeueConflict",
+            payload: { test: "requeue" },
+            status: "processing",
+            attempts: 0,
+            maxAttempts: 5,
+            lastAttemptAt,
+            nextRetryAt: null,
+            error: null,
+            nonce,
+          });
+        })
+        .execute();
+    });
+
+    await expect(
+      fragment.inContext(async function () {
+        await this.handlerTx({ retryPolicy: new NoRetryPolicy() })
+          .withServiceCalls(
+            () =>
+              [
+                fragment.services.hookService.markHookProcessing(eventId),
+                fragment.services.hookService.requeueStuckProcessingHooks(namespace, staleBefore),
+              ] as const,
+          )
+          .execute();
+      }),
+    ).rejects.toThrow(ConcurrencyConflictError);
+  });
+
   it("should not retrieve events from different namespace", async () => {
     const nonce = "test-nonce-7";
 
@@ -552,4 +648,358 @@ describe("Hook Service", () => {
     const futureEvent = events.find((e) => e.id.externalId === eventId.externalId);
     expect(futureEvent).toBeUndefined();
   });
+
+  it("should claim only ready pending events and mark them processing", async () => {
+    const namespace = "claim-ready";
+    const pastTime = new Date(Date.now() - 10000);
+    const futureTime = new Date(Date.now() + 60000);
+
+    let nullRetryId!: FragnoId;
+    let pastRetryId!: FragnoId;
+    let futureRetryId!: FragnoId;
+
+    await fragment.inContext(async function () {
+      await this.handlerTx()
+        .mutate(({ forSchema }) => {
+          const uow = forSchema(internalSchema);
+          nullRetryId = uow.create("fragno_hooks", {
+            namespace,
+            hookName: "onNullRetry",
+            payload: { test: "null" },
+            status: "pending",
+            attempts: 0,
+            maxAttempts: 5,
+            lastAttemptAt: null,
+            nextRetryAt: null,
+            error: null,
+            nonce: "test-nonce-claim-null",
+          });
+          pastRetryId = uow.create("fragno_hooks", {
+            namespace,
+            hookName: "onPastRetry",
+            payload: { test: "past" },
+            status: "pending",
+            attempts: 1,
+            maxAttempts: 5,
+            lastAttemptAt: pastTime,
+            nextRetryAt: pastTime,
+            error: "Previous error",
+            nonce: "test-nonce-claim-past",
+          });
+          futureRetryId = uow.create("fragno_hooks", {
+            namespace,
+            hookName: "onFutureRetry",
+            payload: { test: "future" },
+            status: "pending",
+            attempts: 1,
+            maxAttempts: 5,
+            lastAttemptAt: new Date(),
+            nextRetryAt: futureTime,
+            error: "Previous error",
+            nonce: "test-nonce-claim-future",
+          });
+        })
+        .execute();
+    });
+
+    const claimed = await fragment.inContext(async function () {
+      return await this.handlerTx()
+        .withServiceCalls(
+          () => [fragment.services.hookService.claimPendingHookEvents(namespace)] as const,
+        )
+        .transform(({ serviceResult: [result] }) => result)
+        .execute();
+    });
+
+    expect(claimed).toHaveLength(2);
+    const claimedIds = new Set(claimed.map((event) => event.id.externalId));
+    expect(claimedIds.has(nullRetryId.externalId)).toBe(true);
+    expect(claimedIds.has(pastRetryId.externalId)).toBe(true);
+    expect(claimedIds.has(futureRetryId.externalId)).toBe(false);
+
+    const [nullEvent, pastEvent, futureEvent] = await fragment.inContext(async function () {
+      return await this.handlerTx()
+        .withServiceCalls(
+          () =>
+            [
+              fragment.services.hookService.getHookById(nullRetryId),
+              fragment.services.hookService.getHookById(pastRetryId),
+              fragment.services.hookService.getHookById(futureRetryId),
+            ] as const,
+        )
+        .transform(({ serviceResult: [nullResult, pastResult, futureResult] }) => [
+          nullResult,
+          pastResult,
+          futureResult,
+        ])
+        .execute();
+    });
+
+    expect(nullEvent?.status).toBe("processing");
+    expect(nullEvent?.lastAttemptAt).toBeInstanceOf(Date);
+    expect(pastEvent?.status).toBe("processing");
+    expect(pastEvent?.lastAttemptAt).toBeInstanceOf(Date);
+    expect(futureEvent?.status).toBe("pending");
+  });
+
+  it("should return claimed ids with incremented versions", async () => {
+    const namespace = "claim-version";
+    const nonce = "test-nonce-claim-version";
+    let createdId!: FragnoId;
+
+    await fragment.inContext(async function () {
+      createdId = await this.handlerTx()
+        .mutate(({ forSchema }) => {
+          const uow = forSchema(internalSchema);
+          return uow.create("fragno_hooks", {
+            namespace,
+            hookName: "onClaimVersion",
+            payload: { test: "version" },
+            status: "pending",
+            attempts: 0,
+            maxAttempts: 5,
+            lastAttemptAt: null,
+            nextRetryAt: null,
+            error: null,
+            nonce,
+          });
+        })
+        .execute();
+    });
+
+    const claimed = await fragment.inContext(async function () {
+      return await this.handlerTx()
+        .withServiceCalls(
+          () => [fragment.services.hookService.claimPendingHookEvents(namespace)] as const,
+        )
+        .transform(({ serviceResult: [result] }) => result)
+        .execute();
+    });
+
+    expect(claimed).toHaveLength(1);
+    expect(claimed[0]?.id.externalId).toBe(createdId.externalId);
+    expect(claimed[0]?.id.version).toBe(createdId.version + 1);
+  });
+
+  it("should return now when pending hooks have no nextRetryAt", async () => {
+    const namespace = "wake-now";
+
+    await fragment.inContext(async function () {
+      await this.handlerTx()
+        .mutate(({ forSchema }) => {
+          const uow = forSchema(internalSchema);
+          uow.create("fragno_hooks", {
+            namespace,
+            hookName: "onImmediate",
+            payload: { test: "now" },
+            status: "pending",
+            attempts: 0,
+            maxAttempts: 5,
+            lastAttemptAt: null,
+            nextRetryAt: null,
+            error: null,
+            nonce: "test-nonce-now",
+          });
+        })
+        .execute();
+    });
+
+    const wakeAt = await fragment.inContext(async function () {
+      return await this.handlerTx()
+        .withServiceCalls(
+          () => [fragment.services.hookService.getNextHookWakeAt(namespace)] as const,
+        )
+        .transform(({ serviceResult: [result] }) => result)
+        .execute();
+    });
+
+    expect(wakeAt).toBeInstanceOf(Date);
+    expect(Math.abs((wakeAt as Date).getTime() - Date.now())).toBeLessThan(5000);
+  });
+
+  it("should return earliest scheduled hook time", async () => {
+    const namespace = "wake-future";
+    const soon = new Date(Date.now() + 10000);
+    const later = new Date(Date.now() + 60000);
+
+    await fragment.inContext(async function () {
+      await this.handlerTx()
+        .mutate(({ forSchema }) => {
+          const uow = forSchema(internalSchema);
+          uow.create("fragno_hooks", {
+            namespace,
+            hookName: "onSoon",
+            payload: { test: "soon" },
+            status: "pending",
+            attempts: 0,
+            maxAttempts: 5,
+            lastAttemptAt: null,
+            nextRetryAt: soon,
+            error: null,
+            nonce: "test-nonce-soon",
+          });
+          uow.create("fragno_hooks", {
+            namespace,
+            hookName: "onLater",
+            payload: { test: "later" },
+            status: "pending",
+            attempts: 0,
+            maxAttempts: 5,
+            lastAttemptAt: null,
+            nextRetryAt: later,
+            error: null,
+            nonce: "test-nonce-later",
+          });
+        })
+        .execute();
+    });
+
+    const wakeAt = await fragment.inContext(async function () {
+      return await this.handlerTx()
+        .withServiceCalls(
+          () => [fragment.services.hookService.getNextHookWakeAt(namespace)] as const,
+        )
+        .transform(({ serviceResult: [result] }) => result)
+        .execute();
+    });
+
+    expect(wakeAt).toEqual(soon);
+  });
+
+  it("should return null when no pending hooks exist", async () => {
+    const namespace = "wake-none";
+    const wakeAt = await fragment.inContext(async function () {
+      return await this.handlerTx()
+        .withServiceCalls(
+          () => [fragment.services.hookService.getNextHookWakeAt(namespace)] as const,
+        )
+        .transform(({ serviceResult: [result] }) => result)
+        .execute();
+    });
+
+    expect(wakeAt).toBeNull();
+  });
+});
+
+describe("getSchemaVersionFromDatabase", () => {
+  function createTestSetup() {
+    const sqliteDatabase = new SQLite(":memory:");
+    const dialect = new SqliteDialect({ database: sqliteDatabase });
+    const adapter = new SqlAdapter({
+      dialect,
+      driverConfig: new BetterSQLite3DriverConfig(),
+    });
+
+    function instantiateFragment(options: FragnoPublicConfigWithDatabase) {
+      return instantiate(internalFragmentDef).withConfig({}).withOptions(options).build();
+    }
+
+    return { sqliteDatabase, adapter, instantiateFragment };
+  }
+
+  async function setupAndMigrate() {
+    const { sqliteDatabase, adapter, instantiateFragment } = createTestSetup();
+    // Create tables without writing a version record, so tests control version state
+    const migrations = adapter.prepareMigrations(internalSchema, "");
+    await migrations.executeWithDriver(adapter.driver, 0, undefined, {
+      updateVersionInMigration: false,
+    });
+    const fragment = instantiateFragment({
+      databaseAdapter: adapter,
+      databaseNamespace: null,
+    });
+    return { sqliteDatabase, adapter, fragment };
+  }
+
+  it("should return 0 when no version exists", async () => {
+    const { fragment } = await setupAndMigrate();
+
+    const version = await getSchemaVersionFromDatabase(fragment, "nonexistent");
+    expect(version).toBe(0);
+  });
+
+  it("should find version stored under empty-string namespace", async () => {
+    const { fragment } = await setupAndMigrate();
+
+    // Write version under empty-string namespace (key = ".schema_version")
+    await fragment.inContext(async function () {
+      await this.handlerTx()
+        .withServiceCalls(() => [fragment.services.settingsService.set("", "schema_version", "5")])
+        .execute();
+    });
+
+    const version = await getSchemaVersionFromDatabase(fragment, "");
+    expect(version).toBe(5);
+  });
+
+  it("should find version via back-compat when stored under internalSchema.name but read with empty string", async () => {
+    const { fragment } = await setupAndMigrate();
+
+    // Write version under "fragno_internal" namespace (legacy key from buggy code)
+    await fragment.inContext(async function () {
+      await this.handlerTx()
+        .withServiceCalls(() => [
+          fragment.services.settingsService.set(internalSchema.name, "schema_version", "3"),
+        ])
+        .execute();
+    });
+
+    // Reading with "" should find it via back-compat fallback
+    const version = await getSchemaVersionFromDatabase(fragment, "");
+    expect(version).toBe(3);
+  });
+
+  it("should find version via back-compat when stored under empty string but read with internalSchema.name", async () => {
+    const { fragment } = await setupAndMigrate();
+
+    // Write version under empty-string namespace
+    await fragment.inContext(async function () {
+      await this.handlerTx()
+        .withServiceCalls(() => [fragment.services.settingsService.set("", "schema_version", "7")])
+        .execute();
+    });
+
+    // Reading with internalSchema.name should find it via back-compat fallback
+    const version = await getSchemaVersionFromDatabase(fragment, internalSchema.name);
+    expect(version).toBe(7);
+  });
+
+  it("should prefer primary namespace over back-compat fallback", async () => {
+    const { fragment } = await setupAndMigrate();
+
+    // Write version under BOTH namespaces with different values
+    await fragment.inContext(async function () {
+      await this.handlerTx()
+        .withServiceCalls(() => [
+          fragment.services.settingsService.set("", "schema_version", "10"),
+          fragment.services.settingsService.set(internalSchema.name, "schema_version", "20"),
+        ])
+        .execute();
+    });
+
+    // Reading with "" should find 10 (primary), not 20 (back-compat)
+    const versionEmpty = await getSchemaVersionFromDatabase(fragment, "");
+    expect(versionEmpty).toBe(10);
+
+    // Reading with internalSchema.name should find 20 (primary), not 10 (back-compat)
+    const versionNamed = await getSchemaVersionFromDatabase(fragment, internalSchema.name);
+    expect(versionNamed).toBe(20);
+  });
+
+  it("should not use back-compat for non-internal namespaces", async () => {
+    const { fragment } = await setupAndMigrate();
+
+    // Write version under "some-fragment"
+    await fragment.inContext(async function () {
+      await this.handlerTx()
+        .withServiceCalls(() => [
+          fragment.services.settingsService.set("some-fragment", "schema_version", "4"),
+        ])
+        .execute();
+    });
+
+    // Reading with a different non-internal namespace should NOT find it
+    const version = await getSchemaVersionFromDatabase(fragment, "other-fragment");
+    expect(version).toBe(0);
+  });
 });