@fragno-dev/db 0.2.2 → 0.4.1

package/src/outbox/outbox.test.ts

@@ -0,0 +1,424 @@
+import { describe, expect, it } from "vitest";
+
+import { KyselyPGlite } from "kysely-pglite";
+import { SQLocalKysely } from "sqlocal/kysely";
+import superjson, { type SuperJSONResult } from "superjson";
+
+import { defineFragment, instantiate } from "@fragno-dev/core";
+
+import { PGlite } from "@electric-sql/pglite";
+
+import { PGLiteDriverConfig, SQLocalDriverConfig } from "../adapters/generic-sql/driver-config";
+import { SqlAdapter } from "../adapters/generic-sql/generic-sql-adapter";
+import { internalSchema, type InternalFragmentInstance } from "../fragments/internal-fragment";
+import { getInternalFragment } from "../internal/adapter-registry";
+import type { AnyFragnoInstantiatedDatabaseFragment, DatabaseRequestContext } from "../mod";
+import type { AnySchema } from "../schema/create";
+import { schema, idColumn, column, referenceColumn, FragnoReference } from "../schema/create";
+import { withDatabase } from "../with-database";
+import type { OutboxEntry, OutboxPayload } from "./outbox";
+
+const outboxSchema = schema("outbox", (s) => {
+  return s
+    .addTable("users", (t) => {
+      return t
+        .addColumn("id", idColumn())
+        .addColumn("email", column("string"))
+        .createIndex("idx_users_email", ["email"], { unique: true });
+    })
+    .addTable("posts", (t) => {
+      return t
+        .addColumn("id", idColumn())
+        .addColumn("authorId", referenceColumn())
+        .addColumn("title", column("string"));
+    })
+    .addReference("author", {
+      type: "one",
+      from: { table: "posts", column: "authorId" },
+      to: { table: "users", column: "id" },
+    });
+});
+
+const alphaSchema = schema("alpha", (s) =>
+  s.addTable("alpha_items", (t) =>
+    t.addColumn("id", idColumn()).addColumn("name", column("string")),
+  ),
+);
+
+const betaSchema = schema("beta", (s) =>
+  s.addTable("beta_items", (t) =>
+    t.addColumn("id", idColumn()).addColumn("title", column("string")),
+  ),
+);
+
+const outboxFragmentName = "outbox-test";
+const outboxFragmentDef = defineFragment(outboxFragmentName)
+  .extend(withDatabase(outboxSchema))
+  .build();
+
+type OutboxAdapterConfig =
+  | { type: "kysely-sqlite"; outboxEnabled?: boolean }
+  | { type: "kysely-pglite"; outboxEnabled?: boolean };
+
+type OutboxTestContext = {
+  fragment: AnyFragnoInstantiatedDatabaseFragment<typeof outboxSchema>;
+  internalFragment: InternalFragmentInstance;
+  cleanup: () => Promise<void>;
+};
+
+async function migrateSchema(
+  adapter: SqlAdapter,
+  schemaToMigrate: AnySchema,
+  namespace: string,
+): Promise<void> {
+  const migrations = adapter.prepareMigrations(schemaToMigrate, namespace);
+  await migrations.executeWithDriver(adapter.driver, 0);
+}
+
+async function createAdapter(config: OutboxAdapterConfig): Promise<{
+  adapter: SqlAdapter;
+  cleanup: () => Promise<void>;
+}> {
+  if (config.type === "kysely-sqlite") {
+    const { dialect } = new SQLocalKysely(":memory:");
+    const adapter = new SqlAdapter({
+      dialect,
+      driverConfig: new SQLocalDriverConfig(),
+    });
+
+    await migrateSchema(adapter, internalSchema, "");
+    await migrateSchema(adapter, outboxSchema, outboxSchema.name);
+
+    return {
+      adapter,
+      cleanup: async () => {
+        await adapter.close();
+      },
+    };
+  }
+
+  const pgliteDatabase = new PGlite();
+  const { dialect } = new KyselyPGlite(pgliteDatabase);
+  const adapter = new SqlAdapter({
+    dialect,
+    driverConfig: new PGLiteDriverConfig(),
+  });
+
+  await migrateSchema(adapter, internalSchema, "");
+  await migrateSchema(adapter, outboxSchema, outboxSchema.name);
+
+  return {
+    adapter,
+    cleanup: async () => {
+      await adapter.close();
+    },
+  };
+}
+
+async function buildOutboxTest(adapterConfig: OutboxAdapterConfig): Promise<OutboxTestContext> {
+  const { adapter, cleanup } = await createAdapter(adapterConfig);
+  const fragment = instantiate(outboxFragmentDef)
+    .withConfig({})
+    .withRoutes([])
+    .withOptions({
+      databaseAdapter: adapter,
+      outbox: adapterConfig.outboxEnabled ? { enabled: true } : undefined,
+    })
+    .build();
+
+  return {
+    fragment,
+    internalFragment: getInternalFragment(adapter),
+    cleanup,
+  };
+}
+
+async function listOutbox(
+  internalFragment: InternalFragmentInstance,
+  options?: { afterVersionstamp?: string; limit?: number },
+): Promise<OutboxEntry[]> {
+  return internalFragment.inContext(async function (this: DatabaseRequestContext) {
+    return (await this.handlerTx()
+      .withServiceCalls(() => [internalFragment.services.outboxService.list(options)] as const)
+      .transform(({ serviceResult: [result] }) => result)
+      .execute()) as OutboxEntry[];
+  });
+}
+
+async function listOutboxMutations(internalFragment: InternalFragmentInstance): Promise<
+  Array<{
+    entryVersionstamp: string;
+    mutationVersionstamp: string;
+    uowId: string;
+    schema: string;
+    table: string;
+    externalId: string;
+    op: string;
+  }>
+> {
+  return internalFragment.inContext(async function (this: DatabaseRequestContext) {
+    return await this.handlerTx()
+      .retrieve(({ forSchema }) =>
+        forSchema(internalSchema).find("fragno_db_outbox_mutations", (b) =>
+          b
+            .whereIndex("idx_outbox_mutations_entry")
+            .orderByIndex("idx_outbox_mutations_entry", "asc"),
+        ),
+      )
+      .transformRetrieve(([result]) => result)
+      .execute();
+  });
+}
+
+async function createUser(
+  fragment: AnyFragnoInstantiatedDatabaseFragment<typeof outboxSchema>,
+  email: string,
+) {
+  return fragment.inContext(async function (this: DatabaseRequestContext) {
+    await this.handlerTx()
+      .mutate(({ forSchema }) => forSchema(outboxSchema).create("users", { email }))
+      .execute();
+
+    const user = await this.handlerTx()
+      .retrieve(({ forSchema }) =>
+        forSchema(outboxSchema).findFirst("users", (b) =>
+          b.whereIndex("idx_users_email", (eb) => eb("email", "=", email)),
+        ),
+      )
+      .transformRetrieve(([result]) => result)
+      .execute();
+
+    if (!user) {
+      throw new Error("Expected user to be created.");
+    }
+
+    return user.id;
+  });
+}
+
+async function createPost(
+  fragment: AnyFragnoInstantiatedDatabaseFragment<typeof outboxSchema>,
+  title: string,
+  authorId: FragnoReference,
+) {
+  return fragment.inContext(async function (this: DatabaseRequestContext) {
+    return await this.handlerTx()
+      .mutate(({ forSchema }) => forSchema(outboxSchema).create("posts", { title, authorId }))
+      .transform(({ mutateResult }) => mutateResult)
+      .execute();
+  });
+}
+
+const adapterConfigs = [{ type: "kysely-sqlite" as const }, { type: "kysely-pglite" as const }];
+
+describe("Fragno DB Outbox", () => {
+  it("does not write outbox entries when disabled", async () => {
+    const { fragment, internalFragment, cleanup } = await buildOutboxTest({
+      type: "kysely-sqlite",
+    });
+
+    await createUser(fragment, "disabled@example.com");
+
+    const entries = await listOutbox(internalFragment);
+    expect(entries).toHaveLength(0);
+    const mutations = await listOutboxMutations(internalFragment);
+    expect(mutations).toHaveLength(0);
+
+    await cleanup();
+  });
+
+  it("stores refMap placeholders and lists entries in order", async () => {
+    const { fragment, internalFragment, cleanup } = await buildOutboxTest({
+      type: "kysely-sqlite",
+      outboxEnabled: true,
+    });
+
+    const userId = await createUser(fragment, "alpha@example.com");
+    expect(userId.internalId).toBeDefined();
+
+    await createPost(fragment, "Hello", FragnoReference.fromInternal(userId.internalId!));
+
+    const entries = await listOutbox(internalFragment);
+    expect(entries).toHaveLength(2);
+    expect(entries[0].versionstamp < entries[1].versionstamp).toBe(true);
+
+    const filtered = await listOutbox(internalFragment, {
+      afterVersionstamp: entries[0].versionstamp,
+      limit: 1,
+    });
+    expect(filtered).toHaveLength(1);
+    expect(filtered[0].versionstamp).toBe(entries[1].versionstamp);
+
+    const payload = superjson.deserialize(entries[1].payload as SuperJSONResult) as OutboxPayload;
+    expect(payload.version).toBe(1);
+    expect(payload.mutations).toHaveLength(1);
+    const [mutation] = payload.mutations;
+    if (mutation.op !== "create") {
+      throw new Error("Expected create mutation in outbox payload.");
+    }
+    expect(mutation.values).toMatchObject({
+      authorId: { __fragno_ref: "0.authorId" },
+    });
+    expect(entries[1].refMap).toEqual({
+      "0.authorId": userId.externalId,
+    });
+
+    await internalFragment.inContext(async function (this: DatabaseRequestContext) {
+      await this.handlerTx()
+        .withServiceCalls(() => [
+          internalFragment.services.settingsService.set("outbox-test", "noop", "1"),
+        ])
+        .execute();
+    });
+
+    const afterInternal = await listOutbox(internalFragment);
+    expect(afterInternal).toHaveLength(2);
+
+    await cleanup();
+  });
+
+  it("writes mutation log rows for each outbox entry", async () => {
+    const { fragment, internalFragment, cleanup } = await buildOutboxTest({
+      type: "kysely-sqlite",
+      outboxEnabled: true,
+    });
+
+    const userId = await createUser(fragment, "log-alpha@example.com");
+    expect(userId.internalId).toBeDefined();
+
+    await createPost(fragment, "Log", FragnoReference.fromInternal(userId.internalId!));
+
+    const entries = await listOutbox(internalFragment);
+    const mutations = await listOutboxMutations(internalFragment);
+
+    expect(entries).toHaveLength(2);
+    expect(mutations).toHaveLength(2);
+
+    const entryByVersion = new Map(entries.map((entry) => [entry.versionstamp, entry]));
+
+    for (const entry of entries) {
+      const payload = superjson.deserialize(entry.payload as SuperJSONResult) as OutboxPayload;
+      expect(payload.mutations).toHaveLength(1);
+    }
+
+    for (const mutationRow of mutations) {
+      const entry = entryByVersion.get(mutationRow.entryVersionstamp);
+      expect(entry).toBeDefined();
+      const payload = superjson.deserialize(entry!.payload as SuperJSONResult) as OutboxPayload;
+      const mutation = payload.mutations[0];
+      expect(mutationRow.mutationVersionstamp).toBe(mutation.versionstamp);
+      expect(mutationRow.uowId).toBe(entry!.uowId);
+      expect(mutationRow.schema).toBe(mutation.schema);
+      expect(mutationRow.table).toBe(mutation.table);
+      expect(mutationRow.externalId).toBe(mutation.externalId);
+      expect(mutationRow.op).toBe(mutation.op);
+    }
+
+    await cleanup();
+  });
+
+  it("orders outbox entries by commit order across concurrent UOWs", async () => {
+    const { fragment, internalFragment, cleanup } = await buildOutboxTest({
+      type: "kysely-sqlite",
+      outboxEnabled: true,
+    });
+
+    const createUow = fragment.$internal.deps.createUnitOfWork;
+    const uow1 = createUow();
+    const uow2 = createUow();
+    const uow1Id = uow1.idempotencyKey;
+    const uow2Id = uow2.idempotencyKey;
+
+    uow1.forSchema(outboxSchema).create("users", { email: "order-1@example.com" });
+    uow2.forSchema(outboxSchema).create("users", { email: "order-2@example.com" });
+
+    const completionOrder: string[] = [];
+    await Promise.all([
+      uow1.executeMutations().then(() => completionOrder.push(uow1Id)),
+      uow2.executeMutations().then(() => completionOrder.push(uow2Id)),
+    ]);
+
+    const entries = await listOutbox(internalFragment);
+    expect(entries.map((entry) => entry.uowId)).toEqual(completionOrder);
+    expect(entries[0].versionstamp < entries[1].versionstamp).toBe(true);
+
+    await cleanup();
+  });
+
+  it("only writes outbox entries for schemas that opt in", async () => {
+    const { dialect } = new SQLocalKysely(":memory:");
+    const adapter = new SqlAdapter({
+      dialect,
+      driverConfig: new SQLocalDriverConfig(),
+    });
+
+    try {
+      await migrateSchema(adapter, internalSchema, "");
+      await migrateSchema(adapter, alphaSchema, alphaSchema.name);
+      await migrateSchema(adapter, betaSchema, betaSchema.name);
+
+      const alphaDef = defineFragment("alpha-fragment").extend(withDatabase(alphaSchema)).build();
+      const betaDef = defineFragment("beta-fragment").extend(withDatabase(betaSchema)).build();
+
+      const alphaFragment = instantiate(alphaDef)
+        .withConfig({})
+        .withRoutes([])
+        .withOptions({ databaseAdapter: adapter, outbox: { enabled: true } })
+        .build();
+
+      const betaFragment = instantiate(betaDef)
+        .withConfig({})
+        .withRoutes([])
+        .withOptions({ databaseAdapter: adapter })
+        .build();
+
+      await alphaFragment.inContext(async function (this: DatabaseRequestContext) {
+        await this.handlerTx()
+          .mutate(({ forSchema }) =>
+            forSchema(alphaSchema).create("alpha_items", { name: "alpha" }),
+          )
+          .execute();
+      });
+
+      await betaFragment.inContext(async function (this: DatabaseRequestContext) {
+        await this.handlerTx()
+          .mutate(({ forSchema }) => forSchema(betaSchema).create("beta_items", { title: "beta" }))
+          .execute();
+      });
+
+      const entries = await listOutbox(getInternalFragment(adapter));
+      const mutationSchemas = entries
+        .map((entry) => superjson.deserialize(entry.payload as SuperJSONResult) as OutboxPayload)
+        .flatMap((payload) => payload.mutations.map((mutation) => mutation.schema));
+
+      expect(mutationSchemas).toContain(alphaSchema.name);
+      expect(mutationSchemas).not.toContain(betaSchema.name);
+    } finally {
+      await adapter.close();
+    }
+  });
+
+  describe.each(adapterConfigs)("adapter opt-in (%s)", (adapterConfig) => {
+    it("writes outbox rows only when enabled", async () => {
+      const { fragment, internalFragment, cleanup } = await buildOutboxTest(adapterConfig);
+
+      await createUser(fragment, "disabled@example.com");
+      const disabledEntries = await listOutbox(internalFragment);
+      expect(disabledEntries).toHaveLength(0);
+      await cleanup();
+
+      const {
+        fragment: enabledFragment,
+        internalFragment: enabledInternal,
+        cleanup: enabledCleanup,
+      } = await buildOutboxTest({
+        ...adapterConfig,
+        outboxEnabled: true,
+      });
+      await createUser(enabledFragment, "enabled@example.com");
+      const enabledEntries = await listOutbox(enabledInternal);
+      expect(enabledEntries).toHaveLength(1);
+      await enabledCleanup();
+    }, 10_000);
+  });
+});

package/src/outbox/outbox.ts

@@ -0,0 +1,139 @@
+import type { MutationOperation } from "../query/unit-of-work/unit-of-work";
+import type { AnySchema, AnyTable, FragnoId } from "../schema/create";
+
+export type OutboxConfig = {
+  enabled: boolean;
+  shouldInclude?: (operation: MutationOperation<AnySchema>) => boolean;
+};
+
+export type OutboxVersionstampStrategy =
+  | "update-returning"
+  | "insert-on-conflict-returning"
+  | "insert-on-duplicate-last-insert-id";
+
+export type OutboxPayload = {
+  version: 1;
+  mutations: OutboxMutation[];
+};
+
+export type OutboxMutation =
+  | {
+      op: "create";
+      schema: string;
+      namespace?: string;
+      table: string;
+      externalId: string;
+      versionstamp: string;
+      values: Record<string, unknown>;
+    }
+  | {
+      op: "update";
+      schema: string;
+      namespace?: string;
+      table: string;
+      externalId: string;
+      versionstamp: string;
+      set: Record<string, unknown>;
+      checkVersion?: number;
+    }
+  | {
+      op: "delete";
+      schema: string;
+      namespace?: string;
+      table: string;
+      externalId: string;
+      versionstamp: string;
+      checkVersion?: number;
+    };
+
+export type OutboxRefMap = Record<string, string>;
+
+export type OutboxPayloadSerialized = {
+  json: unknown;
+  meta?: Record<string, unknown>;
+};
+
+export type OutboxEntry = {
+  id: FragnoId;
+  versionstamp: string;
+  uowId: string;
+  payload: OutboxPayloadSerialized;
+  refMap?: OutboxRefMap;
+  createdAt: Date;
+};
+
+export type OutboxRefLookup = {
+  key: string;
+  internalId: bigint | number;
+  table: AnyTable;
+  namespace?: string;
+};
+
+export function encodeVersionstamp(transactionVersion: bigint, userVersion: number): Uint8Array {
+  if (userVersion < 0 || userVersion > 0xffff) {
+    throw new Error(`Invalid outbox user version: ${userVersion}`);
+  }
+
+  const txBytes = bigintToBytes(transactionVersion, 10);
+  const userBytes = new Uint8Array(2);
+  userBytes[0] = (userVersion >> 8) & 0xff;
+  userBytes[1] = userVersion & 0xff;
+
+  const combined = new Uint8Array(12);
+  combined.set(txBytes, 0);
+  combined.set(userBytes, 10);
+
+  return combined;
+}
+
+export function versionstampToHex(bytes: Uint8Array): string {
+  let hex = "";
+  for (const byte of bytes) {
+    hex += byte.toString(16).padStart(2, "0");
+  }
+  return hex;
+}
+
+export function hexToVersionstamp(hex: string): Uint8Array {
+  if (hex.length % 2 !== 0) {
+    throw new Error(`Invalid versionstamp hex length: ${hex.length}`);
+  }
+
+  const bytes = new Uint8Array(hex.length / 2);
+  for (let i = 0; i < hex.length; i += 2) {
+    bytes[i / 2] = Number.parseInt(hex.slice(i, i + 2), 16);
+  }
+  return bytes;
+}
+
+export function parseOutboxVersionValue(value: unknown): bigint {
+  if (typeof value === "bigint") {
+    return value;
+  }
+
+  if (typeof value === "number") {
+    return BigInt(value);
+  }
+
+  if (typeof value === "string") {
+    return BigInt(value);
+  }
+
+  throw new Error(`Invalid outbox version value: ${String(value)}`);
+}
+
+function bigintToBytes(value: bigint, length: number): Uint8Array {
+  const bytes = new Uint8Array(length);
+  let remaining = value;
+
+  for (let i = length - 1; i >= 0; i -= 1) {
+    bytes[i] = Number(remaining & 0xffn);
+    remaining >>= 8n;
+  }
+
+  if (remaining !== 0n) {
+    throw new Error(`Outbox version ${value} exceeds ${length * 8} bits`);
+  }
+
+  return bytes;
+}

package/src/query/column-defaults.ts

@@ -1,5 +1,10 @@
-import type { AnyColumn } from "../schema/create";
 import { createId } from "../id";
+import type { AnyColumn } from "../schema/create";
+
+export type RuntimeDefaultContext = {
+  now?: () => Date;
+  createId?: () => string;
+};
 
 /**
  * Generate a runtime default value for a column that has defaultTo$()
@@ -12,7 +17,10 @@ import { createId } from "../id";
  *
  * @internal
  */
-export function generateRuntimeDefault(column: AnyColumn): unknown {
+export function generateRuntimeDefault(
+  column: AnyColumn,
+  context: RuntimeDefaultContext = {},
+): unknown {
   // Check if column has a default value configuration
   if (!column.default) {
     return undefined;
@@ -34,11 +42,11 @@ export function generateRuntimeDefault(column: AnyColumn): unknown {
   const runtime = column.default.runtime;
 
   if (runtime === "cuid") {
-    return createId();
+    return (context.createId ?? createId)();
   }
 
   if (runtime === "now") {
-    return new Date();
+    return (context.now ?? (() => new Date()))();
   }
 
   if (typeof runtime === "function") {
@@ -47,3 +55,33 @@ export function generateRuntimeDefault(column: AnyColumn): unknown {
 
   return undefined;
 }
+
+/**
+ * Generate a fallback value for database-level defaults.
+ *
+ * This is used by adapters that cannot rely on database DEFAULT constraints,
+ * such as the in-memory adapter.
+ *
+ * @internal
+ */
+export function generateDatabaseDefault(
+  column: AnyColumn,
+  context: RuntimeDefaultContext = {},
+): unknown {
+  if (!column.default) {
+    return undefined;
+  }
+
+  if ("value" in column.default) {
+    return column.default.value;
+  }
+
+  if ("dbSpecial" in column.default) {
+    if (column.default.dbSpecial === "now") {
+      return (context.now ?? (() => new Date()))();
+    }
+    return undefined;
+  }
+
+  return undefined;
+}

package/src/query/condition-builder.test.ts

@@ -1,9 +1,10 @@
 import { describe, expect, it } from "vitest";
+
 import { column, idColumn, schema } from "../schema/create";
 import { createBuilder, createIndexedBuilder } from "./condition-builder";
 
 describe("ConditionBuilder", () => {
-  const testSchema = schema((s) =>
+  const testSchema = schema("test", (s) =>
     s
       .addTable("users", (t) =>
         t
@@ -11,7 +12,7 @@ describe("ConditionBuilder", () => {
           .addColumn("email", column("string"))
           .addColumn("name", column("string"))
           .addColumn("age", column("integer").nullable())
-          .createIndex("_primary", ["id"], { unique: true })
+          .createIndex("idx_users_primary", ["id"], { unique: true })
           .createIndex("idx_email", ["email"], { unique: true })
           .createIndex("idx_name_age", ["name", "age"]),
       )
@@ -21,7 +22,7 @@ describe("ConditionBuilder", () => {
           .addColumn("title", column("string"))
           .addColumn("content", column("string"))
           .addColumn("published", column("bool"))
-          .createIndex("_primary", ["id"], { unique: true })
+          .createIndex("idx_posts_primary", ["id"], { unique: true })
           .createIndex("idx_published", ["published"]),
       ),
   );
@@ -101,6 +102,20 @@ describe("ConditionBuilder", () => {
       });
     });
 
+    it("should expose db now helper", () => {
+      const builder = createBuilder(usersTable.columns);
+
+      const now = builder.now();
+      expect(now).toMatchObject({ tag: "db-now" });
+      expect(typeof now.plus).toBe("function");
+      expect(now.plus({ seconds: 1 })).toMatchObject({ tag: "db-now", offsetMs: 1000 });
+      expect(now.plus({ seconds: 1 }).plus({ minutes: 1 })).toMatchObject({
+        tag: "db-now",
+        offsetMs: 61_000,
+      });
+      expect(builder.interval({ minutes: 1 })).toEqual({ tag: "db-interval", ms: 60_000 });
+    });
+
     it("should support boolean columns", () => {
       const postsTable = testSchema.tables.posts;
       const builder = createBuilder(postsTable.columns);