@fragno-dev/db 0.3.0 → 0.4.1

package/src/adapters/in-memory/in-memory-uow.ts

@@ -1,6 +1,29 @@
-import type { AnySchema, AnyTable } from "../../schema/create";
-import { FragnoId, FragnoReference } from "../../schema/create";
-import { SQLocalDriverConfig } from "../generic-sql/driver-config";
+import superjson from "superjson";
+
+import {
+  SETTINGS_NAMESPACE,
+  SETTINGS_TABLE_NAME,
+  internalSchema,
+} from "../../fragments/internal-fragment";
+import type { NamingResolver } from "../../naming/sql-naming";
+import {
+  type OutboxRefLookup,
+  type OutboxRefMap,
+  encodeVersionstamp,
+  versionstampToHex,
+  parseOutboxVersionValue,
+} from "../../outbox/outbox";
+import { buildOutboxPlan, finalizeOutboxPayload } from "../../outbox/outbox-builder";
+import { buildCondition } from "../../query/condition-builder";
+import {
+  createCursorFromRecord,
+  decodeCursor,
+  type Cursor,
+  type CursorResult,
+} from "../../query/cursor";
+import { getDbNowOffsetMs, isDbNow } from "../../query/db-now";
+import type { CompiledJoin } from "../../query/orm/orm";
+import { createSQLSerializer } from "../../query/serialize/create-sql-serializer";
 import type {
   CompiledMutation,
   MutationOperation,
@@ -10,21 +33,17 @@ import type {
   UOWDecoder,
   UOWExecutor,
 } from "../../query/unit-of-work/unit-of-work";
-import { buildCondition } from "../../query/condition-builder";
 import {
   encodeValues,
   encodeValuesWithDbDefaults,
   ReferenceSubquery,
 } from "../../query/value-encoding";
-import { isDbNow } from "../../query/db-now";
-import { createSQLSerializer } from "../../query/serialize/create-sql-serializer";
-import type { CompiledJoin } from "../../query/orm/orm";
-import {
-  createCursorFromRecord,
-  decodeCursor,
-  type Cursor,
-  type CursorResult,
-} from "../../query/cursor";
+import type { AnySchema, AnyTable } from "../../schema/create";
+import { FragnoId, FragnoReference } from "../../schema/create";
+import { SQLocalDriverConfig } from "../generic-sql/driver-config";
+import { evaluateCondition } from "./condition-evaluator";
+import type { ResolvedInMemoryAdapterOptions } from "./options";
+import { resolveReferenceSubqueries } from "./reference-resolution";
 import type {
   InMemoryNamespaceStore,
   InMemoryRow,
@@ -32,16 +51,15 @@ import type {
   InMemoryTableStore,
 } from "./store";
 import { buildIndexKey, ensureNamespaceStore, normalizeIndexValue } from "./store";
-import { evaluateCondition } from "./condition-evaluator";
-import { resolveReferenceSubqueries } from "./reference-resolution";
-import type { ResolvedInMemoryAdapterOptions } from "./options";
 import { compareNormalizedValues } from "./value-comparison";
-import type { NamingResolver } from "../../naming/sql-naming";
 
 type InMemoryCompiledQuery = RetrievalOperation<AnySchema> | MutationOperation<AnySchema>;
 type InMemoryRawResult = InMemoryRow[] | { count: number }[];
 type CursorInput = string | Cursor | undefined;
 type ResolverFactory = (schema: AnySchema, namespace: string | null) => NamingResolver;
+type SchemaNamespaceEntry = { schema: AnySchema; namespace: string | null };
+
+const OUTBOX_VERSION_KEY = `${SETTINGS_NAMESPACE}.outbox_version`;
 
 const getResolver = (
   schema: AnySchema,
@@ -255,20 +273,32 @@ const findJoinMatches = (
         throw new Error(`Column "${right}" not found on table "${targetTable.name}".`);
       }
 
-      const actualRight = rightColumn.role === "external-id" ? "_internalId" : right;
+      const actualLeft =
+        leftColumn.role === "external-id" && rightColumn.role !== "external-id"
+          ? "_internalId"
+          : left;
+      const actualLeftColumn = parentTable.columns[actualLeft];
+      if (!actualLeftColumn) {
+        throw new Error(`Column "${actualLeft}" not found on table "${parentTable.name}".`);
+      }
+
+      const actualRight =
+        rightColumn.role === "external-id" && leftColumn.role !== "external-id"
+          ? "_internalId"
+          : right;
       const actualRightColumn = targetTable.columns[actualRight];
       if (!actualRightColumn) {
         throw new Error(`Column "${actualRight}" not found on table "${targetTable.name}".`);
       }
 
-      const leftValue = parentRow[getPhysicalColumnName(parentTable, left, resolver)];
+      const leftValue = parentRow[getPhysicalColumnName(parentTable, actualLeft, resolver)];
       const rightValue = row[getPhysicalColumnName(targetTable, actualRight, resolver)];
       if (isNullish(leftValue) || isNullish(rightValue)) {
         matchesJoin = false;
         break;
       }
 
-      const leftNormalized = normalizeIndexValue(leftValue, leftColumn);
+      const leftNormalized = normalizeIndexValue(leftValue, actualLeftColumn);
       const rightNormalized = normalizeIndexValue(rightValue, actualRightColumn);
       if (compareNormalizedValues(leftNormalized, rightNormalized) !== 0) {
         matchesJoin = false;
@@ -433,7 +463,7 @@ const enforceOutgoingForeignKeys = (
   resolver?: NamingResolver,
 ): void => {
   for (const relation of Object.values(table.relations)) {
-    if (relation.type !== "one") {
+    if (relation.type !== "one" || relation.foreignKey === false) {
       continue;
     }
 
@@ -500,7 +530,7 @@ const enforceNoIncomingForeignKeys = (
 ): void => {
   for (const sourceTable of Object.values(schema.tables)) {
     for (const relation of Object.values(sourceTable.relations)) {
-      if (relation.type !== "one") {
+      if (relation.type !== "one" || relation.foreignKey === false) {
         continue;
       }
 
@@ -645,7 +675,9 @@ const findRows = (
 
   const scanOptions = {
     direction,
-    limit,
+    // Apply result limits after where/join filtering. Passing the limit into the
+    // raw index scan can drop matching rows that appear later in the index.
+    limit: undefined as number | undefined,
     start: undefined as readonly unknown[] | undefined,
     startInclusive: true,
     end: undefined as readonly unknown[] | undefined,
@@ -758,8 +790,14 @@ const countRows = (
   return count;
 };
 
-const resolveDbNowValue = (value: unknown, options: ResolvedInMemoryAdapterOptions): unknown =>
-  isDbNow(value) ? options.clock.now() : value;
+const resolveDbNowValue = (value: unknown, options: ResolvedInMemoryAdapterOptions): unknown => {
+  if (!isDbNow(value)) {
+    return value;
+  }
+  const now = options.clock.now();
+  const offsetMs = getDbNowOffsetMs(value);
+  return offsetMs === 0 ? now : new Date(now.getTime() + offsetMs);
+};
 
 const createRow = (
   op: Extract<MutationOperation<AnySchema>, { type: "create" }>,
@@ -1001,6 +1039,279 @@ const checkRow = (
   }
 };
 
+const resolveSchemaForLookup = (
+  table: AnyTable,
+  namespace: string | undefined,
+  schemaByNamespace?: Map<string, SchemaNamespaceEntry>,
+): SchemaNamespaceEntry | null => {
+  if (!schemaByNamespace) {
+    return null;
+  }
+
+  if (namespace !== undefined) {
+    const entry = schemaByNamespace.get(namespace);
+    if (entry && entry.schema.tables[table.name] === table) {
+      return entry;
+    }
+    return null;
+  }
+
+  for (const entry of schemaByNamespace.values()) {
+    if (entry.schema.tables[table.name] === table) {
+      return entry;
+    }
+  }
+
+  return null;
+};
+
+const reserveOutboxVersion = (
+  store: InMemoryStore,
+  options: ResolvedInMemoryAdapterOptions,
+  resolverFactory?: ResolverFactory,
+): { version: bigint; rollback: () => void } => {
+  const resolver = getResolver(internalSchema, null, resolverFactory);
+  const namespaceStore = getNamespaceStore(store, internalSchema, null, resolver);
+  const settingsTable = internalSchema.tables[SETTINGS_TABLE_NAME];
+  if (!settingsTable) {
+    throw new Error("Missing internal settings table definition.");
+  }
+  const tableStore = getTableStore(namespaceStore, settingsTable, resolver);
+  const keyColumnName = getPhysicalColumnName(settingsTable, "key", resolver);
+  const valueColumnName = getPhysicalColumnName(settingsTable, "value", resolver);
+  const idColumnName = getPhysicalColumnName(
+    settingsTable,
+    settingsTable.getIdColumn().name,
+    resolver,
+  );
+
+  for (const row of tableStore.rows.values()) {
+    if (row[keyColumnName] !== OUTBOX_VERSION_KEY) {
+      continue;
+    }
+
+    const rawValue = row[valueColumnName];
+    const current = parseOutboxVersionValue(rawValue);
+    const next = current + 1n;
+    const externalId = row[idColumnName];
+    if (typeof externalId !== "string") {
+      throw new Error("Outbox version row is missing external id.");
+    }
+
+    const updateOp: Extract<MutationOperation<AnySchema>, { type: "update" }> = {
+      type: "update",
+      schema: internalSchema,
+      namespace: null,
+      table: settingsTable.name,
+      id: externalId,
+      checkVersion: false,
+      set: { value: next.toString() },
+    };
+    const rollback = updateRow(updateOp, namespaceStore, tableStore, options, resolver);
+    return { version: next, rollback: rollback ?? (() => {}) };
+  }
+
+  const createOp: Extract<MutationOperation<AnySchema>, { type: "create" }> = {
+    type: "create",
+    schema: internalSchema,
+    namespace: null,
+    table: settingsTable.name,
+    values: { key: OUTBOX_VERSION_KEY, value: "0" },
+    generatedExternalId: options.idGenerator(),
+  };
+  const previousInternalId = tableStore.nextInternalId;
+  const internalId = createRow(createOp, namespaceStore, tableStore, options, resolver);
+  const rollback = () => {
+    const existingRow = tableStore.rows.get(internalId);
+    if (existingRow) {
+      for (const indexStore of tableStore.indexes.values()) {
+        const key = buildIndexKey(settingsTable, indexStore.definition, existingRow, resolver);
+        indexStore.index.remove(key, internalId);
+      }
+      tableStore.rows.delete(internalId);
+    }
+    tableStore.nextInternalId = previousInternalId;
+  };
+
+  return { version: 0n, rollback };
+};
+
+const resolveOutboxRefMap = (
+  store: InMemoryStore,
+  lookups: OutboxRefLookup[],
+  resolverFactory: ResolverFactory | undefined,
+  schemaByNamespace?: Map<string, SchemaNamespaceEntry>,
+): OutboxRefMap | undefined => {
+  if (lookups.length === 0) {
+    return undefined;
+  }
+
+  const refMap: OutboxRefMap = {};
+
+  for (const lookup of lookups) {
+    const schemaEntry = resolveSchemaForLookup(lookup.table, lookup.namespace, schemaByNamespace);
+    if (!schemaEntry) {
+      throw new Error(`Failed to resolve schema for outbox lookup on ${lookup.table.name}.`);
+    }
+
+    const resolver = getResolver(schemaEntry.schema, schemaEntry.namespace, resolverFactory);
+    const namespaceStore = getNamespaceStore(
+      store,
+      schemaEntry.schema,
+      schemaEntry.namespace,
+      resolver,
+    );
+    const tableStore = getTableStore(namespaceStore, lookup.table, resolver);
+    const internalId =
+      typeof lookup.internalId === "number" ? BigInt(lookup.internalId) : lookup.internalId;
+    const row = tableStore.rows.get(internalId);
+
+    if (!row) {
+      const tableName = resolver ? resolver.getTableName(lookup.table.name) : lookup.table.name;
+      const internalColumn = resolver
+        ? resolver.getColumnName(lookup.table.name, lookup.table.getInternalIdColumn().name)
+        : lookup.table.getInternalIdColumn().name;
+      throw new Error(
+        `Failed to resolve outbox reference for ${tableName}.${internalColumn}=${String(lookup.internalId)}`,
+      );
+    }
+
+    const externalColumnName = getPhysicalColumnName(
+      lookup.table,
+      lookup.table.getIdColumn().name,
+      resolver,
+    );
+    const externalId = row[externalColumnName];
+    if (typeof externalId !== "string") {
+      throw new Error("Outbox reference row is missing external id.");
+    }
+
+    refMap[lookup.key] = externalId;
+  }
+
+  return Object.keys(refMap).length > 0 ? refMap : undefined;
+};
+
+const insertOutboxRow = (
+  store: InMemoryStore,
+  options: ResolvedInMemoryAdapterOptions,
+  resolverFactory: ResolverFactory | undefined,
+  payload: {
+    versionstamp: string;
+    uowId: string;
+    payload: { json: unknown; meta?: Record<string, unknown> };
+    refMap?: OutboxRefMap;
+  },
+): (() => void) => {
+  const resolver = getResolver(internalSchema, null, resolverFactory);
+  const namespaceStore = getNamespaceStore(store, internalSchema, null, resolver);
+  const outboxTable = internalSchema.tables.fragno_db_outbox;
+  if (!outboxTable) {
+    throw new Error("Missing internal outbox table definition.");
+  }
+  const tableStore = getTableStore(namespaceStore, outboxTable, resolver);
+  const createOp: Extract<MutationOperation<AnySchema>, { type: "create" }> = {
+    type: "create",
+    schema: internalSchema,
+    namespace: null,
+    table: outboxTable.name,
+    values: {
+      versionstamp: payload.versionstamp,
+      uowId: payload.uowId,
+      payload: payload.payload,
+      ...(payload.refMap ? { refMap: payload.refMap } : {}),
+    },
+    generatedExternalId: options.idGenerator(),
+  };
+  const previousInternalId = tableStore.nextInternalId;
+  const internalId = createRow(createOp, namespaceStore, tableStore, options, resolver);
+
+  return () => {
+    const existingRow = tableStore.rows.get(internalId);
+    if (existingRow) {
+      for (const indexStore of tableStore.indexes.values()) {
+        const key = buildIndexKey(outboxTable, indexStore.definition, existingRow, resolver);
+        indexStore.index.remove(key, internalId);
+      }
+      tableStore.rows.delete(internalId);
+    }
+    tableStore.nextInternalId = previousInternalId;
+  };
+};
+
+const insertOutboxMutationRows = (
+  store: InMemoryStore,
+  options: ResolvedInMemoryAdapterOptions,
+  resolverFactory: ResolverFactory | undefined,
+  payload: {
+    entryVersionstamp: string;
+    uowId: string;
+    mutations: {
+      versionstamp: string;
+      schema: string;
+      table: string;
+      externalId: string;
+      op: string;
+    }[];
+  },
+): Array<() => void> => {
+  if (payload.mutations.length === 0) {
+    return [];
+  }
+
+  const resolver = getResolver(internalSchema, null, resolverFactory);
+  const namespaceStore = getNamespaceStore(store, internalSchema, null, resolver);
+  const mutationsTable = internalSchema.tables.fragno_db_outbox_mutations;
+  if (!mutationsTable) {
+    throw new Error("Missing internal outbox mutations table definition.");
+  }
+  const tableStore = getTableStore(namespaceStore, mutationsTable, resolver);
+  const rollbackActions: Array<() => void> = [];
+
+  try {
+    for (const mutation of payload.mutations) {
+      const createOp: Extract<MutationOperation<AnySchema>, { type: "create" }> = {
+        type: "create",
+        schema: internalSchema,
+        namespace: null,
+        table: mutationsTable.name,
+        values: {
+          entryVersionstamp: payload.entryVersionstamp,
+          mutationVersionstamp: mutation.versionstamp,
+          uowId: payload.uowId,
+          schema: mutation.schema,
+          table: mutation.table,
+          externalId: mutation.externalId,
+          op: mutation.op,
+        },
+        generatedExternalId: options.idGenerator(),
+      };
+
+      const previousInternalId = tableStore.nextInternalId;
+      const internalId = createRow(createOp, namespaceStore, tableStore, options, resolver);
+
+      rollbackActions.push(() => {
+        const existingRow = tableStore.rows.get(internalId);
+        if (existingRow) {
+          for (const indexStore of tableStore.indexes.values()) {
+            const key = buildIndexKey(mutationsTable, indexStore.definition, existingRow, resolver);
+            indexStore.index.remove(key, internalId);
+          }
+          tableStore.rows.delete(internalId);
+        }
+        tableStore.nextInternalId = previousInternalId;
+      });
+    }
+  } catch (error) {
+    for (const rollback of rollbackActions.reverse()) {
+      rollback();
+    }
+    throw error;
+  }
+
+  return rollbackActions;
+};
+
 export const createInMemoryUowCompiler = (): UOWCompiler<InMemoryCompiledQuery> => ({
   compileRetrievalOperation(op: RetrievalOperation<AnySchema>): InMemoryCompiledQuery | null {
     return op;
@@ -1022,6 +1333,7 @@ export const createInMemoryUowExecutor = (
   store: InMemoryStore,
   options: ResolvedInMemoryAdapterOptions,
   resolverFactory?: ResolverFactory,
+  schemaByNamespace?: Map<string, SchemaNamespaceEntry>,
 ): UOWExecutor<InMemoryCompiledQuery, InMemoryRawResult> => ({
   async executeRetrievalPhase(
     retrievalBatch: InMemoryCompiledQuery[],
@@ -1060,8 +1372,31 @@ export const createInMemoryUowExecutor = (
   ): Promise<MutationResult> {
     const createdInternalIds: (bigint | null)[] = [];
     const rollbackActions: Array<() => void> = [];
+    const outboxEnabled = options.outbox?.enabled ?? false;
+    const shouldInclude = options.outbox?.shouldInclude;
+    const outboxOperations = outboxEnabled
+      ? mutationBatch.flatMap((mutation) => {
+          const operation = mutation.operation;
+          if (!operation) {
+            return [];
+          }
+          if (shouldInclude && !shouldInclude(operation)) {
+            return [];
+          }
+          return [operation];
+        })
+      : [];
+    const outboxPlan = outboxOperations.length > 0 ? buildOutboxPlan(outboxOperations) : null;
+    const shouldWriteOutbox = outboxEnabled && outboxPlan !== null && outboxPlan.drafts.length > 0;
+    let outboxVersion: bigint | null = null;
 
     try {
+      if (shouldWriteOutbox) {
+        const reservation = reserveOutboxVersion(store, options, resolverFactory);
+        outboxVersion = reservation.version;
+        rollbackActions.push(reservation.rollback);
+      }
+
       for (const compiled of mutationBatch) {
         const operation = compiled.query;
 
@@ -1161,6 +1496,37 @@ export const createInMemoryUowExecutor = (
 
         throw new Error(`Unsupported in-memory mutation "${operation.type}".`);
       }
+
+      if (shouldWriteOutbox && outboxPlan && outboxVersion !== null) {
+        const uowId = mutationBatch[0]?.uowId;
+        if (!uowId) {
+          throw new Error("Outbox mutation batch is missing uowId.");
+        }
+
+        const refMap = resolveOutboxRefMap(
+          store,
+          outboxPlan.lookups,
+          resolverFactory,
+          schemaByNamespace,
+        );
+        const payload = finalizeOutboxPayload(outboxPlan, outboxVersion);
+        const payloadSerialized = superjson.serialize(payload);
+        const versionstamp = versionstampToHex(encodeVersionstamp(outboxVersion, 0));
+        rollbackActions.push(
+          ...insertOutboxMutationRows(store, options, resolverFactory, {
+            entryVersionstamp: versionstamp,
+            uowId,
+            mutations: payload.mutations,
+          }),
+        );
+        const rollback = insertOutboxRow(store, options, resolverFactory, {
+          versionstamp,
+          uowId,
+          payload: payloadSerialized,
+          refMap,
+        });
+        rollbackActions.push(rollback);
+      }
     } catch (error) {
       for (const rollback of rollbackActions.reverse()) {
         rollback();
@@ -1262,7 +1628,17 @@ export class InMemoryUowDecoder implements UOWDecoder<InMemoryRawResult> {
       if (!relation) {
         continue;
       }
-      output[relationName] = this.decodeRow(relationData[relationName], relation.table, resolver);
+      const relationRow = relationData[relationName];
+      const internalIdKey = relation.table.getInternalIdColumn().name;
+      if (
+        Object.prototype.hasOwnProperty.call(relationRow, internalIdKey) &&
+        isNullish(relationRow[internalIdKey])
+      ) {
+        output[relationName] = relation.type === "many" ? [] : null;
+        continue;
+      }
+
+      output[relationName] = this.decodeRow(relationRow, relation.table, resolver);
     }
 
     for (const key in columnValues) {

package/src/adapters/in-memory/options.test.ts

@@ -1,4 +1,5 @@
 import { afterEach, describe, expect, it, vi } from "vitest";
+
 import { resolveInMemoryAdapterOptions } from "./options";
 
 describe("resolveInMemoryAdapterOptions", () => {

package/src/adapters/in-memory/options.ts

@@ -1,5 +1,7 @@
-import { createId, init } from "@paralleldrive/cuid2";
+import { createId, init } from "@fragno-dev/core";
+
 import type { SqlNamingStrategy } from "../../naming/sql-naming";
+import type { OutboxConfig } from "../../outbox/outbox";
 
 export type InMemoryAdapterOptions = {
   clock?: { now: () => Date };
@@ -19,6 +21,7 @@ export type ResolvedInMemoryAdapterOptions = {
   internalIdGeneratorProvided: boolean;
   enforceConstraints: boolean;
   btreeOrder: number;
+  outbox?: OutboxConfig;
 };
 
 const defaultClock = {
@@ -83,5 +86,6 @@ export const resolveInMemoryAdapterOptions = (
     internalIdGeneratorProvided,
     enforceConstraints: options.enforceConstraints ?? defaultEnforceConstraints,
     btreeOrder: options.btreeOrder ?? defaultBtreeOrder,
+    outbox: undefined,
   };
 };