@fragno-dev/db 0.2.2 → 0.4.1

package/src/fragments/internal-fragment.ts

@@ -1,5 +1,6 @@
 import { FragmentDefinitionBuilder } from "@fragno-dev/core";
 import type { InstantiatedFragmentFromDefinition } from "@fragno-dev/core";
+
 import {
   DatabaseFragmentDefinitionBuilder,
   type DatabaseHandlerContext,
@@ -8,51 +9,94 @@ import {
   type FragnoPublicConfigWithDatabase,
   type ImplicitDatabaseDependencies,
 } from "../db-fragment-definition-builder";
-import type { FragnoId } from "../schema/create";
-import { schema, idColumn, column } from "../schema/create";
+import { isHookStatus, type HookStatus } from "../hooks/hooks";
+import type { Cursor } from "../query/cursor";
+import { dbNow, type DbNow } from "../query/db-now";
 import type { RetryPolicy } from "../query/unit-of-work/retry-policy";
+import { FragnoId } from "../schema/create";
+import {
+  internalSchema,
+  SETTINGS_NAMESPACE,
+  SETTINGS_TABLE_NAME,
+} from "./internal-fragment.schema";
 
-// Constants for Fragno's internal settings table
-export const SETTINGS_TABLE_NAME = "fragno_db_settings" as const;
-// FIXME: In some places we simply use empty string "" as namespace, which is not correct.
-export const SETTINGS_NAMESPACE = "fragno-db-settings" as const;
-
-export const internalSchema = schema((s) => {
-  return s
-    .addTable(SETTINGS_TABLE_NAME, (t) => {
-      return t
-        .addColumn("id", idColumn())
-        .addColumn("key", column("string"))
-        .addColumn("value", column("string"))
-        .createIndex("unique_key", ["key"], { unique: true });
-    })
-    .addTable("fragno_hooks", (t) => {
-      return t
-        .addColumn("id", idColumn())
-        .addColumn("namespace", column("string"))
-        .addColumn("hookName", column("string"))
-        .addColumn("payload", column("json"))
-        .addColumn("status", column("string")) // "pending" | "processing" | "completed" | "failed"
-        .addColumn("attempts", column("integer").defaultTo(0))
-        .addColumn("maxAttempts", column("integer").defaultTo(5))
-        .addColumn("lastAttemptAt", column("timestamp").nullable())
-        .addColumn("nextRetryAt", column("timestamp").nullable())
-        .addColumn("error", column("string").nullable())
-        .addColumn(
-          "createdAt",
-          column("timestamp").defaultTo((b) => b.now()),
-        )
-        .addColumn("nonce", column("string"))
-        .createIndex("idx_namespace_status_retry", ["namespace", "status", "nextRetryAt"])
-        .createIndex("idx_nonce", ["nonce"]);
-    });
-});
+type AdapterRegistry = {
+  listSchemas: () => Array<{
+    name: string;
+    namespace: string | null;
+    version: number;
+    tables: string[];
+  }>;
+  listOutboxFragments: () => Array<{ name: string; mountRoute: string }>;
+  isOutboxEnabled: () => boolean;
+  resolveSyncCommand: (
+    fragmentName: string,
+    schemaName: string,
+    commandName: string,
+  ) => { command: unknown; namespace: string | null } | undefined;
+};
+
+export class SchemaRegistryCollisionError extends Error {
+  readonly code = "SCHEMA_REGISTRY_COLLISION" as const;
+  readonly namespaceKey: string;
+  readonly existing: { name: string; namespace: string | null };
+  readonly attempted: { name: string; namespace: string | null };
+
+  constructor({
+    namespaceKey,
+    existing,
+    attempted,
+  }: {
+    namespaceKey: string;
+    existing: { name: string; namespace: string | null };
+    attempted: { name: string; namespace: string | null };
+  }) {
+    super(
+      `Schema namespace "${namespaceKey}" is already owned by "${existing.name}" (${existing.namespace ?? "null"}).`,
+    );
+    this.name = "SchemaRegistryCollisionError";
+    this.namespaceKey = namespaceKey;
+    this.existing = existing;
+    this.attempted = attempted;
+  }
+}
+
+export type InternalFragmentConfig = {
+  registry?: AdapterRegistry;
+};
+
+export { internalSchema, SETTINGS_NAMESPACE, SETTINGS_TABLE_NAME };
+
+const INTERNAL_SCHEMA_MIN_VERSION = 4;
+if (internalSchema.version < INTERNAL_SCHEMA_MIN_VERSION) {
+  // Keep the internal schema version monotonic after removing fragno_db_schemas.
+  internalSchema.version = INTERNAL_SCHEMA_MIN_VERSION;
+}
+
+const describeHookStatusSource = (event: { id: FragnoId; hookName: string }) =>
+  `fragno_hooks id=${event.id} hook=${event.hookName}`;
+
+const coerceHookStatus = (status: string, context: string): HookStatus => {
+  if (isHookStatus(status)) {
+    return status;
+  }
+  throw new Error(`Invalid hook status from database (${context}): ${status}`);
+};
+
+const DEFAULT_HOOKS_PAGE_SIZE = 50;
+
+const resolveHookPageSize = (pageSize?: number): number => {
+  if (typeof pageSize !== "number" || !Number.isInteger(pageSize) || pageSize <= 0) {
+    return DEFAULT_HOOKS_PAGE_SIZE;
+  }
+  return pageSize;
+};
 
 // This uses DatabaseFragmentDefinitionBuilder directly
 // to avoid circular dependency (it doesn't need to link to itself)
 export const internalFragmentDef = new DatabaseFragmentDefinitionBuilder(
   new FragmentDefinitionBuilder<
-    {},
+    InternalFragmentConfig,
     FragnoPublicConfigWithDatabase,
     ImplicitDatabaseDependencies<typeof internalSchema>,
     {},
@@ -64,7 +108,6 @@ export const internalFragmentDef = new DatabaseFragmentDefinitionBuilder(
     DatabaseRequestStorage
   >("$fragno-internal-fragment"),
   internalSchema,
-  "", // intentionally blank namespace so there is no prefix
 )
   .providesService("settingsService", ({ defineService }) => {
     return defineService({
@@ -111,6 +154,30 @@ export const internalFragmentDef = new DatabaseFragmentDefinitionBuilder(
           .build();
       },
 
+      /**
+       * Set a setting value only if it does not already exist.
+       */
+      setIfMissing(namespace: string, key: string, value: string) {
+        const fullKey = `${namespace}.${key}`;
+        return this.serviceTx(internalSchema)
+          .retrieve((uow) =>
+            uow.findFirst(SETTINGS_TABLE_NAME, (b) =>
+              b.whereIndex("unique_key", (eb) => eb("key", "=", fullKey)),
+            ),
+          )
+          .transformRetrieve(([result]) => result)
+          .mutate(({ uow, retrieveResult }) => {
+            if (retrieveResult) {
+              return;
+            }
+            uow.create(SETTINGS_TABLE_NAME, {
+              key: fullKey,
+              value,
+            });
+          })
+          .build();
+      },
+
       /**
        * Delete a setting by ID.
        */
@@ -128,30 +195,30 @@ export const internalFragmentDef = new DatabaseFragmentDefinitionBuilder(
        * Returns all pending events for the given namespace that are ready to be processed.
        */
       getPendingHookEvents(namespace: string) {
+        const now = dbNow();
         return this.serviceTx(internalSchema)
           .retrieve((uow) =>
             uow.find("fragno_hooks", (b) =>
               b.whereIndex("idx_namespace_status_retry", (eb) =>
-                eb.and(eb("namespace", "=", namespace), eb("status", "=", "pending")),
+                eb.and(
+                  eb("namespace", "=", namespace),
+                  eb("status", "=", "pending"),
+                  eb.or(eb.isNull("nextRetryAt"), eb("nextRetryAt", "<=", now)),
+                ),
               ),
             ),
           )
           .transformRetrieve(([events]) => {
-            const now = new Date();
-            // FIXME(Wilco): this should be handled by the database query, but there seems to be an issue.
-            const ready = events.filter((event) => {
-              if (!event.nextRetryAt) {
-                return true; // Newly created events (nextRetryAt = null) are ready
-              }
-              return event.nextRetryAt <= now; // Only include if retry time has passed
-            });
-
-            return ready.map((event) => ({
+            return events.map((event) => ({
               id: event.id,
               hookName: event.hookName,
               payload: event.payload as unknown,
+              status: coerceHookStatus(event.status, describeHookStatusSource(event)),
               attempts: event.attempts,
               maxAttempts: event.maxAttempts,
+              lastAttemptAt: event.lastAttemptAt,
+              nextRetryAt: event.nextRetryAt,
+              createdAt: event.createdAt,
               idempotencyKey: event.nonce,
             }));
           })
@@ -159,15 +226,224 @@ export const internalFragmentDef = new DatabaseFragmentDefinitionBuilder(
       },
 
       /**
-       * Mark a hook event as completed.
+       * Claim pending hook events for processing.
+       * Returns ready events and marks them as processing in the same transaction.
        */
-      markHookCompleted(eventId: FragnoId) {
+      claimPendingHookEvents(namespace: string) {
+        const now = dbNow();
         return this.serviceTx(internalSchema)
-          .mutate(({ uow }) =>
-            uow.update("fragno_hooks", eventId, (b) =>
-              b.set({ status: "completed", lastAttemptAt: new Date() }).check(),
+          .retrieve((uow) =>
+            uow.find("fragno_hooks", (b) =>
+              b.whereIndex("idx_namespace_status_retry", (eb) =>
+                eb.and(
+                  eb("namespace", "=", namespace),
+                  eb("status", "=", "pending"),
+                  eb.or(eb.isNull("nextRetryAt"), eb("nextRetryAt", "<=", now)),
+                ),
+              ),
+            ),
+          )
+          .transformRetrieve(([events]) => {
+            return events.map((event) => ({
+              id: event.id,
+              hookName: event.hookName,
+              payload: event.payload,
+              status: coerceHookStatus(event.status, describeHookStatusSource(event)),
+              attempts: event.attempts,
+              maxAttempts: event.maxAttempts,
+              lastAttemptAt: event.lastAttemptAt,
+              nextRetryAt: event.nextRetryAt,
+              createdAt: event.createdAt,
+              idempotencyKey: event.nonce,
+            }));
+          })
+          .mutate(({ uow, retrieveResult }) => {
+            if (retrieveResult.length === 0) {
+              return;
+            }
+            for (const event of retrieveResult) {
+              uow.update("fragno_hooks", event.id, (b) =>
+                b.set({ status: "processing", lastAttemptAt: now }).check(),
+              );
+            }
+          })
+          .transform(({ retrieveResult }) =>
+            retrieveResult.map((event) => ({
+              ...event,
+              status: "processing" as const,
+              id: new FragnoId({
+                externalId: event.id.externalId,
+                internalId: event.id.internalId,
+                version: event.id.version + 1,
+              }),
+            })),
+          )
+          .build();
+      },
+
+      /**
+       * Claim stale processing hook events for processing.
+       * Returns ready events and marks them as processing in the same transaction.
+       */
+      claimStuckProcessingHookEvents(namespace: string, staleBefore: DbNow) {
+        const now = dbNow();
+
+        return this.serviceTx(internalSchema)
+          .retrieve((uow) =>
+            uow.find("fragno_hooks", (b) =>
+              b.whereIndex("idx_namespace_status_last_attempt", (eb) =>
+                eb.and(
+                  eb("namespace", "=", namespace),
+                  eb("status", "=", "processing"),
+                  eb.or(eb.isNull("lastAttemptAt"), eb("lastAttemptAt", "<=", staleBefore)),
+                ),
+              ),
             ),
           )
+          .transformRetrieve(([events]) => {
+            return events.map((event) => ({
+              id: event.id,
+              hookName: event.hookName,
+              payload: event.payload as unknown,
+              status: coerceHookStatus(event.status, describeHookStatusSource(event)),
+              attempts: event.attempts,
+              maxAttempts: event.maxAttempts,
+              idempotencyKey: event.nonce,
+              lastAttemptAt: event.lastAttemptAt,
+              nextRetryAt: event.nextRetryAt,
+              createdAt: event.createdAt,
+            }));
+          })
+          .mutate(({ uow, retrieveResult }) => {
+            if (retrieveResult.length === 0) {
+              return;
+            }
+
+            for (const event of retrieveResult) {
+              uow.update("fragno_hooks", event.id, (b) =>
+                b.set({ status: "processing", lastAttemptAt: now, nextRetryAt: null }).check(),
+              );
+            }
+          })
+          .transform(({ retrieveResult }) => {
+            return {
+              events: retrieveResult.map((event) => ({
+                ...event,
+                id: new FragnoId({
+                  externalId: event.id.externalId,
+                  internalId: event.id.internalId,
+                  version: event.id.version + 1,
+                }),
+              })),
+              stuckEvents: retrieveResult.map((event) => ({
+                id: event.id,
+                hookName: event.hookName,
+                attempts: event.attempts,
+                maxAttempts: event.maxAttempts,
+                lastAttemptAt: event.lastAttemptAt,
+                nextRetryAt: event.nextRetryAt,
+              })),
+            };
+          })
+          .build();
+      },
+
+      /**
+       * Get the earliest pending hook wake time for a namespace.
+       * Optionally considers processing hooks becoming stale when timeoutMinutes is provided.
+       */
+      getNextHookWakeAt(namespace: string, timeoutMinutes?: number | false) {
+        const timeoutMinutesValue =
+          typeof timeoutMinutes === "number" && timeoutMinutes > 0 ? timeoutMinutes : 0;
+        const includeProcessing = timeoutMinutesValue > 0;
+        const now = dbNow();
+        const timeoutMs = timeoutMinutesValue * 60_000;
+        // Sentinel to keep query shape stable when processing checks are disabled.
+        const processingStatus = includeProcessing ? "processing" : "__disabled__";
+        const staleBefore = now.plus({ minutes: -timeoutMinutesValue });
+
+        return this.serviceTx(internalSchema)
+          .retrieve((uow) =>
+            uow
+              .forSchema(internalSchema)
+              .find("fragno_hooks", (b) =>
+                b
+                  .whereIndex("idx_namespace_status_retry", (eb) =>
+                    eb.and(
+                      eb("namespace", "=", namespace),
+                      eb("status", "=", "pending"),
+                      eb.or(eb.isNull("nextRetryAt"), eb("nextRetryAt", "<=", now)),
+                    ),
+                  )
+                  .pageSize(1),
+              )
+              .find("fragno_hooks", (b) =>
+                b
+                  .whereIndex("idx_namespace_status_retry", (eb) =>
+                    eb.and(
+                      eb("namespace", "=", namespace),
+                      eb("status", "=", "pending"),
+                      eb.isNotNull("nextRetryAt"),
+                      eb("nextRetryAt", ">", now),
+                    ),
+                  )
+                  .orderByIndex("idx_namespace_status_retry", "asc")
+                  .pageSize(1)
+                  .select(["nextRetryAt"]),
+              )
+              .find("fragno_hooks", (b) =>
+                b
+                  .whereIndex("idx_namespace_status_last_attempt", (eb) =>
+                    eb.and(
+                      eb("namespace", "=", namespace),
+                      eb("status", "=", processingStatus),
+                      eb.or(eb.isNull("lastAttemptAt"), eb("lastAttemptAt", "<=", staleBefore)),
+                    ),
+                  )
+                  .pageSize(1),
+              )
+              .find("fragno_hooks", (b) =>
+                b
+                  .whereIndex("idx_namespace_status_last_attempt", (eb) =>
+                    eb.and(
+                      eb("namespace", "=", namespace),
+                      eb("status", "=", processingStatus),
+                      eb.isNotNull("lastAttemptAt"),
+                      eb("lastAttemptAt", ">", staleBefore),
+                    ),
+                  )
+                  .orderByIndex("idx_namespace_status_last_attempt", "asc")
+                  .pageSize(1)
+                  .select(["lastAttemptAt"]),
+              ),
+          )
+          .transformRetrieve(
+            ([pendingImmediate, pendingNext, processingImmediate, processingNext]) => {
+              const hasProcessingImmediate = includeProcessing && processingImmediate.length > 0;
+
+              if (pendingImmediate.length > 0 || hasProcessingImmediate) {
+                return new Date();
+              }
+
+              const pendingNextAt = pendingNext[0]?.nextRetryAt ?? null;
+              let processingNextAt: Date | null = null;
+
+              if (includeProcessing) {
+                const lastAttemptAt = processingNext[0]?.lastAttemptAt;
+                if (lastAttemptAt) {
+                  processingNextAt = new Date(lastAttemptAt.getTime() + timeoutMs);
+                }
+              }
+
+              if (!pendingNextAt) {
+                return processingNextAt ?? null;
+              }
+              if (!processingNextAt) {
+                return pendingNextAt;
+              }
+              return pendingNextAt <= processingNextAt ? pendingNextAt : processingNextAt;
+            },
+          )
           .build();
       },
 
@@ -177,18 +453,19 @@ export const internalFragmentDef = new DatabaseFragmentDefinitionBuilder(
       markHookFailed(eventId: FragnoId, error: string, attempts: number, retryPolicy: RetryPolicy) {
         const newAttempts = attempts + 1;
         const shouldRetry = retryPolicy.shouldRetry(newAttempts - 1);
+        const now = dbNow();
 
         return this.serviceTx(internalSchema)
           .mutate(({ uow }) => {
             if (shouldRetry) {
               const delayMs = retryPolicy.getDelayMs(newAttempts - 1);
-              const nextRetryAt = new Date(Date.now() + delayMs);
+              const nextRetryAt = now.plus({ ms: delayMs });
               uow.update("fragno_hooks", eventId, (b) =>
                 b
                   .set({
                     status: "pending",
                     attempts: newAttempts,
-                    lastAttemptAt: new Date(),
+                    lastAttemptAt: now,
                     nextRetryAt,
                     error,
                   })
@@ -200,7 +477,7 @@ export const internalFragmentDef = new DatabaseFragmentDefinitionBuilder(
                   .set({
                     status: "failed",
                     attempts: newAttempts,
-                    lastAttemptAt: new Date(),
+                    lastAttemptAt: now,
                     error,
                   })
                   .check(),
@@ -217,7 +494,7 @@ export const internalFragmentDef = new DatabaseFragmentDefinitionBuilder(
         return this.serviceTx(internalSchema)
           .mutate(({ uow }) =>
             uow.update("fragno_hooks", eventId, (b) =>
-              b.set({ status: "processing", lastAttemptAt: new Date() }).check(),
+              b.set({ status: "processing", lastAttemptAt: dbNow() }).check(),
             ),
           )
           .build();
@@ -233,7 +510,45 @@ export const internalFragmentDef = new DatabaseFragmentDefinitionBuilder(
               b.whereIndex("primary", (eb) => eb("id", "=", eventId)),
             ),
           )
-          .transformRetrieve(([result]) => result ?? undefined)
+          .transformRetrieve(([result]) =>
+            result
+              ? {
+                  ...result,
+                  status: coerceHookStatus(result.status, describeHookStatusSource(result)),
+                }
+              : undefined,
+          )
+          .build();
+      },
+
+      /**
+       * Get hook events for a namespace in newest-first order with pagination.
+       */
+      getHooksByNamespacePage(
+        namespace: string,
+        options: { cursor?: Cursor | string; pageSize?: number } = {},
+      ) {
+        const pageSize = resolveHookPageSize(options.pageSize);
+
+        return this.serviceTx(internalSchema)
+          .retrieve((uow) =>
+            uow.findWithCursor("fragno_hooks", (b) => {
+              const query = b
+                .whereIndex("idx_namespace_created_at", (eb) => eb("namespace", "=", namespace))
+                .orderByIndex("idx_namespace_created_at", "desc")
+                .pageSize(pageSize);
+
+              return options.cursor ? query.after(options.cursor) : query;
+            }),
+          )
+          .transformRetrieve(([page]) => ({
+            items: page.items.map((event) => ({
+              ...event,
+              status: coerceHookStatus(event.status, describeHookStatusSource(event)),
+            })),
+            cursor: page.cursor,
+            hasNextPage: page.hasNextPage,
+          }))
           .build();
       },
 
@@ -247,7 +562,50 @@ export const internalFragmentDef = new DatabaseFragmentDefinitionBuilder(
               b.whereIndex("idx_namespace_status_retry", (eb) => eb("namespace", "=", namespace)),
             ),
           )
-          .transformRetrieve(([events]) => events)
+          .transformRetrieve(([events]) =>
+            events.map((event) => ({
+              ...event,
+              status: coerceHookStatus(event.status, describeHookStatusSource(event)),
+            })),
+          )
+          .build();
+      },
+    });
+  })
+  .providesService("outboxService", ({ defineService }) => {
+    return defineService({
+      /**
+       * List outbox entries ordered by versionstamp (ascending).
+       */
+      list({ afterVersionstamp, limit }: { afterVersionstamp?: string; limit?: number } = {}) {
+        const afterValue = afterVersionstamp?.toLowerCase();
+
+        return this.serviceTx(internalSchema)
+          .retrieve((uow) =>
+            uow.find("fragno_db_outbox", (b) => {
+              let builder = afterValue
+                ? b.whereIndex("idx_outbox_versionstamp", (eb) =>
+                    eb("versionstamp", ">", afterValue),
+                  )
+                : b.whereIndex("idx_outbox_versionstamp");
+
+              builder = builder.orderByIndex("idx_outbox_versionstamp", "asc");
+              if (limit !== undefined) {
+                builder = builder.pageSize(limit);
+              }
+              return builder;
+            }),
+          )
+          .transformRetrieve(([entries]) =>
+            entries.map((entry) => ({
+              id: entry.id,
+              versionstamp: entry.versionstamp,
+              uowId: entry.uowId,
+              payload: entry.payload,
+              refMap: entry.refMap ?? undefined,
+              createdAt: entry.createdAt,
+            })),
+          )
           .build();
       },
     });
@@ -267,15 +625,40 @@ export async function getSchemaVersionFromDatabase(
   namespace: string,
 ): Promise<number> {
   try {
-    const setting = await fragment.inContext(async function () {
-      return await this.handlerTx()
-        .withServiceCalls(
-          () => [fragment.services.settingsService.get(namespace, "schema_version")] as const,
-        )
-        .transform(({ serviceResult: [result] }) => result)
-        .execute();
-    });
-    return setting ? parseInt(setting.value, 10) : 0;
+    const readSchemaVersion = async (targetNamespace: string): Promise<number | undefined> => {
+      const setting = await fragment.inContext(async function () {
+        return await this.handlerTx()
+          .withServiceCalls(
+            () =>
+              [fragment.services.settingsService.get(targetNamespace, "schema_version")] as const,
+          )
+          .transform(({ serviceResult: [result] }) => result)
+          .execute();
+      });
+      if (!setting) {
+        return undefined;
+      }
+      const parsed = parseInt(setting.value, 10);
+      return Number.isNaN(parsed) ? undefined : parsed;
+    };
+
+    const primary = await readSchemaVersion(namespace);
+    if (primary !== undefined) {
+      return primary;
+    }
+
+    // Back-compat: some installs stored internal schema version under a different namespace.
+    // Check the alternate key (empty string ↔ schema name) so we find the version either way.
+    const legacyNamespace =
+      namespace === "" ? internalSchema.name : namespace === internalSchema.name ? "" : null;
+    if (legacyNamespace !== null) {
+      const legacy = await readSchemaVersion(legacyNamespace);
+      if (legacy !== undefined) {
+        return legacy;
+      }
+    }
+
+    return 0;
   } catch {
     return 0;
   }