@restura/core 2.0.2 → 2.1.0

package/dist/index.d.ts

@@ -1,7 +1,7 @@
 import { UUID } from 'crypto';
+import { z } from 'zod';
 import * as express from 'express';
 import { IncomingHttpHeaders } from 'http2';
-import { z } from 'zod';
 import { QueryResultRow, QueryConfigValues, QueryResult, PoolConfig, Pool, ClientConfig, Client } from 'pg';
 import peg from 'pegjs';
 
@@ -18,51 +18,6 @@ interface ResturaLogger {
 declare const logger: ResturaLogger;
 declare function setLogger(impl: ResturaLogger): void;
 
-/**
- * @deprecated This is used for passing around until we finally get to the sending externally
- * // TODO: Remove once backwards compatibility is no longer needed
- */
-interface RsErrorInternalData<T extends Record<string, unknown> = Record<string, unknown>> {
-    err: ErrorCode;
-    msg: string;
-    stack: string;
-    status: number;
-    options?: T;
-}
-declare enum HtmlStatusCodes {
-    BAD_REQUEST = 400,
-    UNAUTHORIZED = 401,
-    PAYMENT_REQUIRED = 402,
-    FORBIDDEN = 403,
-    NOT_FOUND = 404,
-    METHOD_NOT_ALLOWED = 405,
-    REQUEST_TIMEOUT = 408,
-    CONFLICT = 409,
-    GONE = 410,
-    PAYLOAD_TOO_LARGE = 413,
-    UNSUPPORTED_MEDIA_TYPE = 415,
-    UPGRADE_REQUIRED = 426,
-    UNPROCESSABLE_ENTITY = 422,
-    TOO_MANY_REQUESTS = 429,
-    SERVER_ERROR = 500,
-    NOT_IMPLEMENTED = 501,
-    BAD_GATEWAY = 502,
-    SERVICE_UNAVAILABLE = 503,
-    GATEWAY_TIMEOUT = 504,
-    NETWORK_CONNECT_TIMEOUT = 599
-}
-type ErrorCode = 'BAD_REQUEST' | 'UNAUTHORIZED' | 'PAYMENT_REQUIRED' | 'FORBIDDEN' | 'NOT_FOUND' | 'METHOD_NOT_ALLOWED' | 'REQUEST_TIMEOUT' | 'CONFLICT' | 'GONE' | 'PAYLOAD_TOO_LARGE' | 'UNSUPPORTED_MEDIA_TYPE' | 'UPGRADE_REQUIRED' | 'UNPROCESSABLE_ENTITY' | 'TOO_MANY_REQUESTS' | 'SERVER_ERROR' | 'NOT_IMPLEMENTED' | 'BAD_GATEWAY' | 'SERVICE_UNAVAILABLE' | 'GATEWAY_TIMEOUT' | 'NETWORK_CONNECT_TIMEOUT' | 'UNKNOWN_ERROR' | 'RATE_LIMIT_EXCEEDED' | 'INVALID_TOKEN' | 'INCORRECT_EMAIL_OR_PASSWORD' | 'DUPLICATE' | 'CONNECTION_ERROR' | 'SCHEMA_ERROR' | 'DATABASE_ERROR';
-declare class RsError<T extends Record<string, unknown> = Record<string, unknown>> extends Error {
-    err: ErrorCode;
-    msg: string;
-    options?: T;
-    status?: number;
-    constructor(errCode: ErrorCode, message?: string, options?: T);
-    toJSON(): Record<string, unknown>;
-    static htmlStatus(code: ErrorCode): number;
-    static isRsError(error: unknown): error is RsError;
-}
-
 declare const whereDataSchema: z.ZodObject<{
     tableName: z.ZodOptional<z.ZodString>;
     columnName: z.ZodOptional<z.ZodString>;
@@ -869,6 +824,51 @@ declare const resturaSchema: z.ZodObject<{
 type ResturaSchema = z.infer<typeof resturaSchema>;
 declare function isSchemaValid(schemaToCheck: unknown): Promise<boolean>;
 
+/**
+ * @deprecated This is used for passing around until we finally get to the sending externally
+ * // TODO: Remove once backwards compatibility is no longer needed
+ */
+interface RsErrorInternalData<T extends Record<string, unknown> = Record<string, unknown>> {
+    err: ErrorCode;
+    msg: string;
+    stack: string;
+    status: number;
+    options?: T;
+}
+declare enum HtmlStatusCodes {
+    BAD_REQUEST = 400,
+    UNAUTHORIZED = 401,
+    PAYMENT_REQUIRED = 402,
+    FORBIDDEN = 403,
+    NOT_FOUND = 404,
+    METHOD_NOT_ALLOWED = 405,
+    REQUEST_TIMEOUT = 408,
+    CONFLICT = 409,
+    GONE = 410,
+    PAYLOAD_TOO_LARGE = 413,
+    UNSUPPORTED_MEDIA_TYPE = 415,
+    UPGRADE_REQUIRED = 426,
+    UNPROCESSABLE_ENTITY = 422,
+    TOO_MANY_REQUESTS = 429,
+    SERVER_ERROR = 500,
+    NOT_IMPLEMENTED = 501,
+    BAD_GATEWAY = 502,
+    SERVICE_UNAVAILABLE = 503,
+    GATEWAY_TIMEOUT = 504,
+    NETWORK_CONNECT_TIMEOUT = 599
+}
+type ErrorCode = 'BAD_REQUEST' | 'UNAUTHORIZED' | 'PAYMENT_REQUIRED' | 'FORBIDDEN' | 'NOT_FOUND' | 'METHOD_NOT_ALLOWED' | 'REQUEST_TIMEOUT' | 'CONFLICT' | 'GONE' | 'PAYLOAD_TOO_LARGE' | 'UNSUPPORTED_MEDIA_TYPE' | 'UPGRADE_REQUIRED' | 'UNPROCESSABLE_ENTITY' | 'TOO_MANY_REQUESTS' | 'SERVER_ERROR' | 'NOT_IMPLEMENTED' | 'BAD_GATEWAY' | 'SERVICE_UNAVAILABLE' | 'GATEWAY_TIMEOUT' | 'NETWORK_CONNECT_TIMEOUT' | 'UNKNOWN_ERROR' | 'RATE_LIMIT_EXCEEDED' | 'INVALID_TOKEN' | 'INCORRECT_EMAIL_OR_PASSWORD' | 'DUPLICATE' | 'CONNECTION_ERROR' | 'SCHEMA_ERROR' | 'DATABASE_ERROR';
+declare class RsError<T extends Record<string, unknown> = Record<string, unknown>> extends Error {
+    err: ErrorCode;
+    msg: string;
+    options?: T;
+    status?: number;
+    constructor(errCode: ErrorCode, message?: string, options?: T);
+    toJSON(): Record<string, unknown>;
+    static htmlStatus(code: ErrorCode): number;
+    static isRsError(error: unknown): error is RsError;
+}
+
 interface SchemaChangeValue {
     name: string;
     changeType: 'NEW' | 'MODIFIED' | 'DELETED';
@@ -991,18 +991,23 @@ type TriggerResult = {
     previousRecord: DynamicObject;
     requesterId: number;
 };
-type QueryMetadata = RequesterDetails & {
-    connectionInstanceId: UUID;
+type QueryMetadata = Partial<RequesterDetails> & DynamicObject & {
+    connectionInstanceId?: UUID;
 };
+type NotifyValidationMode = 'off' | 'warn' | 'strict';
+interface FireActionOptions {
+    rethrowHandlerErrors?: boolean;
+}
 declare class EventManager {
     private actionHandlers;
     addRowInsertHandler<T>(onInsert: (data: ActionRowInsertData<T>, queryMetadata: QueryMetadata) => Promise<void>, filter?: ActionRowInsertFilter): void;
     addColumnChangeHandler<T>(onUpdate: (data: ActionColumnChangeData<T>, queryMetadata: QueryMetadata) => Promise<void>, filter: ActionColumnChangeFilter): void;
     addRowDeleteHandler<T>(onDelete: (data: ActionRowDeleteData<T>, queryMetadata: QueryMetadata) => Promise<void>, filter?: ActionRowDeleteFilter): void;
-    fireActionFromDbTrigger(sqlMutationData: SqlMutationData, result: TriggerResult): Promise<void>;
+    fireActionFromDbTrigger(sqlMutationData: SqlMutationData, result: TriggerResult, options?: FireActionOptions): Promise<void>;
     private fireInsertActions;
     private fireDeleteActions;
     private fireUpdateActions;
+    validateHandlersAgainstSchema(schema: ResturaSchema, mode: NotifyValidationMode): void;
     private hasHandlersForEventType;
 }
 declare const eventManager: EventManager;
@@ -1021,12 +1026,32 @@ declare const resturaConfigSchema: z.ZodObject<{
     generatedTypesPath: z.ZodDefault<z.ZodString>;
     fileTempCachePath: z.ZodOptional<z.ZodString>;
     scratchDatabaseSuffix: z.ZodOptional<z.ZodString>;
+    eventDelivery: z.ZodDefault<z.ZodEnum<{
+        direct: "direct";
+        outbox: "outbox";
+    }>>;
+    eventOutbox: z.ZodPrefault<z.ZodObject<{
+        channel: z.ZodDefault<z.ZodString>;
+        pollIntervalMs: z.ZodDefault<z.ZodNumber>;
+        batchSize: z.ZodDefault<z.ZodNumber>;
+        maxAttempts: z.ZodDefault<z.ZodNumber>;
+        pruneAfterDays: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strip>>;
+    notifyValidation: z.ZodDefault<z.ZodEnum<{
+        warn: "warn";
+        off: "off";
+        strict: "strict";
+    }>>;
+    queryMetadataKeys: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<"ALL">, z.ZodArray<z.ZodString>]>>;
 }, z.core.$strip>;
 type ResturaConfigSchema = z.infer<typeof resturaConfigSchema>;
 
 declare abstract class PsqlConnection {
+    private static queryMetadataKeys;
+    static setQueryMetadataKeys(keys: string[] | 'ALL'): void;
     readonly instanceId: UUID;
     protected constructor(instanceId?: UUID);
+    private buildQueryMetadata;
     protected abstract query<R extends QueryResultRow = QueryResultRow, T extends Array<unknown> = unknown[]>(query: string, values?: QueryConfigValues<T>): Promise<QueryResult<R>>;
     queryOne<T>(query: string, options: unknown[], requesterDetails: RequesterDetails): Promise<T>;
     queryOneSchema<T>(query: string, params: unknown[], requesterDetails: RequesterDetails, zodSchema: z.ZodSchema<T>): Promise<T>;
@@ -1167,27 +1192,97 @@ declare abstract class SqlEngine {
     protected abstract executeDeleteRequest(req: RsRequest<unknown>, routeData: StandardRouteData, schema: ResturaSchema): Promise<boolean>;
 }
 
+interface EventOutboxOptions {
+    channel: string;
+    pollIntervalMs: number;
+    batchSize: number;
+    maxAttempts: number;
+    pruneAfterDays: number;
+}
+declare const defaultEventOutboxOptions: EventOutboxOptions;
+interface EventDeliveryConfig {
+    mode: 'direct' | 'outbox';
+    outbox?: Partial<EventOutboxOptions>;
+}
+interface OutboxRow {
+    id: number;
+    createdOn: string;
+    tableName: string;
+    operation: MutationType;
+    recordId: number | null;
+    record: DynamicObject | null;
+    previousRecord: DynamicObject | null;
+    queryMetadata: QueryMetadata | null;
+    processedOn: string | null;
+    attempts: number;
+    nextAttemptOn: string | null;
+    isDeadLetter: boolean;
+}
+declare class EventOutboxConsumer {
+    private psqlConnectionPool;
+    private options;
+    private isDraining;
+    private drainRequested;
+    private isStopped;
+    private pollTimer;
+    private pruneTimer;
+    private activeDrain;
+    private lastDrainOn;
+    constructor(psqlConnectionPool: PsqlPool, options?: Partial<EventOutboxOptions>);
+    get channel(): string;
+    start(): void;
+    stop(): Promise<void>;
+    /** Safe to call at any frequency; overlapping calls coalesce into one extra pass. */
+    drain(): Promise<void>;
+    getStats(): Promise<{
+        pendingCount: number;
+        deadLetterCount: number;
+        lastDrainOn: string | null;
+    }>;
+    private processBatch;
+    private processRow;
+    private prune;
+}
+
+interface EventListenerHealth {
+    connected: boolean;
+    lastHeartbeatOn: string | null;
+    reconnectAttempts: number;
+}
 declare class PsqlEngine extends SqlEngine {
     private psqlConnectionPool;
     setupTriggerListeners: Promise<void> | undefined;
     private triggerClient;
     private scratchDbName;
     private reconnectAttempts;
-    private readonly MAX_RECONNECT_ATTEMPTS;
     private readonly INITIAL_RECONNECT_DELAY;
-    constructor(psqlConnectionPool: PsqlPool, shouldListenForDbTriggers?: boolean, scratchDatabaseSuffix?: string);
+    private readonly MAX_RECONNECT_DELAY;
+    private readonly HEARTBEAT_INTERVAL_MS;
+    private eventDelivery;
+    private outboxConsumer;
+    private heartbeatTimer;
+    private isListenerConnected;
+    private isReconnecting;
+    private isClosed;
+    private lastHeartbeatOn;
+    constructor(psqlConnectionPool: PsqlPool, shouldListenForDbTriggers?: boolean, scratchDatabaseSuffix?: string, eventDelivery?: EventDeliveryConfig);
     close(): Promise<void>;
+    getEventListenerHealth(): EventListenerHealth;
+    getOutboxConsumer(): EventOutboxConsumer | undefined;
     /**
      * Setup the return types for the PostgreSQL connection.
      * For example return DATE as a string instead of a Date object and BIGINT as a number instead of a string.
      */
     private setupPgReturnTypes;
     private reconnectTriggerClient;
+    private startHeartbeat;
     private listenForDbTriggers;
+    private warnIfOutboxBacklogInDirectMode;
     private handleTrigger;
     createDatabaseFromSchema(schema: ResturaSchema, connection: PsqlPool): Promise<string>;
     generateDatabaseSchemaFromSchema(schema: ResturaSchema): string;
     diffDatabaseToSchema(schema: ResturaSchema): Promise<string>;
+    private schemaGenerationOptions;
     protected createNestedSelect(req: RsRequest<unknown>, schema: ResturaSchema, item: ResponseData, routeData: StandardRouteData, sqlParams: string[]): string;
     protected executeCreateRequest(req: RsRequest<unknown>, routeData: StandardRouteData, schema: ResturaSchema): Promise<DynamicObject>;
     protected executeGetRequest(req: RsRequest<unknown>, routeData: StandardRouteData, schema: ResturaSchema): Promise<DynamicObject | any[]>;
@@ -1256,12 +1351,26 @@ declare function diffSchemaToDatabase(schema: ResturaSchema, snapshot: DbSnapsho
 
 declare const systemUser: RequesterDetails;
 declare function schemaToPsqlType(column: ColumnData): string;
-declare function createInsertTriggerSql(tableName: string, notify: ResturaSchema['database'][0]['notify']): string;
-declare function createUpdateTriggerSql(tableName: string, notify: ResturaSchema['database'][0]['notify']): string;
-declare function createDeleteTriggerSql(tableName: string, notify: ResturaSchema['database'][0]['notify']): string;
-declare function generateDatabaseSchemaFromSchema(schema: ResturaSchema): string;
+declare const OUTBOX_TABLE_NAME = "dbEventOutbox";
+declare const DEFAULT_OUTBOX_CHANNEL = "restura_outbox";
+declare function isSensitiveColumnName(columnName: string): boolean;
+interface TriggerSqlOptions {
+    delivery?: 'direct' | 'outbox';
+    channel?: string;
+    tableColumns?: string[];
+}
+interface SchemaGenerationOptions {
+    eventDelivery?: 'direct' | 'outbox';
+    outboxChannel?: string;
+}
+declare function createOutboxTableSql(): string;
+declare function createInsertTriggerSql(tableName: string, notify: ResturaSchema['database'][0]['notify'], options?: TriggerSqlOptions): string;
+declare function createUpdateTriggerSql(tableName: string, notify: ResturaSchema['database'][0]['notify'], options?: TriggerSqlOptions): string;
+declare function createDeleteTriggerSql(tableName: string, notify: ResturaSchema['database'][0]['notify'], options?: TriggerSqlOptions): string;
+declare function generateNotifyTriggersSql(schema: ResturaSchema, options?: SchemaGenerationOptions): string[];
+declare function generateDatabaseSchemaFromSchema(schema: ResturaSchema, options?: SchemaGenerationOptions): string;
 declare function getNewPublicSchemaAndScratchPool(targetPool: PsqlPool, scratchDbName: string): Promise<PsqlPool>;
-declare function diffDatabaseToSchema(schema: ResturaSchema, targetPool: PsqlPool, scratchDbName: string): Promise<string>;
+declare function diffDatabaseToSchema(schema: ResturaSchema, targetPool: PsqlPool, scratchDbName: string, options?: SchemaGenerationOptions): Promise<string>;
 
 declare class PsqlTransaction extends PsqlConnection {
     clientConfig: ClientConfig;
@@ -1340,4 +1449,4 @@ declare function SQL(strings: TemplateStringsArray, ...values: unknown[]): strin
  */
 declare function toSqlLiteral(value: unknown): string;
 
-export { type ActionColumnChangeData, type ActionColumnChangeFilter, type ActionRowDeleteData, type ActionRowDeleteFilter, type ActionRowInsertData, type ActionRowInsertFilter, type ApiMethod, type AsyncExpressApplication, type AuthenticateHandler, type AuthenticatedRequesterDetails, type ConjunctionTypes, type DatabaseActionData, type DbCheckConstraint, type DbColumn, type DbForeignKey, type DbIndex, type DbSnapshot, type DbTable, type DynamicObject, type ErrorCode, type EventType, HtmlStatusCodes, type LogLevel, type MatchTypes, type MutationType, type OnValidAuthenticationCallback, type PageQuery, PsqlConnection, PsqlEngine, PsqlPool, PsqlTransaction, type QueryMetadata, type RequesterDetails, type ResturaConfigSchema, type ResturaLogger, type ResturaSchema, RsError, type RsErrorData, type RsErrorInternalData, type RsHeaders, type RsPagedResponseData, type RsRequest, type RsResponse, type RsResponseData, type RsRouteHandler, SQL, type SchemaChangeValue, type SchemaPreview, type SqlMutationData, type StandardOrderTypes, type TriggerResult, apiGenerator, createDeleteTriggerSql, createInsertTriggerSql, createUpdateTriggerSql, diffDatabaseToSchema, diffSchemaToDatabase, escapeColumnName, eventManager, filterPsqlParser, generateDatabaseSchemaFromSchema, getNewPublicSchemaAndScratchPool, insertObjectQuery, introspectDatabase, isSchemaValid, isValueNumber, logger, modelGenerator, questionMarksToOrderedParams, restura, resturaGlobalTypesGenerator, resturaSchema, schemaToPsqlType, setLogger, systemUser, toSqlLiteral, updateObjectQuery };
+export { type ActionColumnChangeData, type ActionColumnChangeFilter, type ActionRowDeleteData, type ActionRowDeleteFilter, type ActionRowInsertData, type ActionRowInsertFilter, type ApiMethod, type AsyncExpressApplication, type AuthenticateHandler, type AuthenticatedRequesterDetails, type ConjunctionTypes, DEFAULT_OUTBOX_CHANNEL, type DatabaseActionData, type DbCheckConstraint, type DbColumn, type DbForeignKey, type DbIndex, type DbSnapshot, type DbTable, type DynamicObject, type ErrorCode, type EventDeliveryConfig, type EventListenerHealth, EventManager, EventOutboxConsumer, type EventOutboxOptions, type EventType, type FireActionOptions, HtmlStatusCodes, type LogLevel, type MatchTypes, type MutationType, type NotifyValidationMode, OUTBOX_TABLE_NAME, type OnValidAuthenticationCallback, type OutboxRow, type PageQuery, PsqlConnection, PsqlEngine, PsqlPool, PsqlTransaction, type QueryMetadata, type RequesterDetails, type ResturaConfigSchema, type ResturaLogger, type ResturaSchema, RsError, type RsErrorData, type RsErrorInternalData, type RsHeaders, type RsPagedResponseData, type RsRequest, type RsResponse, type RsResponseData, type RsRouteHandler, SQL, type SchemaChangeValue, type SchemaGenerationOptions, type SchemaPreview, type SqlMutationData, type StandardOrderTypes, type TriggerResult, type TriggerSqlOptions, apiGenerator, createDeleteTriggerSql, createInsertTriggerSql, createOutboxTableSql, createUpdateTriggerSql, defaultEventOutboxOptions, diffDatabaseToSchema, diffSchemaToDatabase, escapeColumnName, eventManager, filterPsqlParser, generateDatabaseSchemaFromSchema, generateNotifyTriggersSql, getNewPublicSchemaAndScratchPool, insertObjectQuery, introspectDatabase, isSchemaValid, isSensitiveColumnName, isValueNumber, logger, modelGenerator, questionMarksToOrderedParams, restura, resturaGlobalTypesGenerator, resturaSchema, schemaToPsqlType, setLogger, systemUser, toSqlLiteral, updateObjectQuery };