@webiny/api-event-handler-standalone 0.0.0-unstable.5387e5f7c8 → 0.0.0-unstable.9609e4e0c9

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@webiny/api-event-handler-standalone",
-	"version": "0.0.0-unstable.5387e5f7c8",
+	"version": "0.0.0-unstable.9609e4e0c9",
 	"type": "module",
 	"main": "index.js",
 	"exports": {
@@ -18,29 +18,30 @@
 	],
 	"license": "MIT",
 	"dependencies": {
-		"@webiny/api-core": "0.0.0-unstable.5387e5f7c8",
-		"@webiny/api-event-handler-core": "0.0.0-unstable.5387e5f7c8",
-		"@webiny/api-file-manager-standalone": "0.0.0-unstable.5387e5f7c8",
-		"@webiny/api-graphql": "0.0.0-unstable.5387e5f7c8",
-		"@webiny/api-headless-cms-bulk-actions-standalone": "0.0.0-unstable.5387e5f7c8",
-		"@webiny/api-scheduler": "0.0.0-unstable.5387e5f7c8",
-		"@webiny/api-scheduler-standalone": "0.0.0-unstable.5387e5f7c8",
-		"@webiny/api-websockets-standalone": "0.0.0-unstable.5387e5f7c8",
-		"@webiny/background-tasks-standalone": "0.0.0-unstable.5387e5f7c8",
-		"@webiny/event-handler-core": "0.0.0-unstable.5387e5f7c8",
-		"@webiny/event-handler-standalone": "0.0.0-unstable.5387e5f7c8",
-		"@webiny/feature": "0.0.0-unstable.5387e5f7c8",
-		"@webiny/handler": "0.0.0-unstable.5387e5f7c8"
+		"@webiny/api-core": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/api-event-handler-core": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/api-file-manager-standalone": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/api-graphql": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/api-headless-cms-bulk-actions-standalone": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/api-scheduler": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/api-scheduler-standalone": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/api-websockets-standalone": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/api-websockets-standalone-redis": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/background-tasks-standalone": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/event-handler-core": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/event-handler-standalone": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/feature": "0.0.0-unstable.9609e4e0c9",
+		"@webiny/handler": "0.0.0-unstable.9609e4e0c9"
 	},
 	"peerDependencies": {
 		"@webiny/di": "^2.0.0"
 	},
 	"devDependencies": {
-		"@webiny/build-tools": "0.0.0-unstable.5387e5f7c8",
+		"@webiny/build-tools": "0.0.0-unstable.9609e4e0c9",
 		"@webiny/di": "2.0.0",
 		"rimraf": "6.1.3",
 		"typescript": "7.0.2",
-		"vitest": "4.1.11"
+		"vitest": "5.0.5"
 	},
 	"publishConfig": {
 		"access": "public"

package/publicUrls/ConfigPublicUrls.d.ts

@@ -0,0 +1,13 @@
+import { PublicUrls } from "@webiny/api-core/features/publicUrls/index.js";
+import { StandaloneConfig } from "../config/abstractions.js";
+/** Standalone: the URLs come from `WEBINY_API_URL` / `WEBINY_ADMIN_URL`, read at startup. */
+declare class ConfigPublicUrlsImpl implements PublicUrls.Interface {
+    private config;
+    constructor(config: StandaloneConfig.Interface);
+    getApiUrl(): Promise<import("@webiny/api-core/features/publicUrls/PublicUrl").PublicUrl>;
+    getAdminUrl(): Promise<import("@webiny/api-core/features/publicUrls/PublicUrl").PublicUrl>;
+}
+export declare const ConfigPublicUrls: typeof ConfigPublicUrlsImpl & {
+    __abstraction: import("@webiny/di").Abstraction<import("@webiny/api-core/features/publicUrls/abstractions").IPublicUrls>;
+};
+export {};

package/publicUrls/ConfigPublicUrls.js

@@ -0,0 +1,22 @@
+import { PublicUrls } from "@webiny/api-core/features/publicUrls/index.js";
+import { StandaloneConfig } from "../config/abstractions.js";
+class ConfigPublicUrlsImpl {
+    constructor(config){
+        this.config = config;
+    }
+    async getApiUrl() {
+        return this.config.apiUrl;
+    }
+    async getAdminUrl() {
+        return this.config.adminUrl;
+    }
+}
+const ConfigPublicUrls = PublicUrls.createImplementation({
+    implementation: ConfigPublicUrlsImpl,
+    dependencies: [
+        StandaloneConfig
+    ]
+});
+export { ConfigPublicUrls };
+
+//# sourceMappingURL=ConfigPublicUrls.js.map

package/publicUrls/ConfigPublicUrls.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"publicUrls/ConfigPublicUrls.js","sources":["../../src/publicUrls/ConfigPublicUrls.ts"],"sourcesContent":["import { PublicUrls } from \"@webiny/api-core/features/publicUrls/index.js\";\nimport { StandaloneConfig } from \"~/config/abstractions.js\";\n\n/** Standalone: the URLs come from `WEBINY_API_URL` / `WEBINY_ADMIN_URL`, read at startup. */\nclass ConfigPublicUrlsImpl implements PublicUrls.Interface {\n    constructor(private config: StandaloneConfig.Interface) {}\n\n    async getApiUrl() {\n        return this.config.apiUrl;\n    }\n\n    async getAdminUrl() {\n        return this.config.adminUrl;\n    }\n}\n\nexport const ConfigPublicUrls = PublicUrls.createImplementation({\n    implementation: ConfigPublicUrlsImpl,\n    dependencies: [StandaloneConfig]\n});\n"],"names":["ConfigPublicUrlsImpl","config","ConfigPublicUrls","PublicUrls","StandaloneConfig"],"mappings":";;AAIA,MAAMA;IACF,YAAoBC,MAAkC,CAAE;aAApCA,MAAM,GAANA;IAAqC;IAEzD,MAAM,YAAY;QACd,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM;IAC7B;IAEA,MAAM,cAAc;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ;IAC/B;AACJ;AAEO,MAAMC,mBAAmBC,WAAW,oBAAoB,CAAC;IAC5D,gBAAgBH;IAChB,cAAc;QAACI;KAAiB;AACpC"}

package/publicUrls/feature.d.ts

@@ -0,0 +1,4 @@
+export declare const ConfigPublicUrlsFeature: {
+    name: string;
+    register(container: import("@webiny/di").Container): void;
+};

package/publicUrls/feature.js

@@ -0,0 +1,11 @@
+import { createFeature } from "@webiny/feature/api";
+import { ConfigPublicUrls } from "./ConfigPublicUrls.js";
+const ConfigPublicUrlsFeature = createFeature({
+    name: "ConfigPublicUrlsFeature",
+    register (container) {
+        container.register(ConfigPublicUrls);
+    }
+});
+export { ConfigPublicUrlsFeature };
+
+//# sourceMappingURL=feature.js.map

package/publicUrls/feature.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"publicUrls/feature.js","sources":["../../src/publicUrls/feature.ts"],"sourcesContent":["import { createFeature } from \"@webiny/feature/api\";\nimport { ConfigPublicUrls } from \"./ConfigPublicUrls.js\";\n\nexport const ConfigPublicUrlsFeature = createFeature({\n    name: \"ConfigPublicUrlsFeature\",\n    register(container) {\n        container.register(ConfigPublicUrls);\n    }\n});\n"],"names":["ConfigPublicUrlsFeature","createFeature","container","ConfigPublicUrls"],"mappings":";;AAGO,MAAMA,0BAA0BC,cAAc;IACjD,MAAM;IACN,UAASC,SAAS;QACdA,UAAU,QAAQ,CAACC;IACvB;AACJ"}

package/scheduler/RecoverScheduledActionsHandler.d.ts

@@ -1,26 +1,28 @@
-import { RawTenantId } from "@webiny/api-core/features/requestContext/index.js";
-import { RequestTenantLoader } from "@webiny/api-core/features/requestContext/index.js";
-import { GetRootTenantUseCase } from "@webiny/api-core/features/tenancy/GetRootTenant/index.js";
+import { ListTenantsUseCase } from "@webiny/api-core/features/tenancy/ListTenants/index.js";
+import { TenantContext } from "@webiny/api-core/features/tenancy/TenantContext/index.js";
 import { IdentityContext } from "@webiny/api-core/exports/api/security.js";
+import { Logger } from "@webiny/api-core/features/logger/abstractions.js";
 import { ListScheduledActionsUseCase } from "@webiny/api-scheduler/features/ListScheduledActions/index.js";
 import { ScheduledActionRecoverEventHandler } from "./ScheduledActionRecoverEventHandler.js";
 import { SchedulerSingleton } from "./abstractions/SchedulerSingleton.js";
 /**
- * Re-arms the root tenant's pending scheduled actions in the Bree singleton. Overdue ones run at
- * once (see `BreeSchedulerService.recover`).
+ * Re-arms every tenant's pending scheduled actions in the Bree singleton. Bree keeps its timers in
+ * memory, so a restart loses them all. Overdue actions run at once (see
+ * `BreeSchedulerService.recover`).
  *
- * Root tenant only. Recovering other tenants means enumerating them and recovering each; until
- * then a restart drops their pending schedules.
+ * Each tenant is recovered on its own: one that fails is logged and reported in the result, and the
+ * rest are still recovered. A fresh install has no tenants yet and recovers nothing.
  */
 declare class RecoverScheduledActionsHandlerImpl implements ScheduledActionRecoverEventHandler.Interface {
-    private readonly getRootTenant;
-    private readonly rawTenantId;
-    private readonly tenantLoader;
+    private readonly listTenants;
+    private readonly tenantContext;
     private readonly identityContext;
     private readonly listScheduledActions;
     private readonly scheduler;
-    constructor(getRootTenant: GetRootTenantUseCase.Interface, rawTenantId: RawTenantId.Interface, tenantLoader: RequestTenantLoader.Interface, identityContext: IdentityContext.Interface, listScheduledActions: ListScheduledActionsUseCase.Interface, scheduler: SchedulerSingleton.Interface);
+    private readonly logger;
+    constructor(listTenants: ListTenantsUseCase.Interface, tenantContext: TenantContext.Interface, identityContext: IdentityContext.Interface, listScheduledActions: ListScheduledActionsUseCase.Interface, scheduler: SchedulerSingleton.Interface, logger: Logger.Interface);
     execute(): Promise<ScheduledActionRecoverEventHandler.Result>;
+    private recoverTenant;
     private recoverAllPending;
 }
 export declare const RecoverScheduledActionsHandler: typeof RecoverScheduledActionsHandlerImpl & {

package/scheduler/RecoverScheduledActionsHandler.js

@@ -1,32 +1,45 @@
-import { RawTenantId, RequestTenantLoader } from "@webiny/api-core/features/requestContext/index.js";
-import { GetRootTenantUseCase } from "@webiny/api-core/features/tenancy/GetRootTenant/index.js";
+import { ListTenantsUseCase } from "@webiny/api-core/features/tenancy/ListTenants/index.js";
+import { TenantContext } from "@webiny/api-core/features/tenancy/TenantContext/index.js";
 import { IdentityContext } from "@webiny/api-core/exports/api/security.js";
+import { Logger } from "@webiny/api-core/features/logger/abstractions.js";
 import { ListScheduledActionsUseCase } from "@webiny/api-scheduler/features/ListScheduledActions/index.js";
 import { ScheduledActionRecoverEventHandler } from "./ScheduledActionRecoverEventHandler.js";
 import { SchedulerSingleton } from "./abstractions/SchedulerSingleton.js";
 const PAGE_SIZE = 1000;
+function canRecover(tenant) {
+    return tenant.isInstalled && "disabled" !== tenant.status;
+}
 class RecoverScheduledActionsHandlerImpl {
-    constructor(getRootTenant, rawTenantId, tenantLoader, identityContext, listScheduledActions, scheduler){
-        this.getRootTenant = getRootTenant;
-        this.rawTenantId = rawTenantId;
-        this.tenantLoader = tenantLoader;
+    constructor(listTenants, tenantContext, identityContext, listScheduledActions, scheduler, logger){
+        this.listTenants = listTenants;
+        this.tenantContext = tenantContext;
         this.identityContext = identityContext;
         this.listScheduledActions = listScheduledActions;
         this.scheduler = scheduler;
+        this.logger = logger;
     }
     async execute() {
-        const rootResult = await this.getRootTenant.execute();
-        if (rootResult.isFail()) return {
-            recovered: 0
-        };
-        const tenant = rootResult.value.id;
-        this.rawTenantId.set(tenant);
-        await this.tenantLoader.establish();
-        const recovered = await this.identityContext.withoutAuthorization(()=>this.recoverAllPending(tenant));
+        const tenantsResult = await this.listTenants.execute();
+        if (tenantsResult.isFail()) throw tenantsResult.error;
+        let recovered = 0;
+        const failedTenants = [];
+        for (const tenant of tenantsResult.value)if (canRecover(tenant)) try {
+            recovered += await this.recoverTenant(tenant);
+        } catch (error) {
+            failedTenants.push(tenant.id);
+            this.logger.error({
+                error,
+                tenant: tenant.id
+            }, "Could not recover the tenant's scheduled actions.");
+        }
         return {
-            recovered
+            recovered,
+            failedTenants
         };
     }
+    recoverTenant(tenant) {
+        return this.tenantContext.withTenant(tenant, ()=>this.identityContext.withoutAuthorization(()=>this.recoverAllPending(tenant.id)));
+    }
     async recoverAllPending(tenant) {
         let recovered = 0;
         let after;
@@ -55,12 +68,12 @@ class RecoverScheduledActionsHandlerImpl {
 const RecoverScheduledActionsHandler = ScheduledActionRecoverEventHandler.createImplementation({
     implementation: RecoverScheduledActionsHandlerImpl,
     dependencies: [
-        GetRootTenantUseCase,
-        RawTenantId,
-        RequestTenantLoader,
+        ListTenantsUseCase,
+        TenantContext,
         IdentityContext,
         ListScheduledActionsUseCase,
-        SchedulerSingleton
+        SchedulerSingleton,
+        Logger
     ]
 });
 export { RecoverScheduledActionsHandler };

package/scheduler/RecoverScheduledActionsHandler.js.map

@@ -1 +1 @@
-{"version":3,"file":"scheduler/RecoverScheduledActionsHandler.js","sources":["../../src/scheduler/RecoverScheduledActionsHandler.ts"],"sourcesContent":["import { RawTenantId } from \"@webiny/api-core/features/requestContext/index.js\";\nimport { RequestTenantLoader } from \"@webiny/api-core/features/requestContext/index.js\";\nimport { GetRootTenantUseCase } from \"@webiny/api-core/features/tenancy/GetRootTenant/index.js\";\nimport { IdentityContext } from \"@webiny/api-core/exports/api/security.js\";\nimport { ListScheduledActionsUseCase } from \"@webiny/api-scheduler/features/ListScheduledActions/index.js\";\nimport { ScheduledActionRecoverEventHandler } from \"./ScheduledActionRecoverEventHandler.js\";\nimport type { IPendingAction } from \"@webiny/api-scheduler-standalone\";\nimport { SchedulerSingleton } from \"./abstractions/SchedulerSingleton.js\";\n\n// Pending actions read per page.\nconst PAGE_SIZE = 1000;\n\n/**\n * Re-arms the root tenant's pending scheduled actions in the Bree singleton. Overdue ones run at\n * once (see `BreeSchedulerService.recover`).\n *\n * Root tenant only. Recovering other tenants means enumerating them and recovering each; until\n * then a restart drops their pending schedules.\n */\nclass RecoverScheduledActionsHandlerImpl implements ScheduledActionRecoverEventHandler.Interface {\n    public constructor(\n        private readonly getRootTenant: GetRootTenantUseCase.Interface,\n        private readonly rawTenantId: RawTenantId.Interface,\n        private readonly tenantLoader: RequestTenantLoader.Interface,\n        private readonly identityContext: IdentityContext.Interface,\n        private readonly listScheduledActions: ListScheduledActionsUseCase.Interface,\n        private readonly scheduler: SchedulerSingleton.Interface\n    ) {}\n\n    public async execute(): Promise<ScheduledActionRecoverEventHandler.Result> {\n        const rootResult = await this.getRootTenant.execute();\n        if (rootResult.isFail()) {\n            // No root tenant yet: a fresh install has nothing to recover.\n            return { recovered: 0 };\n        }\n        const tenant = rootResult.value.id;\n\n        this.rawTenantId.set(tenant);\n        await this.tenantLoader.establish();\n\n        /*\n         * Nobody is signed in at boot. Listing without authorization is what the run side does too\n         * (ExecuteScheduledActionUseCase); otherwise the anonymous identity fails the permission check.\n         */\n        const recovered = await this.identityContext.withoutAuthorization(() => {\n            return this.recoverAllPending(tenant);\n        });\n\n        return { recovered };\n    }\n\n    /*\n     * Re-arms page by page, so a lookup that fails on a later page still leaves the earlier pages\n     * armed. The failure is rethrown and logged by the boot step.\n     */\n    private async recoverAllPending(tenant: string): Promise<number> {\n        let recovered = 0;\n        let after: string | undefined;\n\n        do {\n            const listResult = await this.listScheduledActions.execute({\n                where: {},\n                limit: PAGE_SIZE,\n                after\n            });\n            if (listResult.isFail()) {\n                throw listResult.error;\n            }\n\n            const page: IPendingAction[] = [];\n            for (const action of listResult.value.items) {\n                page.push({\n                    id: action.id,\n                    namespace: action.namespace,\n                    tenant,\n                    scheduledFor: action.scheduledFor\n                });\n            }\n\n            await this.scheduler.recover(page);\n            recovered += page.length;\n\n            const { meta } = listResult.value;\n            after = meta.hasMoreItems && meta.cursor ? meta.cursor : undefined;\n        } while (after);\n\n        return recovered;\n    }\n}\n\nexport const RecoverScheduledActionsHandler =\n    ScheduledActionRecoverEventHandler.createImplementation({\n        implementation: RecoverScheduledActionsHandlerImpl,\n        dependencies: [\n            GetRootTenantUseCase,\n            RawTenantId,\n            RequestTenantLoader,\n            IdentityContext,\n            ListScheduledActionsUseCase,\n            SchedulerSingleton\n        ]\n    });\n"],"names":["PAGE_SIZE","RecoverScheduledActionsHandlerImpl","getRootTenant","rawTenantId","tenantLoader","identityContext","listScheduledActions","scheduler","rootResult","tenant","recovered","after","listResult","page","action","meta","undefined","RecoverScheduledActionsHandler","ScheduledActionRecoverEventHandler","GetRootTenantUseCase","RawTenantId","RequestTenantLoader","IdentityContext","ListScheduledActionsUseCase","SchedulerSingleton"],"mappings":";;;;;;AAUA,MAAMA,YAAY;AASlB,MAAMC;IACF,YACqBC,aAA6C,EAC7CC,WAAkC,EAClCC,YAA2C,EAC3CC,eAA0C,EAC1CC,oBAA2D,EAC3DC,SAAuC,CAC1D;aANmBL,aAAa,GAAbA;aACAC,WAAW,GAAXA;aACAC,YAAY,GAAZA;aACAC,eAAe,GAAfA;aACAC,oBAAoB,GAApBA;aACAC,SAAS,GAATA;IAClB;IAEH,MAAa,UAA8D;QACvE,MAAMC,aAAa,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO;QACnD,IAAIA,WAAW,MAAM,IAEjB,OAAO;YAAE,WAAW;QAAE;QAE1B,MAAMC,SAASD,WAAW,KAAK,CAAC,EAAE;QAElC,IAAI,CAAC,WAAW,CAAC,GAAG,CAACC;QACrB,MAAM,IAAI,CAAC,YAAY,CAAC,SAAS;QAMjC,MAAMC,YAAY,MAAM,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,IACvD,IAAI,CAAC,iBAAiB,CAACD;QAGlC,OAAO;YAAEC;QAAU;IACvB;IAMA,MAAc,kBAAkBD,MAAc,EAAmB;QAC7D,IAAIC,YAAY;QAChB,IAAIC;QAEJ,GAAG;YACC,MAAMC,aAAa,MAAM,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC;gBACvD,OAAO,CAAC;gBACR,OAAOZ;gBACPW;YACJ;YACA,IAAIC,WAAW,MAAM,IACjB,MAAMA,WAAW,KAAK;YAG1B,MAAMC,OAAyB,EAAE;YACjC,KAAK,MAAMC,UAAUF,WAAW,KAAK,CAAC,KAAK,CACvCC,KAAK,IAAI,CAAC;gBACN,IAAIC,OAAO,EAAE;gBACb,WAAWA,OAAO,SAAS;gBAC3BL;gBACA,cAAcK,OAAO,YAAY;YACrC;YAGJ,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAACD;YAC7BH,aAAaG,KAAK,MAAM;YAExB,MAAM,EAAEE,IAAI,EAAE,GAAGH,WAAW,KAAK;YACjCD,QAAQI,KAAK,YAAY,IAAIA,KAAK,MAAM,GAAGA,KAAK,MAAM,GAAGC;QAC7D,QAASL,MAAO;QAEhB,OAAOD;IACX;AACJ;AAEO,MAAMO,iCACTC,mCAAmC,oBAAoB,CAAC;IACpD,gBAAgBjB;IAChB,cAAc;QACVkB;QACAC;QACAC;QACAC;QACAC;QACAC;KACH;AACL"}
+{"version":3,"file":"scheduler/RecoverScheduledActionsHandler.js","sources":["../../src/scheduler/RecoverScheduledActionsHandler.ts"],"sourcesContent":["import { ListTenantsUseCase } from \"@webiny/api-core/features/tenancy/ListTenants/index.js\";\nimport { TenantContext } from \"@webiny/api-core/features/tenancy/TenantContext/index.js\";\nimport type { Tenant } from \"@webiny/api-core/types/tenancy.js\";\nimport { IdentityContext } from \"@webiny/api-core/exports/api/security.js\";\nimport { Logger } from \"@webiny/api-core/features/logger/abstractions.js\";\nimport { ListScheduledActionsUseCase } from \"@webiny/api-scheduler/features/ListScheduledActions/index.js\";\nimport { ScheduledActionRecoverEventHandler } from \"./ScheduledActionRecoverEventHandler.js\";\nimport type { IPendingAction } from \"@webiny/api-scheduler-standalone\";\nimport { SchedulerSingleton } from \"./abstractions/SchedulerSingleton.js\";\n\n// Pending actions read per page.\nconst PAGE_SIZE = 1000;\n\n/*\n * A tenant that isn't installed has no scheduler model to read, and a disabled one must not run its\n * schedules. Both are left alone; a disabled tenant's schedules come back on the next restart after\n * it is re-enabled.\n */\nfunction canRecover(tenant: Tenant): boolean {\n    return tenant.isInstalled && tenant.status !== \"disabled\";\n}\n\n/**\n * Re-arms every tenant's pending scheduled actions in the Bree singleton. Bree keeps its timers in\n * memory, so a restart loses them all. Overdue actions run at once (see\n * `BreeSchedulerService.recover`).\n *\n * Each tenant is recovered on its own: one that fails is logged and reported in the result, and the\n * rest are still recovered. A fresh install has no tenants yet and recovers nothing.\n */\nclass RecoverScheduledActionsHandlerImpl implements ScheduledActionRecoverEventHandler.Interface {\n    public constructor(\n        private readonly listTenants: ListTenantsUseCase.Interface,\n        private readonly tenantContext: TenantContext.Interface,\n        private readonly identityContext: IdentityContext.Interface,\n        private readonly listScheduledActions: ListScheduledActionsUseCase.Interface,\n        private readonly scheduler: SchedulerSingleton.Interface,\n        private readonly logger: Logger.Interface\n    ) {}\n\n    public async execute(): Promise<ScheduledActionRecoverEventHandler.Result> {\n        // A failed lookup is not a fresh install: it throws, so the boot step reports it.\n        const tenantsResult = await this.listTenants.execute();\n        if (tenantsResult.isFail()) {\n            throw tenantsResult.error;\n        }\n\n        let recovered = 0;\n        const failedTenants: string[] = [];\n\n        for (const tenant of tenantsResult.value) {\n            if (!canRecover(tenant)) {\n                continue;\n            }\n            try {\n                recovered += await this.recoverTenant(tenant);\n            } catch (error) {\n                failedTenants.push(tenant.id);\n                this.logger.error(\n                    { error, tenant: tenant.id },\n                    \"Could not recover the tenant's scheduled actions.\"\n                );\n            }\n        }\n\n        return { recovered, failedTenants };\n    }\n\n    private recoverTenant(tenant: Tenant): Promise<number> {\n        return this.tenantContext.withTenant(tenant, () => {\n            /*\n             * Nobody is signed in at boot. Listing without authorization is what the run side does\n             * too (ExecuteScheduledActionUseCase); otherwise the anonymous identity fails the\n             * permission check.\n             */\n            return this.identityContext.withoutAuthorization(() => {\n                return this.recoverAllPending(tenant.id);\n            });\n        });\n    }\n\n    /*\n     * Re-arms page by page, so a lookup that fails on a later page still leaves the earlier pages\n     * armed. The failure is rethrown and logged by the boot step.\n     */\n    private async recoverAllPending(tenant: string): Promise<number> {\n        let recovered = 0;\n        let after: string | undefined;\n\n        do {\n            const listResult = await this.listScheduledActions.execute({\n                where: {},\n                limit: PAGE_SIZE,\n                after\n            });\n            if (listResult.isFail()) {\n                throw listResult.error;\n            }\n\n            const page: IPendingAction[] = [];\n            for (const action of listResult.value.items) {\n                page.push({\n                    id: action.id,\n                    namespace: action.namespace,\n                    tenant,\n                    scheduledFor: action.scheduledFor\n                });\n            }\n\n            await this.scheduler.recover(page);\n            recovered += page.length;\n\n            const { meta } = listResult.value;\n            after = meta.hasMoreItems && meta.cursor ? meta.cursor : undefined;\n        } while (after);\n\n        return recovered;\n    }\n}\n\nexport const RecoverScheduledActionsHandler =\n    ScheduledActionRecoverEventHandler.createImplementation({\n        implementation: RecoverScheduledActionsHandlerImpl,\n        dependencies: [\n            ListTenantsUseCase,\n            TenantContext,\n            IdentityContext,\n            ListScheduledActionsUseCase,\n            SchedulerSingleton,\n            Logger\n        ]\n    });\n"],"names":["PAGE_SIZE","canRecover","tenant","RecoverScheduledActionsHandlerImpl","listTenants","tenantContext","identityContext","listScheduledActions","scheduler","logger","tenantsResult","recovered","failedTenants","error","after","listResult","page","action","meta","undefined","RecoverScheduledActionsHandler","ScheduledActionRecoverEventHandler","ListTenantsUseCase","TenantContext","IdentityContext","ListScheduledActionsUseCase","SchedulerSingleton","Logger"],"mappings":";;;;;;;AAWA,MAAMA,YAAY;AAOlB,SAASC,WAAWC,MAAc;IAC9B,OAAOA,OAAO,WAAW,IAAIA,AAAkB,eAAlBA,OAAO,MAAM;AAC9C;AAUA,MAAMC;IACF,YACqBC,WAAyC,EACzCC,aAAsC,EACtCC,eAA0C,EAC1CC,oBAA2D,EAC3DC,SAAuC,EACvCC,MAAwB,CAC3C;aANmBL,WAAW,GAAXA;aACAC,aAAa,GAAbA;aACAC,eAAe,GAAfA;aACAC,oBAAoB,GAApBA;aACAC,SAAS,GAATA;aACAC,MAAM,GAANA;IAClB;IAEH,MAAa,UAA8D;QAEvE,MAAMC,gBAAgB,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO;QACpD,IAAIA,cAAc,MAAM,IACpB,MAAMA,cAAc,KAAK;QAG7B,IAAIC,YAAY;QAChB,MAAMC,gBAA0B,EAAE;QAElC,KAAK,MAAMV,UAAUQ,cAAc,KAAK,CACpC,IAAKT,WAAWC,SAGhB,IAAI;YACAS,aAAa,MAAM,IAAI,CAAC,aAAa,CAACT;QAC1C,EAAE,OAAOW,OAAO;YACZD,cAAc,IAAI,CAACV,OAAO,EAAE;YAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CACb;gBAAEW;gBAAO,QAAQX,OAAO,EAAE;YAAC,GAC3B;QAER;QAGJ,OAAO;YAAES;YAAWC;QAAc;IACtC;IAEQ,cAAcV,MAAc,EAAmB;QACnD,OAAO,IAAI,CAAC,aAAa,CAAC,UAAU,CAACA,QAAQ,IAMlC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,IACtC,IAAI,CAAC,iBAAiB,CAACA,OAAO,EAAE;IAGnD;IAMA,MAAc,kBAAkBA,MAAc,EAAmB;QAC7D,IAAIS,YAAY;QAChB,IAAIG;QAEJ,GAAG;YACC,MAAMC,aAAa,MAAM,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC;gBACvD,OAAO,CAAC;gBACR,OAAOf;gBACPc;YACJ;YACA,IAAIC,WAAW,MAAM,IACjB,MAAMA,WAAW,KAAK;YAG1B,MAAMC,OAAyB,EAAE;YACjC,KAAK,MAAMC,UAAUF,WAAW,KAAK,CAAC,KAAK,CACvCC,KAAK,IAAI,CAAC;gBACN,IAAIC,OAAO,EAAE;gBACb,WAAWA,OAAO,SAAS;gBAC3Bf;gBACA,cAAce,OAAO,YAAY;YACrC;YAGJ,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAACD;YAC7BL,aAAaK,KAAK,MAAM;YAExB,MAAM,EAAEE,IAAI,EAAE,GAAGH,WAAW,KAAK;YACjCD,QAAQI,KAAK,YAAY,IAAIA,KAAK,MAAM,GAAGA,KAAK,MAAM,GAAGC;QAC7D,QAASL,MAAO;QAEhB,OAAOH;IACX;AACJ;AAEO,MAAMS,iCACTC,mCAAmC,oBAAoB,CAAC;IACpD,gBAAgBlB;IAChB,cAAc;QACVmB;QACAC;QACAC;QACAC;QACAC;QACAC;KACH;AACL"}

package/scheduler/RunScheduledActionOnceDecorator.d.ts

@@ -0,0 +1,37 @@
+import { ScheduledActionEventHandler } from "@webiny/event-handler-core";
+import type { IScheduledActionEvent } from "@webiny/event-handler-core";
+import type { IScheduledActionResult } from "@webiny/event-handler-core";
+import type { EventContext } from "@webiny/event-handler-core";
+import type { NextFunction } from "@webiny/event-handler-core";
+import { GetTenantByIdUseCase } from "@webiny/api-core/exports/api/tenancy.js";
+import { TenantContext } from "@webiny/api-core/features/tenancy/TenantContext/index.js";
+import { IdentityContext } from "@webiny/api-core/exports/api/security.js";
+import { GlobalKeyValueStore } from "@webiny/api-core/features/keyValueStore/index.js";
+import { GetScheduledActionUseCase } from "@webiny/api-scheduler/features/GetScheduledAction/index.js";
+/**
+ * Runs a fired scheduled action at most once across all processes. Every standalone process holds its
+ * own timers for the same actions, so the same action fires in each of them; on AWS, EventBridge
+ * fires it once and this decorator is not registered.
+ *
+ * Before the action runs, it is read again. Gone means it was cancelled or already ran. Due later
+ * means another process moved it and this timer is stale. Otherwise the process takes a claim on the
+ * action at its due time, and only the process that gets it runs the action. Rescheduling changes
+ * the due time, so a moved action gets a fresh claim.
+ */
+declare class RunScheduledActionOnceDecoratorImpl implements ScheduledActionEventHandler.Interface {
+    private readonly getTenantById;
+    private readonly tenantContext;
+    private readonly identityContext;
+    private readonly getScheduledAction;
+    private readonly store;
+    private readonly decoratee;
+    constructor(getTenantById: GetTenantByIdUseCase.Interface, tenantContext: TenantContext.Interface, identityContext: IdentityContext.Interface, getScheduledAction: GetScheduledActionUseCase.Interface, store: GlobalKeyValueStore.Interface, decoratee: ScheduledActionEventHandler.Interface);
+    execute(eventCtx: EventContext<IScheduledActionEvent>, next: NextFunction): Promise<IScheduledActionResult>;
+    private findDueTime;
+    private takeClaim;
+    private holdAfterFailure;
+}
+export declare const RunScheduledActionOnceDecorator: typeof RunScheduledActionOnceDecoratorImpl & {
+    __abstraction: import("@webiny/di").Abstraction<import("@webiny/event-handler-core/features/scheduledAction/ScheduledActionEventHandler").IScheduledActionEventHandler>;
+};
+export {};

package/scheduler/RunScheduledActionOnceDecorator.js

@@ -0,0 +1,91 @@
+import { hostname } from "node:os";
+import { SCHEDULED_ACTION_EVENT_IDENTIFIER, ScheduledActionEventHandler } from "@webiny/event-handler-core";
+import { GetTenantByIdUseCase } from "@webiny/api-core/exports/api/tenancy.js";
+import { TenantContext } from "@webiny/api-core/features/tenancy/TenantContext/index.js";
+import { IdentityContext } from "@webiny/api-core/exports/api/security.js";
+import { GlobalKeyValueStore, KeyValueConditionFailedError } from "@webiny/api-core/features/keyValueStore/index.js";
+import { GetScheduledActionUseCase } from "@webiny/api-scheduler/features/GetScheduledAction/index.js";
+const CLAIM_SCOPE = "scheduler/claims";
+const RUN_CLAIM_MS = 600000;
+const RETRY_AFTER_FAILURE_MS = 86400000;
+const EARLY_TOLERANCE_MS = 1000;
+const SUCCESS = {
+    success: true
+};
+const PROCESS_NAME = `${hostname()}:${process.pid}`;
+const expiresIn = (ms)=>{
+    const expiresAtMs = Date.now() + ms;
+    return new Date(expiresAtMs);
+};
+class RunScheduledActionOnceDecoratorImpl {
+    constructor(getTenantById, tenantContext, identityContext, getScheduledAction, store, decoratee){
+        this.getTenantById = getTenantById;
+        this.tenantContext = tenantContext;
+        this.identityContext = identityContext;
+        this.getScheduledAction = getScheduledAction;
+        this.store = store;
+        this.decoratee = decoratee;
+    }
+    async execute(eventCtx, next) {
+        const input = eventCtx.event[SCHEDULED_ACTION_EVENT_IDENTIFIER];
+        const tenantId = input.tenant || this.tenantContext.getTenant().id;
+        const dueAt = await this.findDueTime(tenantId, input);
+        if (!dueAt) return SUCCESS;
+        if (dueAt.getTime() > Date.now() + EARLY_TOLERANCE_MS) return SUCCESS;
+        const key = `${tenantId}/${input.id}/${dueAt.toISOString()}`;
+        const claimed = await this.takeClaim(key);
+        if (!claimed) return SUCCESS;
+        try {
+            return await this.decoratee.execute(eventCtx, next);
+        } catch (error) {
+            await this.holdAfterFailure(key);
+            throw error;
+        }
+    }
+    async findDueTime(tenantId, input) {
+        const tenantResult = await this.getTenantById.execute(tenantId);
+        if (tenantResult.isFail()) throw tenantResult.error;
+        return this.tenantContext.withTenant(tenantResult.value, ()=>this.identityContext.withoutAuthorization(async ()=>{
+                const result = await this.getScheduledAction.execute({
+                    id: input.id,
+                    namespace: input.namespace
+                });
+                if (result.isOk()) return result.value.scheduledFor;
+                if ("Scheduler/ScheduledAction/NotFound" === result.error.code) return null;
+                throw result.error;
+            }));
+    }
+    async takeClaim(key) {
+        const expiresAt = expiresIn(RUN_CLAIM_MS);
+        const result = await this.store.set(key, PROCESS_NAME, {
+            scope: CLAIM_SCOPE,
+            expiresAt,
+            condition: {
+                exists: false
+            }
+        });
+        if (result.isOk()) return true;
+        if (result.error instanceof KeyValueConditionFailedError) return false;
+        throw result.error;
+    }
+    async holdAfterFailure(key) {
+        const expiresAt = expiresIn(RETRY_AFTER_FAILURE_MS);
+        await this.store.set(key, PROCESS_NAME, {
+            scope: CLAIM_SCOPE,
+            expiresAt
+        });
+    }
+}
+const RunScheduledActionOnceDecorator = ScheduledActionEventHandler.createDecorator({
+    decorator: RunScheduledActionOnceDecoratorImpl,
+    dependencies: [
+        GetTenantByIdUseCase,
+        TenantContext,
+        IdentityContext,
+        GetScheduledActionUseCase,
+        GlobalKeyValueStore
+    ]
+});
+export { RunScheduledActionOnceDecorator };
+
+//# sourceMappingURL=RunScheduledActionOnceDecorator.js.map

package/scheduler/RunScheduledActionOnceDecorator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduler/RunScheduledActionOnceDecorator.js","sources":["../../src/scheduler/RunScheduledActionOnceDecorator.ts"],"sourcesContent":["import { hostname } from \"node:os\";\nimport { ScheduledActionEventHandler } from \"@webiny/event-handler-core\";\nimport { SCHEDULED_ACTION_EVENT_IDENTIFIER } from \"@webiny/event-handler-core\";\nimport type { IScheduledActionEvent } from \"@webiny/event-handler-core\";\nimport type { IScheduledActionEventPayload } from \"@webiny/event-handler-core\";\nimport type { IScheduledActionResult } from \"@webiny/event-handler-core\";\nimport type { EventContext } from \"@webiny/event-handler-core\";\nimport type { NextFunction } from \"@webiny/event-handler-core\";\nimport { GetTenantByIdUseCase } from \"@webiny/api-core/exports/api/tenancy.js\";\nimport { TenantContext } from \"@webiny/api-core/features/tenancy/TenantContext/index.js\";\nimport { IdentityContext } from \"@webiny/api-core/exports/api/security.js\";\nimport { GlobalKeyValueStore } from \"@webiny/api-core/features/keyValueStore/index.js\";\nimport { KeyValueConditionFailedError } from \"@webiny/api-core/features/keyValueStore/index.js\";\nimport { GetScheduledActionUseCase } from \"@webiny/api-scheduler/features/GetScheduledAction/index.js\";\n\nconst CLAIM_SCOPE = \"scheduler/claims\";\n\n/*\n * How long one run of an action holds its claim. Long enough for any action to finish; a run that\n * succeeds deletes the action, so the claim only matters while it runs.\n */\nconst RUN_CLAIM_MS = 10 * 60_000;\n\n// After a failed run, other processes leave the action alone this long before trying it again.\nconst RETRY_AFTER_FAILURE_MS = 24 * 60 * 60_000;\n\n/*\n * Timers fire slightly early or late; an action due within this much is run. One due later than that\n * belongs to a timer another process moved, and this firing is stale.\n */\nconst EARLY_TOLERANCE_MS = 1000;\n\nconst SUCCESS: IScheduledActionResult = { success: true };\n\n// Names this process in the claims it writes, for whoever reads them in the database.\nconst PROCESS_NAME = `${hostname()}:${process.pid}`;\n\nconst expiresIn = (ms: number): Date => {\n    const expiresAtMs = Date.now() + ms;\n    return new Date(expiresAtMs);\n};\n\n/**\n * Runs a fired scheduled action at most once across all processes. Every standalone process holds its\n * own timers for the same actions, so the same action fires in each of them; on AWS, EventBridge\n * fires it once and this decorator is not registered.\n *\n * Before the action runs, it is read again. Gone means it was cancelled or already ran. Due later\n * means another process moved it and this timer is stale. Otherwise the process takes a claim on the\n * action at its due time, and only the process that gets it runs the action. Rescheduling changes\n * the due time, so a moved action gets a fresh claim.\n */\nclass RunScheduledActionOnceDecoratorImpl implements ScheduledActionEventHandler.Interface {\n    public constructor(\n        private readonly getTenantById: GetTenantByIdUseCase.Interface,\n        private readonly tenantContext: TenantContext.Interface,\n        private readonly identityContext: IdentityContext.Interface,\n        private readonly getScheduledAction: GetScheduledActionUseCase.Interface,\n        private readonly store: GlobalKeyValueStore.Interface,\n        private readonly decoratee: ScheduledActionEventHandler.Interface\n    ) {}\n\n    public async execute(\n        eventCtx: EventContext<IScheduledActionEvent>,\n        next: NextFunction\n    ): Promise<IScheduledActionResult> {\n        const input = eventCtx.event[SCHEDULED_ACTION_EVENT_IDENTIFIER];\n        const tenantId = input.tenant || this.tenantContext.getTenant().id;\n\n        const dueAt = await this.findDueTime(tenantId, input);\n        if (!dueAt) {\n            return SUCCESS;\n        }\n        if (dueAt.getTime() > Date.now() + EARLY_TOLERANCE_MS) {\n            return SUCCESS;\n        }\n\n        const key = `${tenantId}/${input.id}/${dueAt.toISOString()}`;\n        const claimed = await this.takeClaim(key);\n        if (!claimed) {\n            return SUCCESS;\n        }\n\n        try {\n            return await this.decoratee.execute(eventCtx, next);\n        } catch (error) {\n            await this.holdAfterFailure(key);\n            throw error;\n        }\n    }\n\n    // The action's due time, or `null` when it no longer exists.\n    private async findDueTime(\n        tenantId: string,\n        input: IScheduledActionEventPayload\n    ): Promise<Date | null> {\n        const tenantResult = await this.getTenantById.execute(tenantId);\n        if (tenantResult.isFail()) {\n            throw tenantResult.error;\n        }\n\n        return this.tenantContext.withTenant(tenantResult.value, () => {\n            // GetScheduledAction checks permissions, and nobody is signed in when a timer fires.\n            return this.identityContext.withoutAuthorization(async () => {\n                const result = await this.getScheduledAction.execute({\n                    id: input.id,\n                    namespace: input.namespace\n                });\n                if (result.isOk()) {\n                    return result.value.scheduledFor;\n                }\n                if (result.error.code === \"Scheduler/ScheduledAction/NotFound\") {\n                    return null;\n                }\n                throw result.error;\n            });\n        });\n    }\n\n    private async takeClaim(key: string): Promise<boolean> {\n        const expiresAt = expiresIn(RUN_CLAIM_MS);\n        const result = await this.store.set(key, PROCESS_NAME, {\n            scope: CLAIM_SCOPE,\n            expiresAt,\n            condition: { exists: false }\n        });\n        if (result.isOk()) {\n            return true;\n        }\n        if (result.error instanceof KeyValueConditionFailedError) {\n            return false;\n        }\n        throw result.error;\n    }\n\n    /*\n     * The action stays stored after a failed run, with the error on it, and every process's\n     * recovery fires overdue actions each minute. Holding the claim keeps that to one retry a day.\n     */\n    private async holdAfterFailure(key: string): Promise<void> {\n        const expiresAt = expiresIn(RETRY_AFTER_FAILURE_MS);\n        await this.store.set(key, PROCESS_NAME, { scope: CLAIM_SCOPE, expiresAt });\n    }\n}\n\nexport const RunScheduledActionOnceDecorator = ScheduledActionEventHandler.createDecorator({\n    decorator: RunScheduledActionOnceDecoratorImpl,\n    dependencies: [\n        GetTenantByIdUseCase,\n        TenantContext,\n        IdentityContext,\n        GetScheduledActionUseCase,\n        GlobalKeyValueStore\n    ]\n});\n"],"names":["CLAIM_SCOPE","RUN_CLAIM_MS","RETRY_AFTER_FAILURE_MS","EARLY_TOLERANCE_MS","SUCCESS","PROCESS_NAME","hostname","process","expiresIn","ms","expiresAtMs","Date","RunScheduledActionOnceDecoratorImpl","getTenantById","tenantContext","identityContext","getScheduledAction","store","decoratee","eventCtx","next","input","SCHEDULED_ACTION_EVENT_IDENTIFIER","tenantId","dueAt","key","claimed","error","tenantResult","result","expiresAt","KeyValueConditionFailedError","RunScheduledActionOnceDecorator","ScheduledActionEventHandler","GetTenantByIdUseCase","TenantContext","IdentityContext","GetScheduledActionUseCase","GlobalKeyValueStore"],"mappings":";;;;;;;AAeA,MAAMA,cAAc;AAMpB,MAAMC,eAAe;AAGrB,MAAMC,yBAAyB;AAM/B,MAAMC,qBAAqB;AAE3B,MAAMC,UAAkC;IAAE,SAAS;AAAK;AAGxD,MAAMC,eAAe,GAAGC,WAAW,CAAC,EAAEC,QAAQ,GAAG,EAAE;AAEnD,MAAMC,YAAY,CAACC;IACf,MAAMC,cAAcC,KAAK,GAAG,KAAKF;IACjC,OAAO,IAAIE,KAAKD;AACpB;AAYA,MAAME;IACF,YACqBC,aAA6C,EAC7CC,aAAsC,EACtCC,eAA0C,EAC1CC,kBAAuD,EACvDC,KAAoC,EACpCC,SAAgD,CACnE;aANmBL,aAAa,GAAbA;aACAC,aAAa,GAAbA;aACAC,eAAe,GAAfA;aACAC,kBAAkB,GAAlBA;aACAC,KAAK,GAALA;aACAC,SAAS,GAATA;IAClB;IAEH,MAAa,QACTC,QAA6C,EAC7CC,IAAkB,EACa;QAC/B,MAAMC,QAAQF,SAAS,KAAK,CAACG,kCAAkC;QAC/D,MAAMC,WAAWF,MAAM,MAAM,IAAI,IAAI,CAAC,aAAa,CAAC,SAAS,GAAG,EAAE;QAElE,MAAMG,QAAQ,MAAM,IAAI,CAAC,WAAW,CAACD,UAAUF;QAC/C,IAAI,CAACG,OACD,OAAOpB;QAEX,IAAIoB,MAAM,OAAO,KAAKb,KAAK,GAAG,KAAKR,oBAC/B,OAAOC;QAGX,MAAMqB,MAAM,GAAGF,SAAS,CAAC,EAAEF,MAAM,EAAE,CAAC,CAAC,EAAEG,MAAM,WAAW,IAAI;QAC5D,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAACD;QACrC,IAAI,CAACC,SACD,OAAOtB;QAGX,IAAI;YACA,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAACe,UAAUC;QAClD,EAAE,OAAOO,OAAO;YACZ,MAAM,IAAI,CAAC,gBAAgB,CAACF;YAC5B,MAAME;QACV;IACJ;IAGA,MAAc,YACVJ,QAAgB,EAChBF,KAAmC,EACf;QACpB,MAAMO,eAAe,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAACL;QACtD,IAAIK,aAAa,MAAM,IACnB,MAAMA,aAAa,KAAK;QAG5B,OAAO,IAAI,CAAC,aAAa,CAAC,UAAU,CAACA,aAAa,KAAK,EAAE,IAE9C,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC;gBAC7C,MAAMC,SAAS,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC;oBACjD,IAAIR,MAAM,EAAE;oBACZ,WAAWA,MAAM,SAAS;gBAC9B;gBACA,IAAIQ,OAAO,IAAI,IACX,OAAOA,OAAO,KAAK,CAAC,YAAY;gBAEpC,IAAIA,AAAsB,yCAAtBA,OAAO,KAAK,CAAC,IAAI,EACjB,OAAO;gBAEX,MAAMA,OAAO,KAAK;YACtB;IAER;IAEA,MAAc,UAAUJ,GAAW,EAAoB;QACnD,MAAMK,YAAYtB,UAAUP;QAC5B,MAAM4B,SAAS,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAACJ,KAAKpB,cAAc;YACnD,OAAOL;YACP8B;YACA,WAAW;gBAAE,QAAQ;YAAM;QAC/B;QACA,IAAID,OAAO,IAAI,IACX,OAAO;QAEX,IAAIA,OAAO,KAAK,YAAYE,8BACxB,OAAO;QAEX,MAAMF,OAAO,KAAK;IACtB;IAMA,MAAc,iBAAiBJ,GAAW,EAAiB;QACvD,MAAMK,YAAYtB,UAAUN;QAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAACuB,KAAKpB,cAAc;YAAE,OAAOL;YAAa8B;QAAU;IAC5E;AACJ;AAEO,MAAME,kCAAkCC,4BAA4B,eAAe,CAAC;IACvF,WAAWrB;IACX,cAAc;QACVsB;QACAC;QACAC;QACAC;QACAC;KACH;AACL"}

package/scheduler/ScheduledActionRecoverEventHandler.d.ts

@@ -3,6 +3,7 @@ import type { IEventHandler } from "@webiny/event-handler-core";
 import type { IScheduledActionRecoverEvent } from "./ScheduledActionRecoverEventType.js";
 export interface IScheduledActionRecoverResult {
     recovered: number;
+    failedTenants: string[];
 }
 export interface IScheduledActionRecoverEventHandler extends IEventHandler<IScheduledActionRecoverEvent, IScheduledActionRecoverResult> {
 }

package/scheduler/ScheduledActionRecoverEventHandler.js.map

@@ -1 +1 @@
-{"version":3,"file":"scheduler/ScheduledActionRecoverEventHandler.js","sources":["../../src/scheduler/ScheduledActionRecoverEventHandler.ts"],"sourcesContent":["import { Abstraction } from \"@webiny/di\";\nimport type { IEventHandler } from \"@webiny/event-handler-core\";\nimport type { IScheduledActionRecoverEvent } from \"./ScheduledActionRecoverEventType.js\";\n\nexport interface IScheduledActionRecoverResult {\n    recovered: number;\n}\n\nexport interface IScheduledActionRecoverEventHandler extends IEventHandler<\n    IScheduledActionRecoverEvent,\n    IScheduledActionRecoverResult\n> {}\n\nexport const ScheduledActionRecoverEventHandler =\n    new Abstraction<IScheduledActionRecoverEventHandler>(\"ScheduledActionRecoverEventHandler\");\n\nexport namespace ScheduledActionRecoverEventHandler {\n    export type Interface = IScheduledActionRecoverEventHandler;\n    export type Result = IScheduledActionRecoverResult;\n}\n"],"names":["ScheduledActionRecoverEventHandler","Abstraction"],"mappings":";AAaO,MAAMA,qCACT,IAAIC,YAAiD"}
+{"version":3,"file":"scheduler/ScheduledActionRecoverEventHandler.js","sources":["../../src/scheduler/ScheduledActionRecoverEventHandler.ts"],"sourcesContent":["import { Abstraction } from \"@webiny/di\";\nimport type { IEventHandler } from \"@webiny/event-handler-core\";\nimport type { IScheduledActionRecoverEvent } from \"./ScheduledActionRecoverEventType.js\";\n\nexport interface IScheduledActionRecoverResult {\n    recovered: number;\n    // Tenants whose schedules could not be read or re-armed; the others were still recovered.\n    failedTenants: string[];\n}\n\nexport interface IScheduledActionRecoverEventHandler extends IEventHandler<\n    IScheduledActionRecoverEvent,\n    IScheduledActionRecoverResult\n> {}\n\nexport const ScheduledActionRecoverEventHandler =\n    new Abstraction<IScheduledActionRecoverEventHandler>(\"ScheduledActionRecoverEventHandler\");\n\nexport namespace ScheduledActionRecoverEventHandler {\n    export type Interface = IScheduledActionRecoverEventHandler;\n    export type Result = IScheduledActionRecoverResult;\n}\n"],"names":["ScheduledActionRecoverEventHandler","Abstraction"],"mappings":";AAeO,MAAMA,qCACT,IAAIC,YAAiD"}

package/scheduler/schedulerServer.d.ts

@@ -10,8 +10,16 @@ import type { Container } from "@webiny/di";
  */
 export declare function registerSchedulerServer(rootContainer: Container): void;
 /**
- * Boot step (onServer): start the timers, then re-arm the persisted schedules. Recovery runs in the
- * background: it executes overdue actions one by one, and the server should not wait for that
- * before it starts listening. A failure is logged and never crashes startup.
+ * Boot step (onServer): start the timers, then re-arm the persisted schedules, at boot and then every
+ * minute. Recovery runs in the background: it executes overdue actions one by one, and the server
+ * should not wait for that before it starts listening. A failure is logged and never crashes startup.
+ *
+ * Running it every minute is what makes several processes work. Each holds its own timers, so an
+ * action scheduled or moved through one process reaches the others' timers within a minute, and an
+ * action whose process died is still fired, as an overdue one. `RunScheduledActionOnceDecorator`
+ * keeps the processes from all running it. Runs never overlap.
+ *
+ * Returns a function for the shutdown: it stops the timers and recovery, then waits up to
+ * `timeoutMs` for actions already running.
  */
-export declare function startSchedulerServer(rootContainer: Container): Promise<void>;
+export declare function startSchedulerServer(rootContainer: Container): Promise<(timeoutMs: number) => Promise<void>>;

package/scheduler/schedulerServer.js

@@ -5,20 +5,13 @@ import { BreeSchedulerService } from "@webiny/api-scheduler-standalone";
 import { SchedulerSingleton } from "./abstractions/SchedulerSingleton.js";
 import { SCHEDULED_ACTION_RECOVER_EVENT_IDENTIFIER, ScheduledActionRecoverEventType } from "./ScheduledActionRecoverEventType.js";
 import { RecoverScheduledActionsHandler } from "./RecoverScheduledActionsHandler.js";
-const consoleLogger = {
-    trace: (...args)=>console.trace(...args),
-    debug: (...args)=>console.debug(...args),
-    info: (...args)=>console.info(...args),
-    warn: (...args)=>console.warn(...args),
-    error: (...args)=>console.error(...args),
-    fatal: (...args)=>console.error(...args),
-    log: (...args)=>console.log(...args)
-};
+import { RunScheduledActionOnceDecorator } from "./RunScheduledActionOnceDecorator.js";
+import { consoleLogger } from "../logger/consoleLogger.js";
 function registerSchedulerServer(rootContainer) {
     const dispatcher = rootContainer.resolve(EventDispatcher);
     const service = new BreeSchedulerService({
         logger: consoleLogger,
-        onTrigger: async (id, namespace, tenant)=>{
+        runAction: async (id, namespace, tenant)=>{
             const event = {
                 [SCHEDULED_ACTION_EVENT_IDENTIFIER]: {
                     id,
@@ -30,7 +23,11 @@ function registerSchedulerServer(rootContainer) {
             try {
                 await dispatcher.dispatch(event);
             } catch (err) {
-                console.error(`[scheduler] scheduled action "${id}" failed:`, err);
+                consoleLogger.error({
+                    error: err,
+                    id,
+                    tenant
+                }, "[scheduler] scheduled action failed");
             }
         }
     });
@@ -38,24 +35,52 @@ function registerSchedulerServer(rootContainer) {
     rootContainer.registerInstance(SchedulerService, service);
     rootContainer.register(ScheduledActionEventType);
     rootContainer.register(ScheduledActionLambdaHandler);
+    rootContainer.registerDecorator(RunScheduledActionOnceDecorator);
     rootContainer.register(ScheduledActionRecoverEventType);
     rootContainer.register(RecoverScheduledActionsHandler);
 }
-async function recoverPendingSchedules(dispatcher) {
+const RECOVER_EVERY_MS = 60000;
+async function recoverPendingSchedules(dispatcher, atBoot) {
     const event = {
         [SCHEDULED_ACTION_RECOVER_EVENT_IDENTIFIER]: true
     };
     try {
         const result = await dispatcher.dispatch(event);
-        console.log(`[scheduler] boot recovery: re-armed ${result.recovered} pending action(s)`);
+        if (atBoot) consoleLogger.info({
+            recovered: result.recovered
+        }, "[scheduler] boot recovery re-armed pending actions");
+        if (result.failedTenants.length > 0) consoleLogger.error({
+            tenants: result.failedTenants
+        }, "[scheduler] recovery failed for some tenants");
     } catch (err) {
-        console.error("[scheduler] boot recovery failed:", err);
+        consoleLogger.error({
+            error: err
+        }, "[scheduler] recovery failed");
     }
 }
 async function startSchedulerServer(rootContainer) {
-    await rootContainer.resolve(SchedulerSingleton).start();
+    const scheduler = rootContainer.resolve(SchedulerSingleton);
+    await scheduler.start();
     const dispatcher = rootContainer.resolve(EventDispatcher);
-    recoverPendingSchedules(dispatcher);
+    let recovering = false;
+    const recover = async (atBoot)=>{
+        if (recovering) return;
+        recovering = true;
+        try {
+            await recoverPendingSchedules(dispatcher, atBoot);
+        } finally{
+            recovering = false;
+        }
+    };
+    recover(true);
+    const everyMinute = setInterval(()=>{
+        recover(false);
+    }, RECOVER_EVERY_MS);
+    return async (timeoutMs)=>{
+        clearInterval(everyMinute);
+        await scheduler.stop();
+        await scheduler.waitForRunning(timeoutMs);
+    };
 }
 export { registerSchedulerServer, startSchedulerServer };