alepha 0.20.1 → 0.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/files/index.js +2 -1
- package/dist/api/files/index.js.map +1 -1
- package/dist/api/jobs/index.browser.js +64 -148
- package/dist/api/jobs/index.browser.js.map +1 -1
- package/dist/api/jobs/index.d.ts +371 -573
- package/dist/api/jobs/index.d.ts.map +1 -1
- package/dist/api/jobs/index.js +605 -1012
- package/dist/api/jobs/index.js.map +1 -1
- package/dist/api/notifications/index.d.ts +78 -17
- package/dist/api/notifications/index.d.ts.map +1 -1
- package/dist/api/notifications/index.js +90 -23
- package/dist/api/notifications/index.js.map +1 -1
- package/dist/api/payments/index.d.ts +2 -1
- package/dist/api/payments/index.d.ts.map +1 -1
- package/dist/api/payments/index.js +4 -2
- package/dist/api/payments/index.js.map +1 -1
- package/dist/api/users/index.d.ts +34 -31
- package/dist/api/users/index.d.ts.map +1 -1
- package/dist/api/users/index.js +13 -7
- package/dist/api/users/index.js.map +1 -1
- package/dist/api/verifications/index.js +2 -1
- package/dist/api/verifications/index.js.map +1 -1
- package/dist/cli/core/index.d.ts +8 -34
- package/dist/cli/core/index.d.ts.map +1 -1
- package/dist/cli/core/index.js +43 -232
- package/dist/cli/core/index.js.map +1 -1
- package/dist/cli/platform/index.d.ts +36 -11
- package/dist/cli/platform/index.d.ts.map +1 -1
- package/dist/cli/platform/index.js +93 -27
- package/dist/cli/platform/index.js.map +1 -1
- package/dist/command/index.d.ts +1 -1
- package/dist/core/index.browser.js +6 -0
- package/dist/core/index.browser.js.map +1 -1
- package/dist/core/index.d.ts +6 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +6 -0
- package/dist/core/index.js.map +1 -1
- package/dist/core/index.native.js +6 -0
- package/dist/core/index.native.js.map +1 -1
- package/dist/core/index.workerd.js +6 -0
- package/dist/core/index.workerd.js.map +1 -1
- package/dist/react/form/index.d.ts +60 -1
- package/dist/react/form/index.d.ts.map +1 -1
- package/dist/react/form/index.js +86 -1
- package/dist/react/form/index.js.map +1 -1
- package/dist/react/head/index.browser.js +16 -1
- package/dist/react/head/index.browser.js.map +1 -1
- package/dist/react/head/index.d.ts +6 -0
- package/dist/react/head/index.d.ts.map +1 -1
- package/dist/react/head/index.js +16 -1
- package/dist/react/head/index.js.map +1 -1
- package/dist/react/router/index.browser.js +0 -10
- package/dist/react/router/index.browser.js.map +1 -1
- package/dist/react/router/index.d.ts +35 -12
- package/dist/react/router/index.d.ts.map +1 -1
- package/dist/react/router/index.js +0 -10
- package/dist/react/router/index.js.map +1 -1
- package/dist/react/ui/index.d.ts +124 -0
- package/dist/react/ui/index.d.ts.map +1 -0
- package/dist/react/ui/index.js +206 -0
- package/dist/react/ui/index.js.map +1 -0
- package/dist/router/index.d.ts +13 -13
- package/dist/router/index.d.ts.map +1 -1
- package/dist/router/index.js +45 -32
- package/dist/router/index.js.map +1 -1
- package/dist/system/index.d.ts.map +1 -1
- package/dist/system/index.js +1 -0
- package/dist/system/index.js.map +1 -1
- package/dist/topic/core/index.js +1 -1
- package/dist/topic/core/index.js.map +1 -1
- package/package.json +6 -23
- package/src/api/files/jobs/FileJobs.ts +2 -1
- package/src/api/jobs/__tests__/$job.spec.ts +316 -2867
- package/src/api/jobs/controllers/AdminJobController.ts +29 -138
- package/src/api/jobs/entities/jobExecutionEntity.ts +27 -19
- package/src/api/jobs/index.browser.ts +5 -7
- package/src/api/jobs/index.ts +23 -51
- package/src/api/jobs/primitives/$job.ts +66 -58
- package/src/api/jobs/providers/JobProvider.ts +561 -566
- package/src/api/jobs/providers/JobQueueProvider.ts +18 -19
- package/src/api/jobs/schemas/jobConfigAtom.ts +20 -23
- package/src/api/jobs/schemas/jobExecutionQuerySchema.ts +3 -27
- package/src/api/jobs/schemas/jobExecutionResourceSchema.ts +5 -7
- package/src/api/jobs/schemas/jobRegistrationSchema.ts +7 -4
- package/src/api/jobs/schemas/triggerJobSchema.ts +0 -1
- package/src/api/jobs/services/JobService.ts +90 -483
- package/src/api/notifications/controllers/AdminNotificationController.ts +19 -12
- package/src/api/notifications/index.ts +7 -4
- package/src/api/notifications/jobs/NotificationJobs.ts +83 -12
- package/src/api/payments/services/PaymentService.ts +4 -2
- package/src/api/users/__tests__/UserJobs.spec.ts +10 -49
- package/src/api/users/audits/UserAudits.ts +3 -1
- package/src/api/users/buckets/UserBuckets.ts +2 -1
- package/src/api/users/index.ts +1 -4
- package/src/api/users/jobs/UserJobs.ts +5 -4
- package/src/api/verifications/jobs/VerificationJobs.ts +2 -1
- package/src/cli/core/__tests__/init.spec.ts +1 -1
- package/src/cli/core/commands/init.ts +0 -12
- package/src/cli/core/services/PackageManagerUtils.ts +2 -9
- package/src/cli/core/services/ProjectScaffolder.ts +17 -65
- package/src/cli/core/templates/agentMd.ts +2 -8
- package/src/cli/core/templates/apiIndexTs.ts +4 -18
- package/src/cli/core/templates/mainCss.ts +1 -36
- package/src/cli/core/templates/vitestConfigTs.ts +17 -0
- package/src/cli/core/templates/webAppRouterTs.ts +2 -85
- package/src/cli/platform/__tests__/CloudflareAdapter.spec.ts +22 -71
- package/src/cli/platform/adapters/CloudflareAdapter.ts +12 -11
- package/src/cli/platform/atoms/platformOptions.ts +9 -0
- package/src/cli/platform/schemas/cloudflare.ts +3 -2
- package/src/cli/platform/services/CloudflareApi.ts +164 -25
- package/src/cli/platform/services/WranglerApi.ts +0 -17
- package/src/core/Alepha.ts +9 -0
- package/src/react/form/index.ts +2 -0
- package/src/react/form/services/parseField.ts +163 -0
- package/src/react/form/services/prettyName.ts +19 -0
- package/src/react/head/providers/BrowserHeadProvider.ts +31 -10
- package/src/react/router/primitives/$page.ts +35 -12
- package/src/react/ui/atoms/uiAtom.ts +28 -0
- package/src/react/ui/components/ColorScheme.tsx +36 -0
- package/src/react/ui/hooks/useColorMode.ts +49 -0
- package/src/react/ui/hooks/useSidebarState.ts +26 -0
- package/src/react/ui/hooks/useTheme.ts +22 -0
- package/src/react/ui/index.ts +35 -0
- package/src/react/ui/services/UiPersistence.ts +41 -0
- package/src/router/TemplatedPathParser.ts +50 -51
- package/src/router/__tests__/RouterProvider.spec.ts +62 -0
- package/src/router/__tests__/TemplatedPathParser.spec.ts +18 -0
- package/src/router/providers/RouterProvider.ts +10 -5
- package/src/system/providers/NodeShellProvider.ts +1 -0
- package/src/topic/core/providers/TopicProvider.ts +1 -1
- package/dist/api/invitations/index.d.ts +0 -790
- package/dist/api/invitations/index.d.ts.map +0 -1
- package/dist/api/invitations/index.js +0 -662
- package/dist/api/invitations/index.js.map +0 -1
- package/dist/api/issues/index.d.ts +0 -810
- package/dist/api/issues/index.d.ts.map +0 -1
- package/dist/api/issues/index.js +0 -444
- package/dist/api/issues/index.js.map +0 -1
- package/dist/api/subscriptions/index.d.ts +0 -1692
- package/dist/api/subscriptions/index.d.ts.map +0 -1
- package/dist/api/subscriptions/index.js +0 -1867
- package/dist/api/subscriptions/index.js.map +0 -1
- package/dist/api/workflows/index.browser.js +0 -246
- package/dist/api/workflows/index.browser.js.map +0 -1
- package/dist/api/workflows/index.d.ts +0 -1618
- package/dist/api/workflows/index.d.ts.map +0 -1
- package/dist/api/workflows/index.js +0 -1495
- package/dist/api/workflows/index.js.map +0 -1
- package/src/api/invitations/__tests__/InvitationService.spec.ts +0 -439
- package/src/api/invitations/controllers/AdminInvitationController.ts +0 -86
- package/src/api/invitations/controllers/InvitationController.ts +0 -84
- package/src/api/invitations/entities/invitations.ts +0 -33
- package/src/api/invitations/index.ts +0 -58
- package/src/api/invitations/jobs/InvitationJobs.ts +0 -37
- package/src/api/invitations/providers/InvitationProvider.ts +0 -45
- package/src/api/invitations/schemas/createInvitationSchema.ts +0 -12
- package/src/api/invitations/schemas/invitationConfigAtom.ts +0 -20
- package/src/api/invitations/schemas/invitationQuerySchema.ts +0 -15
- package/src/api/invitations/schemas/invitationResourceSchema.ts +0 -6
- package/src/api/invitations/schemas/invitationWithResourceInfoSchema.ts +0 -22
- package/src/api/invitations/schemas/myInvitationsQuerySchema.ts +0 -10
- package/src/api/invitations/services/InvitationService.ts +0 -556
- package/src/api/issues/__tests__/IssueService.spec.ts +0 -263
- package/src/api/issues/controllers/AdminIssueController.ts +0 -149
- package/src/api/issues/controllers/IssueController.ts +0 -44
- package/src/api/issues/entities/issues.ts +0 -49
- package/src/api/issues/index.ts +0 -50
- package/src/api/issues/schemas/createIssueSchema.ts +0 -13
- package/src/api/issues/schemas/issueConfigAtom.ts +0 -13
- package/src/api/issues/schemas/issueQuerySchema.ts +0 -18
- package/src/api/issues/schemas/issueResourceSchema.ts +0 -6
- package/src/api/issues/schemas/myIssueQuerySchema.ts +0 -10
- package/src/api/issues/schemas/updateIssueSchema.ts +0 -13
- package/src/api/issues/services/IssueService.ts +0 -264
- package/src/api/jobs/__tests__/$job-middleware.spec.ts +0 -126
- package/src/api/jobs/__tests__/JobService.spec.ts +0 -31
- package/src/api/jobs/entities/jobExecutionLogEntity.ts +0 -13
- package/src/api/jobs/schemas/jobActivitySchema.ts +0 -15
- package/src/api/jobs/schemas/jobCronInfoSchema.ts +0 -22
- package/src/api/jobs/schemas/jobExecutionDetailResourceSchema.ts +0 -20
- package/src/api/jobs/schemas/jobFailureSchema.ts +0 -9
- package/src/api/jobs/schemas/jobQueueDepthSchema.ts +0 -14
- package/src/api/jobs/schemas/jobStatsSchema.ts +0 -14
- package/src/api/jobs/services/JobService-tests.ts +0 -157
- package/src/api/subscriptions/__tests__/BillingService.spec.ts +0 -218
- package/src/api/subscriptions/__tests__/SubscriptionService.spec.ts +0 -278
- package/src/api/subscriptions/controllers/AdminSubscriptionController.ts +0 -212
- package/src/api/subscriptions/controllers/SubscriptionController.ts +0 -189
- package/src/api/subscriptions/entities/subscriptionEvents.ts +0 -54
- package/src/api/subscriptions/entities/subscriptions.ts +0 -68
- package/src/api/subscriptions/index.ts +0 -133
- package/src/api/subscriptions/jobs/SubscriptionJobs.ts +0 -382
- package/src/api/subscriptions/middleware/$requireLimit.ts +0 -50
- package/src/api/subscriptions/middleware/$requirePlan.ts +0 -49
- package/src/api/subscriptions/notifications/SubscriptionNotifications.ts +0 -110
- package/src/api/subscriptions/schemas/cancelSubscriptionSchema.ts +0 -8
- package/src/api/subscriptions/schemas/changePlanSchema.ts +0 -9
- package/src/api/subscriptions/schemas/createSubscriptionSchema.ts +0 -11
- package/src/api/subscriptions/schemas/entitlementsSchema.ts +0 -21
- package/src/api/subscriptions/schemas/mrrSchema.ts +0 -13
- package/src/api/subscriptions/schemas/planDefinitionSchema.ts +0 -71
- package/src/api/subscriptions/schemas/planResourceSchema.ts +0 -25
- package/src/api/subscriptions/schemas/subscriptionEventResourceSchema.ts +0 -8
- package/src/api/subscriptions/schemas/subscriptionQuerySchema.ts +0 -19
- package/src/api/subscriptions/schemas/subscriptionResourceSchema.ts +0 -6
- package/src/api/subscriptions/schemas/subscriptionSettingsSchema.ts +0 -32
- package/src/api/subscriptions/schemas/subscriptionStatsSchema.ts +0 -23
- package/src/api/subscriptions/services/BillingService.ts +0 -437
- package/src/api/subscriptions/services/SubscriptionConfig.ts +0 -56
- package/src/api/subscriptions/services/SubscriptionService.ts +0 -867
- package/src/api/subscriptions/services/UsageService.ts +0 -118
- package/src/api/workflows/__tests__/$workflow.spec.ts +0 -616
- package/src/api/workflows/controllers/AdminWorkflowController.ts +0 -191
- package/src/api/workflows/entities/workflowExecutions.ts +0 -74
- package/src/api/workflows/entities/workflowStepExecutions.ts +0 -74
- package/src/api/workflows/entities/workflowStepLogs.ts +0 -13
- package/src/api/workflows/index.browser.ts +0 -22
- package/src/api/workflows/index.ts +0 -115
- package/src/api/workflows/jobs/WorkflowJobs.ts +0 -77
- package/src/api/workflows/primitives/$workflow.ts +0 -202
- package/src/api/workflows/providers/WorkflowProvider.ts +0 -1284
- package/src/api/workflows/schemas/workflowActivitySchema.ts +0 -15
- package/src/api/workflows/schemas/workflowConfigAtom.ts +0 -51
- package/src/api/workflows/schemas/workflowExecutionDetailSchema.ts +0 -18
- package/src/api/workflows/schemas/workflowExecutionQuerySchema.ts +0 -26
- package/src/api/workflows/schemas/workflowExecutionResourceSchema.ts +0 -30
- package/src/api/workflows/schemas/workflowRegistrationSchema.ts +0 -26
- package/src/api/workflows/schemas/workflowStatsSchema.ts +0 -16
- package/src/api/workflows/schemas/workflowStepExecutionResourceSchema.ts +0 -15
- package/src/api/workflows/services/WorkflowService.ts +0 -382
- package/src/cli/core/templates/apiAppSecurityTs.ts +0 -43
- package/src/cli/core/templates/webAdminDashboardTsx.ts +0 -17
|
@@ -8,7 +8,6 @@ import {
|
|
|
8
8
|
type TSchema,
|
|
9
9
|
} from "alepha";
|
|
10
10
|
import { DateTimeProvider, type DurationLike } from "alepha/datetime";
|
|
11
|
-
import { LockProvider } from "alepha/lock";
|
|
12
11
|
import type { LogEntry } from "alepha/logger";
|
|
13
12
|
import { $logger } from "alepha/logger";
|
|
14
13
|
import { $repository } from "alepha/orm";
|
|
@@ -17,9 +16,7 @@ import {
|
|
|
17
16
|
type JobStatus,
|
|
18
17
|
jobExecutionEntity,
|
|
19
18
|
} from "../entities/jobExecutionEntity.ts";
|
|
20
|
-
import { jobExecutionLogEntity } from "../entities/jobExecutionLogEntity.ts";
|
|
21
19
|
import type {
|
|
22
|
-
JobItem,
|
|
23
20
|
JobPrimitiveOptions,
|
|
24
21
|
JobPriority,
|
|
25
22
|
JobRetryBackoff,
|
|
@@ -29,7 +26,7 @@ import { jobConfig } from "../schemas/jobConfigAtom.ts";
|
|
|
29
26
|
|
|
30
27
|
// -----------------------------------------------------------------------------------------------------------------
|
|
31
28
|
|
|
32
|
-
const PRIORITY_MAP: Record<
|
|
29
|
+
const PRIORITY_MAP: Record<JobPriority, number> = {
|
|
33
30
|
critical: 0,
|
|
34
31
|
high: 1,
|
|
35
32
|
normal: 2,
|
|
@@ -43,6 +40,8 @@ const PRIORITY_REVERSE: Record<number, JobPriority> = {
|
|
|
43
40
|
3: "low",
|
|
44
41
|
};
|
|
45
42
|
|
|
43
|
+
const SWEEP_CRON = "*/5 * * * *";
|
|
44
|
+
|
|
46
45
|
// -----------------------------------------------------------------------------------------------------------------
|
|
47
46
|
|
|
48
47
|
export interface PushOptions {
|
|
@@ -50,6 +49,8 @@ export interface PushOptions {
|
|
|
50
49
|
key?: string;
|
|
51
50
|
priority?: JobPriority;
|
|
52
51
|
scheduledAt?: Date;
|
|
52
|
+
triggeredBy?: string;
|
|
53
|
+
triggeredByName?: string;
|
|
53
54
|
}
|
|
54
55
|
|
|
55
56
|
export interface PushManyItem<T extends TSchema = TSchema> {
|
|
@@ -60,8 +61,8 @@ export interface PushManyItem<T extends TSchema = TSchema> {
|
|
|
60
61
|
scheduledAt?: Date;
|
|
61
62
|
}
|
|
62
63
|
|
|
63
|
-
export interface JobTriggerContext {
|
|
64
|
-
payload?:
|
|
64
|
+
export interface JobTriggerContext<T extends TSchema = TSchema> {
|
|
65
|
+
payload?: Static<T>;
|
|
65
66
|
triggeredBy?: string;
|
|
66
67
|
triggeredByName?: string;
|
|
67
68
|
}
|
|
@@ -71,49 +72,73 @@ export interface CancelContext {
|
|
|
71
72
|
cancelledByName?: string;
|
|
72
73
|
}
|
|
73
74
|
|
|
74
|
-
interface
|
|
75
|
+
interface JobRuntimeRegistration {
|
|
75
76
|
name: string;
|
|
76
77
|
options: JobPrimitiveOptions;
|
|
78
|
+
type: "cron" | "queue";
|
|
77
79
|
}
|
|
78
80
|
|
|
79
81
|
// -----------------------------------------------------------------------------------------------------------------
|
|
80
82
|
|
|
83
|
+
/**
|
|
84
|
+
* Coordinates cron (scheduler) and queue (push) jobs with a durable outbox
|
|
85
|
+
* table and a single reconciliation sweep.
|
|
86
|
+
*
|
|
87
|
+
* Queue-mode flow:
|
|
88
|
+
* push() → INSERT row (pending) + queue.send({ executionId })
|
|
89
|
+
* worker → SELECT row → UPDATE running → handler → DELETE (ok) / UPDATE (error)
|
|
90
|
+
*
|
|
91
|
+
* Cron-mode flow:
|
|
92
|
+
* scheduler tick → handler runs inline → INSERT row only on error
|
|
93
|
+
*
|
|
94
|
+
* Sweep responsibilities (every `sweepInterval`):
|
|
95
|
+
* - re-enqueue pending rows older than `staleThreshold`
|
|
96
|
+
* - fail running rows older than `max(timeout*2, runTimeout)`
|
|
97
|
+
* - move `scheduled` rows with `scheduledAt <= now` to pending + enqueue
|
|
98
|
+
* - trim per-job history beyond `keepLastSuccess` / `keepLastError`
|
|
99
|
+
*/
|
|
81
100
|
export class JobProvider {
|
|
82
101
|
protected readonly alepha = $inject(Alepha);
|
|
83
102
|
protected readonly dt = $inject(DateTimeProvider);
|
|
84
103
|
protected readonly cronProvider = $inject(CronProvider);
|
|
85
|
-
protected readonly lockProvider = $inject(LockProvider);
|
|
86
104
|
protected readonly config = $state(jobConfig);
|
|
87
105
|
protected readonly log = $logger();
|
|
88
106
|
protected readonly executions = $repository(jobExecutionEntity);
|
|
89
|
-
protected readonly executionLogs = $repository(jobExecutionLogEntity);
|
|
90
107
|
|
|
91
|
-
protected readonly jobs = new Map<string,
|
|
92
|
-
protected readonly pausedJobs = new Set<string>();
|
|
108
|
+
protected readonly jobs = new Map<string, JobRuntimeRegistration>();
|
|
93
109
|
protected readonly inFlight = new Set<Promise<void>>();
|
|
110
|
+
protected readonly abortControllers = new Map<string, AbortController>();
|
|
111
|
+
protected readonly perExecutionLogs = new Map<string, LogEntry[]>();
|
|
112
|
+
protected stopping = false;
|
|
94
113
|
|
|
95
114
|
/**
|
|
96
|
-
*
|
|
97
|
-
* When null,
|
|
115
|
+
* Set by `JobQueueProvider` when `AlephaApiJobsQueue` is loaded.
|
|
116
|
+
* When null, queue-mode jobs cannot be pushed.
|
|
98
117
|
*/
|
|
99
118
|
public queueDispatch:
|
|
100
119
|
| ((jobName: string, executionId: string) => Promise<void>)
|
|
101
120
|
| null = null;
|
|
102
|
-
protected readonly logs = new Map<string, LogEntry[]>();
|
|
103
|
-
protected readonly abortControllers = new Map<string, AbortController>();
|
|
104
|
-
protected static readonly SWEEP_CRON = "*/5 * * * *";
|
|
105
|
-
protected stopping = false;
|
|
106
|
-
protected workerId = "";
|
|
107
121
|
|
|
108
|
-
// --- Registration
|
|
122
|
+
// --- Registration -----------------------------------------------------------------------------------------------
|
|
109
123
|
|
|
110
124
|
public registerJob(name: string, options: JobPrimitiveOptions): void {
|
|
111
125
|
if (this.jobs.has(name)) {
|
|
112
126
|
throw new AlephaError(`Job already registered: ${name}`);
|
|
113
127
|
}
|
|
128
|
+
if (options.cron && options.schema) {
|
|
129
|
+
throw new AlephaError(
|
|
130
|
+
`Job '${name}' declares both 'cron' and 'schema'. A job must be either cron-only (recurring) or queue-only (push-based). Split into two jobs.`,
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
if (!options.cron && !options.schema) {
|
|
134
|
+
throw new AlephaError(
|
|
135
|
+
`Job '${name}' must declare either 'cron' (for recurring tasks) or 'schema' (for queue-mode tasks).`,
|
|
136
|
+
);
|
|
137
|
+
}
|
|
114
138
|
|
|
115
|
-
|
|
116
|
-
this.
|
|
139
|
+
const type: "cron" | "queue" = options.cron ? "cron" : "queue";
|
|
140
|
+
this.jobs.set(name, { name, options, type });
|
|
141
|
+
this.log.debug(`Registered ${type} job '${name}'`, {
|
|
117
142
|
cron: options.cron,
|
|
118
143
|
priority: options.priority ?? "normal",
|
|
119
144
|
retries: options.retry?.retries ?? 0,
|
|
@@ -122,25 +147,180 @@ export class JobProvider {
|
|
|
122
147
|
if (options.cron) {
|
|
123
148
|
this.cronProvider.createCronJob(name, options.cron, async () => {
|
|
124
149
|
try {
|
|
125
|
-
await this.
|
|
126
|
-
triggeredBy: "system",
|
|
127
|
-
triggeredByName: "system (cron)",
|
|
128
|
-
});
|
|
150
|
+
await this.runCron(name);
|
|
129
151
|
} catch (error) {
|
|
130
|
-
this.log.error(`Cron
|
|
152
|
+
this.log.error(`Cron tick failed for job '${name}'`, error);
|
|
131
153
|
}
|
|
132
154
|
});
|
|
133
155
|
}
|
|
134
156
|
}
|
|
135
157
|
|
|
158
|
+
public getRegisteredJobs(): Map<string, JobRuntimeRegistration> {
|
|
159
|
+
return this.jobs;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// --- Cron execution (inline, no queue) --------------------------------------------------------------------------
|
|
163
|
+
|
|
164
|
+
protected async runCron(name: string): Promise<void> {
|
|
165
|
+
const registration = this.getRegistration(name);
|
|
166
|
+
if (registration.type !== "cron") {
|
|
167
|
+
throw new AlephaError(`Job '${name}' is not cron-mode`);
|
|
168
|
+
}
|
|
169
|
+
if (this.stopping) return;
|
|
170
|
+
|
|
171
|
+
const executionId = crypto.randomUUID();
|
|
172
|
+
const promise = this.executeInline(registration, executionId, {
|
|
173
|
+
payload: undefined,
|
|
174
|
+
attempt: 1,
|
|
175
|
+
triggeredBy: "system",
|
|
176
|
+
triggeredByName: "system (cron)",
|
|
177
|
+
});
|
|
178
|
+
this.inFlight.add(promise);
|
|
179
|
+
try {
|
|
180
|
+
await promise;
|
|
181
|
+
} finally {
|
|
182
|
+
this.inFlight.delete(promise);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
136
186
|
/**
|
|
137
|
-
*
|
|
187
|
+
* Execute a cron handler inline. Records a row only on error (or always,
|
|
188
|
+
* when `record: 'all'`). No DB writes on the happy path by default.
|
|
138
189
|
*/
|
|
139
|
-
|
|
140
|
-
|
|
190
|
+
protected async executeInline(
|
|
191
|
+
registration: JobRuntimeRegistration,
|
|
192
|
+
executionId: string,
|
|
193
|
+
ctx: {
|
|
194
|
+
payload: unknown;
|
|
195
|
+
attempt: number;
|
|
196
|
+
triggeredBy?: string;
|
|
197
|
+
triggeredByName?: string;
|
|
198
|
+
},
|
|
199
|
+
): Promise<void> {
|
|
200
|
+
const opts = registration.options;
|
|
201
|
+
const name = registration.name;
|
|
202
|
+
const record = opts.record ?? "error";
|
|
203
|
+
const contextId = this.alepha.context.createContextId();
|
|
204
|
+
this.perExecutionLogs.set(contextId, []);
|
|
205
|
+
|
|
206
|
+
const abortController = new AbortController();
|
|
207
|
+
this.abortControllers.set(executionId, abortController);
|
|
208
|
+
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
|
209
|
+
if (opts.timeout) {
|
|
210
|
+
const ms = this.dt.duration(opts.timeout).as("milliseconds");
|
|
211
|
+
timeoutId = setTimeout(() => abortController.abort(), ms);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
const startedAt = this.dt.now();
|
|
215
|
+
|
|
216
|
+
try {
|
|
217
|
+
await this.alepha.context.run(
|
|
218
|
+
async () => {
|
|
219
|
+
await this.alepha.events.emit("job:begin", {
|
|
220
|
+
name,
|
|
221
|
+
now: startedAt,
|
|
222
|
+
executionId,
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
try {
|
|
226
|
+
await opts.handler({
|
|
227
|
+
payload: ctx.payload,
|
|
228
|
+
attempt: ctx.attempt,
|
|
229
|
+
now: startedAt,
|
|
230
|
+
signal: abortController.signal,
|
|
231
|
+
executionId,
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
if (record === "all") {
|
|
235
|
+
await this.writeTerminalRow(executionId, name, "ok", {
|
|
236
|
+
payload: ctx.payload,
|
|
237
|
+
attempt: ctx.attempt,
|
|
238
|
+
startedAt,
|
|
239
|
+
error: undefined,
|
|
240
|
+
context: contextId,
|
|
241
|
+
triggeredBy: ctx.triggeredBy,
|
|
242
|
+
triggeredByName: ctx.triggeredByName,
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
await this.alepha.events.emit(
|
|
247
|
+
"job:success",
|
|
248
|
+
{ name, executionId },
|
|
249
|
+
{ catch: true },
|
|
250
|
+
);
|
|
251
|
+
} catch (error) {
|
|
252
|
+
const err =
|
|
253
|
+
error instanceof Error ? error : new Error(String(error));
|
|
254
|
+
if (record !== "none") {
|
|
255
|
+
await this.writeTerminalRow(executionId, name, "error", {
|
|
256
|
+
payload: ctx.payload,
|
|
257
|
+
attempt: ctx.attempt,
|
|
258
|
+
startedAt,
|
|
259
|
+
error: err,
|
|
260
|
+
context: contextId,
|
|
261
|
+
triggeredBy: ctx.triggeredBy,
|
|
262
|
+
triggeredByName: ctx.triggeredByName,
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
await this.alepha.events.emit(
|
|
266
|
+
"job:error",
|
|
267
|
+
{ name, error: err, executionId },
|
|
268
|
+
{ catch: true },
|
|
269
|
+
);
|
|
270
|
+
} finally {
|
|
271
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
272
|
+
this.abortControllers.delete(executionId);
|
|
273
|
+
await this.alepha.events.emit(
|
|
274
|
+
"job:end",
|
|
275
|
+
{ name, executionId },
|
|
276
|
+
{ catch: true },
|
|
277
|
+
);
|
|
278
|
+
}
|
|
279
|
+
},
|
|
280
|
+
{ context: contextId },
|
|
281
|
+
);
|
|
282
|
+
} finally {
|
|
283
|
+
this.perExecutionLogs.delete(contextId);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
protected async writeTerminalRow(
|
|
288
|
+
executionId: string,
|
|
289
|
+
jobName: string,
|
|
290
|
+
status: "ok" | "error",
|
|
291
|
+
fields: {
|
|
292
|
+
payload: unknown;
|
|
293
|
+
attempt: number;
|
|
294
|
+
startedAt: ReturnType<DateTimeProvider["now"]>;
|
|
295
|
+
error?: Error;
|
|
296
|
+
context: string;
|
|
297
|
+
triggeredBy?: string;
|
|
298
|
+
triggeredByName?: string;
|
|
299
|
+
},
|
|
300
|
+
): Promise<void> {
|
|
301
|
+
try {
|
|
302
|
+
const logs =
|
|
303
|
+
status === "error" ? this.snapshotLogs(fields.context) : undefined;
|
|
304
|
+
await this.executions.create({
|
|
305
|
+
id: executionId,
|
|
306
|
+
jobName,
|
|
307
|
+
status,
|
|
308
|
+
payload: fields.payload as Record<string, unknown> | undefined,
|
|
309
|
+
attempt: fields.attempt,
|
|
310
|
+
maxAttempts: fields.attempt,
|
|
311
|
+
startedAt: fields.startedAt.toISOString(),
|
|
312
|
+
completedAt: this.dt.nowISOString(),
|
|
313
|
+
error: fields.error?.message,
|
|
314
|
+
logs,
|
|
315
|
+
triggeredBy: fields.triggeredBy,
|
|
316
|
+
triggeredByName: fields.triggeredByName,
|
|
317
|
+
});
|
|
318
|
+
} catch (e) {
|
|
319
|
+
this.log.warn(`Failed to write terminal row for ${executionId}`, e);
|
|
320
|
+
}
|
|
141
321
|
}
|
|
142
322
|
|
|
143
|
-
// ---
|
|
323
|
+
// --- Queue push -------------------------------------------------------------------------------------------------
|
|
144
324
|
|
|
145
325
|
public async push(
|
|
146
326
|
name: string,
|
|
@@ -148,15 +328,13 @@ export class JobProvider {
|
|
|
148
328
|
options?: PushOptions,
|
|
149
329
|
): Promise<string> {
|
|
150
330
|
const registration = this.getRegistration(name);
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
if (!opts.schema) {
|
|
331
|
+
if (registration.type !== "queue") {
|
|
154
332
|
throw new AlephaError(
|
|
155
|
-
`
|
|
333
|
+
`Job '${name}' is not queue-mode (no schema declared). Use trigger() instead.`,
|
|
156
334
|
);
|
|
157
335
|
}
|
|
158
|
-
|
|
159
|
-
const validated = this.alepha.codec.validate(opts.schema
|
|
336
|
+
const opts = registration.options;
|
|
337
|
+
const validated = this.alepha.codec.validate(opts.schema!, payload);
|
|
160
338
|
|
|
161
339
|
const priority =
|
|
162
340
|
PRIORITY_MAP[options?.priority ?? opts.priority ?? "normal"];
|
|
@@ -169,38 +347,38 @@ export class JobProvider {
|
|
|
169
347
|
if (options?.scheduledAt) {
|
|
170
348
|
scheduledAt = options.scheduledAt.toISOString();
|
|
171
349
|
} else if (options?.delay) {
|
|
172
|
-
|
|
173
|
-
|
|
350
|
+
scheduledAt = this.dt
|
|
351
|
+
.now()
|
|
352
|
+
.add(this.dt.duration(options.delay))
|
|
353
|
+
.toISOString();
|
|
174
354
|
}
|
|
175
355
|
|
|
176
|
-
// Keyed path: atomic upsert to avoid race between concurrent pushes
|
|
177
356
|
if (options?.key) {
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
) {
|
|
201
|
-
|
|
357
|
+
// Key-based dedup: check for existing row first, then insert.
|
|
358
|
+
// Two queries in the no-conflict path, but deterministic across dialects.
|
|
359
|
+
const existing = await this.executions.findMany({
|
|
360
|
+
where: { jobName: { eq: name }, key: { eq: options.key } },
|
|
361
|
+
limit: 1,
|
|
362
|
+
});
|
|
363
|
+
if (existing.length > 0) {
|
|
364
|
+
return existing[0].id;
|
|
365
|
+
}
|
|
366
|
+
const execution = await this.executions.create({
|
|
367
|
+
jobName: name,
|
|
368
|
+
key: options.key,
|
|
369
|
+
payload: validated as Record<string, unknown>,
|
|
370
|
+
status,
|
|
371
|
+
priority,
|
|
372
|
+
maxAttempts,
|
|
373
|
+
scheduledAt,
|
|
374
|
+
triggeredBy: options.triggeredBy,
|
|
375
|
+
triggeredByName: options.triggeredByName,
|
|
376
|
+
});
|
|
377
|
+
if (status === "pending") {
|
|
378
|
+
await this.dispatchToQueue(name, execution.id);
|
|
379
|
+
} else if (status === "scheduled" && scheduledAt) {
|
|
380
|
+
this.scheduleOptimisticDispatch(name, execution.id, scheduledAt);
|
|
202
381
|
}
|
|
203
|
-
|
|
204
382
|
return execution.id;
|
|
205
383
|
}
|
|
206
384
|
|
|
@@ -211,22 +389,38 @@ export class JobProvider {
|
|
|
211
389
|
priority,
|
|
212
390
|
maxAttempts,
|
|
213
391
|
scheduledAt,
|
|
392
|
+
triggeredBy: options?.triggeredBy,
|
|
393
|
+
triggeredByName: options?.triggeredByName,
|
|
214
394
|
});
|
|
215
395
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
});
|
|
221
|
-
|
|
222
|
-
// Dispatch to processing if immediate
|
|
223
|
-
if (status === "pending" && !this.stopping) {
|
|
224
|
-
await this.scheduleProcessing(name, execution.id);
|
|
396
|
+
if (status === "pending") {
|
|
397
|
+
await this.dispatchToQueue(name, execution.id);
|
|
398
|
+
} else if (status === "scheduled" && scheduledAt) {
|
|
399
|
+
this.scheduleOptimisticDispatch(name, execution.id, scheduledAt);
|
|
225
400
|
}
|
|
226
|
-
|
|
227
401
|
return execution.id;
|
|
228
402
|
}
|
|
229
403
|
|
|
404
|
+
/**
|
|
405
|
+
* Fire a local setTimeout so delayed/retrying rows dispatch as close to
|
|
406
|
+
* `scheduledAt` as possible, rather than waiting for the next sweep tick.
|
|
407
|
+
* No-op on stateless runtimes where timers won't survive (the sweep
|
|
408
|
+
* handles those).
|
|
409
|
+
*/
|
|
410
|
+
protected scheduleOptimisticDispatch(
|
|
411
|
+
jobName: string,
|
|
412
|
+
executionId: string,
|
|
413
|
+
scheduledAt: string,
|
|
414
|
+
): void {
|
|
415
|
+
const delayMs = Math.max(
|
|
416
|
+
0,
|
|
417
|
+
new Date(scheduledAt).getTime() - this.dt.nowMillis(),
|
|
418
|
+
);
|
|
419
|
+
this.dt.createTimeout(() => {
|
|
420
|
+
void this.dispatchScheduled(jobName, executionId);
|
|
421
|
+
}, delayMs);
|
|
422
|
+
}
|
|
423
|
+
|
|
230
424
|
public async pushMany(
|
|
231
425
|
name: string,
|
|
232
426
|
items: Array<PushManyItem>,
|
|
@@ -234,19 +428,16 @@ export class JobProvider {
|
|
|
234
428
|
if (items.length === 0) return [];
|
|
235
429
|
|
|
236
430
|
const registration = this.getRegistration(name);
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
if (!opts.schema) {
|
|
431
|
+
if (registration.type !== "queue") {
|
|
240
432
|
throw new AlephaError(
|
|
241
|
-
`
|
|
433
|
+
`Job '${name}' is not queue-mode (no schema declared).`,
|
|
242
434
|
);
|
|
243
435
|
}
|
|
244
|
-
|
|
436
|
+
const opts = registration.options;
|
|
245
437
|
const maxAttempts = (opts.retry?.retries ?? 0) + 1;
|
|
246
438
|
|
|
247
|
-
// Keyed items need upsert logic — fall back to individual push
|
|
248
439
|
const keyed: PushManyItem[] = [];
|
|
249
|
-
const
|
|
440
|
+
const bulk: Array<{
|
|
250
441
|
jobName: string;
|
|
251
442
|
payload: Record<string, unknown>;
|
|
252
443
|
status: JobStatus;
|
|
@@ -256,35 +447,34 @@ export class JobProvider {
|
|
|
256
447
|
}> = [];
|
|
257
448
|
|
|
258
449
|
for (const item of items) {
|
|
259
|
-
const validated = this.alepha.codec.validate(opts.schema
|
|
450
|
+
const validated = this.alepha.codec.validate(opts.schema!, item.payload);
|
|
260
451
|
if (item.key) {
|
|
261
452
|
keyed.push({ ...item, payload: validated as Static<TSchema> });
|
|
262
|
-
|
|
263
|
-
const isDelayed = item.delay || item.scheduledAt;
|
|
264
|
-
const status: JobStatus = isDelayed ? "scheduled" : "pending";
|
|
265
|
-
let scheduledAt: string | undefined;
|
|
266
|
-
if (item.scheduledAt) {
|
|
267
|
-
scheduledAt = item.scheduledAt.toISOString();
|
|
268
|
-
} else if (item.delay) {
|
|
269
|
-
scheduledAt = this.dt
|
|
270
|
-
.now()
|
|
271
|
-
.add(this.dt.duration(item.delay))
|
|
272
|
-
.toISOString();
|
|
273
|
-
}
|
|
274
|
-
bulkRows.push({
|
|
275
|
-
jobName: name,
|
|
276
|
-
payload: validated as Record<string, unknown>,
|
|
277
|
-
status,
|
|
278
|
-
priority: PRIORITY_MAP[item.priority ?? opts.priority ?? "normal"],
|
|
279
|
-
maxAttempts,
|
|
280
|
-
scheduledAt,
|
|
281
|
-
});
|
|
453
|
+
continue;
|
|
282
454
|
}
|
|
455
|
+
const isDelayed = item.delay || item.scheduledAt;
|
|
456
|
+
const status: JobStatus = isDelayed ? "scheduled" : "pending";
|
|
457
|
+
let scheduledAt: string | undefined;
|
|
458
|
+
if (item.scheduledAt) {
|
|
459
|
+
scheduledAt = item.scheduledAt.toISOString();
|
|
460
|
+
} else if (item.delay) {
|
|
461
|
+
scheduledAt = this.dt
|
|
462
|
+
.now()
|
|
463
|
+
.add(this.dt.duration(item.delay))
|
|
464
|
+
.toISOString();
|
|
465
|
+
}
|
|
466
|
+
bulk.push({
|
|
467
|
+
jobName: name,
|
|
468
|
+
payload: validated as Record<string, unknown>,
|
|
469
|
+
status,
|
|
470
|
+
priority: PRIORITY_MAP[item.priority ?? opts.priority ?? "normal"],
|
|
471
|
+
maxAttempts,
|
|
472
|
+
scheduledAt,
|
|
473
|
+
});
|
|
283
474
|
}
|
|
284
475
|
|
|
285
476
|
const ids: string[] = [];
|
|
286
477
|
|
|
287
|
-
// Keyed: sequential upserts
|
|
288
478
|
for (const item of keyed) {
|
|
289
479
|
const id = await this.push(name, item.payload, {
|
|
290
480
|
key: item.key,
|
|
@@ -295,69 +485,75 @@ export class JobProvider {
|
|
|
295
485
|
ids.push(id);
|
|
296
486
|
}
|
|
297
487
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
const created = await this.executions.createMany(bulkRows);
|
|
488
|
+
if (bulk.length > 0) {
|
|
489
|
+
const created = await this.executions.createMany(bulk);
|
|
301
490
|
for (const exec of created) {
|
|
302
491
|
ids.push(exec.id);
|
|
303
492
|
if (exec.status === "pending" && !this.stopping) {
|
|
304
|
-
await this.
|
|
493
|
+
await this.dispatchToQueue(name, exec.id);
|
|
494
|
+
} else if (
|
|
495
|
+
exec.status === "scheduled" &&
|
|
496
|
+
exec.scheduledAt &&
|
|
497
|
+
!this.stopping
|
|
498
|
+
) {
|
|
499
|
+
this.scheduleOptimisticDispatch(name, exec.id, exec.scheduledAt);
|
|
305
500
|
}
|
|
306
501
|
}
|
|
307
502
|
}
|
|
308
503
|
|
|
309
504
|
this.log.debug(`pushMany '${name}': ${ids.length} jobs created`, {
|
|
310
|
-
bulk:
|
|
505
|
+
bulk: bulk.length,
|
|
311
506
|
keyed: keyed.length,
|
|
312
507
|
});
|
|
313
508
|
|
|
314
509
|
return ids;
|
|
315
510
|
}
|
|
316
511
|
|
|
317
|
-
|
|
512
|
+
protected async dispatchToQueue(
|
|
513
|
+
jobName: string,
|
|
514
|
+
executionId: string,
|
|
515
|
+
): Promise<void> {
|
|
516
|
+
if (this.stopping) return;
|
|
517
|
+
if (!this.queueDispatch) {
|
|
518
|
+
throw new AlephaError(
|
|
519
|
+
`Queue-mode job '${jobName}' cannot be pushed: AlephaApiJobsQueue is not loaded. Add '.with(AlephaApiJobsQueue)' to your app.`,
|
|
520
|
+
);
|
|
521
|
+
}
|
|
522
|
+
await this.queueDispatch(jobName, executionId);
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
// --- Manual trigger (admin / CLI) ------------------------------------------------------------------------------
|
|
318
526
|
|
|
319
527
|
public async trigger(
|
|
320
528
|
name: string,
|
|
321
529
|
context?: JobTriggerContext,
|
|
322
530
|
): Promise<void> {
|
|
323
531
|
const registration = this.getRegistration(name);
|
|
324
|
-
const opts = registration.options;
|
|
325
532
|
|
|
326
|
-
if (
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
533
|
+
if (registration.type === "cron") {
|
|
534
|
+
const executionId = crypto.randomUUID();
|
|
535
|
+
await this.executeInline(registration, executionId, {
|
|
536
|
+
payload: undefined,
|
|
537
|
+
attempt: 1,
|
|
331
538
|
triggeredBy: context?.triggeredBy,
|
|
332
539
|
triggeredByName: context?.triggeredByName,
|
|
333
540
|
});
|
|
334
541
|
return;
|
|
335
542
|
}
|
|
336
543
|
|
|
337
|
-
//
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
jobName: name,
|
|
343
|
-
status: "pending",
|
|
344
|
-
priority,
|
|
345
|
-
maxAttempts,
|
|
346
|
-
triggeredBy: context?.triggeredBy,
|
|
347
|
-
triggeredByName: context?.triggeredByName,
|
|
348
|
-
});
|
|
349
|
-
|
|
350
|
-
this.log.debug(`Triggered job '${name}'`, {
|
|
351
|
-
executionId: execution.id,
|
|
352
|
-
triggeredBy: context?.triggeredByName ?? context?.triggeredBy,
|
|
353
|
-
});
|
|
354
|
-
|
|
355
|
-
if (!this.stopping) {
|
|
356
|
-
await this.scheduleProcessing(name, execution.id);
|
|
544
|
+
// queue-mode: treat as a normal push with the given payload
|
|
545
|
+
if (!context?.payload) {
|
|
546
|
+
throw new AlephaError(
|
|
547
|
+
`Queue-mode job '${name}' requires a payload for manual trigger.`,
|
|
548
|
+
);
|
|
357
549
|
}
|
|
550
|
+
await this.push(name, context.payload, {
|
|
551
|
+
triggeredBy: context.triggeredBy,
|
|
552
|
+
triggeredByName: context.triggeredByName,
|
|
553
|
+
});
|
|
358
554
|
}
|
|
359
555
|
|
|
360
|
-
// --- Cancel
|
|
556
|
+
// --- Cancel ----------------------------------------------------------------------------------------------------
|
|
361
557
|
|
|
362
558
|
public async cancel(
|
|
363
559
|
executionId: string,
|
|
@@ -367,10 +563,9 @@ export class JobProvider {
|
|
|
367
563
|
if (!execution) {
|
|
368
564
|
throw new AlephaError(`Execution not found: ${executionId}`);
|
|
369
565
|
}
|
|
370
|
-
|
|
371
566
|
if (
|
|
372
|
-
execution.status === "
|
|
373
|
-
execution.status === "
|
|
567
|
+
execution.status === "ok" ||
|
|
568
|
+
execution.status === "error" ||
|
|
374
569
|
execution.status === "cancelled"
|
|
375
570
|
) {
|
|
376
571
|
throw new AlephaError(
|
|
@@ -378,11 +573,8 @@ export class JobProvider {
|
|
|
378
573
|
);
|
|
379
574
|
}
|
|
380
575
|
|
|
381
|
-
// If running, trigger the AbortSignal
|
|
382
576
|
const controller = this.abortControllers.get(executionId);
|
|
383
|
-
if (controller)
|
|
384
|
-
controller.abort();
|
|
385
|
-
}
|
|
577
|
+
if (controller) controller.abort();
|
|
386
578
|
|
|
387
579
|
await this.executions.updateById(executionId, {
|
|
388
580
|
status: "cancelled",
|
|
@@ -398,45 +590,27 @@ export class JobProvider {
|
|
|
398
590
|
});
|
|
399
591
|
}
|
|
400
592
|
|
|
401
|
-
// ---
|
|
593
|
+
// --- Queue consumer (called by JobQueueProvider) --------------------------------------------------------------
|
|
402
594
|
|
|
403
|
-
|
|
595
|
+
public async processExecution(
|
|
404
596
|
jobName: string,
|
|
405
597
|
executionId: string,
|
|
406
598
|
): Promise<void> {
|
|
407
|
-
|
|
408
|
-
|
|
599
|
+
const registration = this.jobs.get(jobName);
|
|
600
|
+
if (!registration) {
|
|
601
|
+
this.log.warn(`Unknown job '${jobName}' — skipping execution`, {
|
|
602
|
+
executionId,
|
|
603
|
+
});
|
|
409
604
|
return;
|
|
410
605
|
}
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
jobName: { eq: jobName },
|
|
416
|
-
status: { eq: "running" },
|
|
417
|
-
});
|
|
418
|
-
if (runningCount >= maxConcurrency) {
|
|
419
|
-
this.log.debug(
|
|
420
|
-
`Job '${jobName}' at concurrency limit (${runningCount}/${maxConcurrency}), deferring`,
|
|
421
|
-
{ executionId },
|
|
422
|
-
);
|
|
606
|
+
if (registration.type !== "queue") {
|
|
607
|
+
this.log.warn(`Job '${jobName}' is not queue-mode — skipping`, {
|
|
608
|
+
executionId,
|
|
609
|
+
});
|
|
423
610
|
return;
|
|
424
611
|
}
|
|
425
612
|
|
|
426
|
-
|
|
427
|
-
this.log.debug(`Dispatching job '${jobName}' via queue`, { executionId });
|
|
428
|
-
await this.queueDispatch(jobName, executionId);
|
|
429
|
-
} else {
|
|
430
|
-
this.log.debug(`Executing job '${jobName}' inline`, { executionId });
|
|
431
|
-
await this.processExecution(jobName, executionId);
|
|
432
|
-
}
|
|
433
|
-
}
|
|
434
|
-
|
|
435
|
-
public async processExecution(
|
|
436
|
-
jobName: string,
|
|
437
|
-
executionId: string,
|
|
438
|
-
): Promise<void> {
|
|
439
|
-
const promise = this.processExecutionInner(jobName, executionId);
|
|
613
|
+
const promise = this.processQueueExecution(registration, executionId);
|
|
440
614
|
this.inFlight.add(promise);
|
|
441
615
|
try {
|
|
442
616
|
await promise;
|
|
@@ -445,41 +619,39 @@ export class JobProvider {
|
|
|
445
619
|
}
|
|
446
620
|
}
|
|
447
621
|
|
|
448
|
-
protected async
|
|
449
|
-
|
|
622
|
+
protected async processQueueExecution(
|
|
623
|
+
registration: JobRuntimeRegistration,
|
|
450
624
|
executionId: string,
|
|
451
625
|
): Promise<void> {
|
|
452
|
-
const
|
|
626
|
+
const jobName = registration.name;
|
|
627
|
+
const opts = registration.options;
|
|
628
|
+
const record = opts.record ?? "error";
|
|
453
629
|
|
|
454
|
-
// Claim the execution atomically
|
|
455
630
|
const claimed = await this.claim(executionId);
|
|
456
631
|
if (!claimed) {
|
|
457
632
|
this.log.debug(`Execution ${executionId} already claimed, skipping`);
|
|
458
633
|
return;
|
|
459
634
|
}
|
|
460
635
|
|
|
461
|
-
const
|
|
462
|
-
|
|
636
|
+
const execution = await this.executions.findById(executionId);
|
|
637
|
+
if (!execution) return;
|
|
463
638
|
|
|
464
|
-
this.
|
|
639
|
+
const contextId = this.alepha.context.createContextId();
|
|
640
|
+
this.perExecutionLogs.set(contextId, []);
|
|
641
|
+
|
|
642
|
+
const abortController = new AbortController();
|
|
643
|
+
this.abortControllers.set(executionId, abortController);
|
|
644
|
+
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
|
645
|
+
if (opts.timeout) {
|
|
646
|
+
const ms = this.dt.duration(opts.timeout).as("milliseconds");
|
|
647
|
+
timeoutId = setTimeout(() => abortController.abort(), ms);
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
const now = this.dt.now();
|
|
465
651
|
|
|
466
652
|
try {
|
|
467
653
|
await this.alepha.context.run(
|
|
468
654
|
async () => {
|
|
469
|
-
// Create AbortController for timeout + cancellation
|
|
470
|
-
const abortController = new AbortController();
|
|
471
|
-
this.abortControllers.set(executionId, abortController);
|
|
472
|
-
|
|
473
|
-
// Set up timeout if configured
|
|
474
|
-
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
|
475
|
-
const opts = registration.options;
|
|
476
|
-
if (opts.timeout) {
|
|
477
|
-
const ms = this.dt.duration(opts.timeout).as("milliseconds");
|
|
478
|
-
timeoutId = setTimeout(() => abortController.abort(), ms);
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
const now = this.dt.now();
|
|
482
|
-
|
|
483
655
|
await this.alepha.events.emit("job:begin", {
|
|
484
656
|
name: jobName,
|
|
485
657
|
now,
|
|
@@ -487,41 +659,27 @@ export class JobProvider {
|
|
|
487
659
|
});
|
|
488
660
|
|
|
489
661
|
try {
|
|
490
|
-
// Build items array
|
|
491
|
-
const execution = await this.executions.findById(executionId);
|
|
492
|
-
const items: Array<JobItem> = [];
|
|
493
|
-
if (execution?.payload) {
|
|
494
|
-
items.push({
|
|
495
|
-
id: executionId,
|
|
496
|
-
payload: execution.payload,
|
|
497
|
-
attempt: execution.attempt,
|
|
498
|
-
});
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
// Execute handler
|
|
502
|
-
this.log.debug(`Running job '${jobName}'`, {
|
|
503
|
-
executionId,
|
|
504
|
-
attempt: execution?.attempt,
|
|
505
|
-
items: items.length,
|
|
506
|
-
});
|
|
507
|
-
|
|
508
662
|
await opts.handler({
|
|
509
|
-
|
|
663
|
+
payload: execution.payload,
|
|
664
|
+
attempt: execution.attempt,
|
|
510
665
|
now,
|
|
511
666
|
signal: abortController.signal,
|
|
667
|
+
executionId,
|
|
512
668
|
});
|
|
513
669
|
|
|
514
|
-
// Success
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
670
|
+
// Success: either DELETE (keepLastSuccess=0 or record=error)
|
|
671
|
+
// or UPDATE to 'ok' (record=all and keepLastSuccess>0).
|
|
672
|
+
const keepSuccess =
|
|
673
|
+
record === "all" && this.config.keepLastSuccess > 0;
|
|
674
|
+
if (keepSuccess) {
|
|
675
|
+
await this.executions.updateById(executionId, {
|
|
676
|
+
status: "ok",
|
|
677
|
+
completedAt: this.dt.nowISOString(),
|
|
678
|
+
key: null,
|
|
679
|
+
});
|
|
680
|
+
} else {
|
|
681
|
+
await this.executions.deleteById(executionId);
|
|
682
|
+
}
|
|
525
683
|
|
|
526
684
|
await this.alepha.events.emit(
|
|
527
685
|
"job:success",
|
|
@@ -532,81 +690,45 @@ export class JobProvider {
|
|
|
532
690
|
const err =
|
|
533
691
|
error instanceof Error ? error : new Error(String(error));
|
|
534
692
|
|
|
535
|
-
// Check if this was a cancellation
|
|
536
693
|
if (abortController.signal.aborted) {
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
await this.executions.findById(executionId);
|
|
540
|
-
if (currentExecution?.status !== "cancelled") {
|
|
541
|
-
// Timeout — treat as failure
|
|
542
|
-
await this.handleFailure(executionId, jobName, err, context);
|
|
543
|
-
} else {
|
|
544
|
-
// Was cancelled explicitly — just write logs
|
|
545
|
-
await this.writeLogs(executionId, context);
|
|
694
|
+
const current = await this.executions.findById(executionId);
|
|
695
|
+
if (current?.status === "cancelled") {
|
|
546
696
|
await this.alepha.events.emit(
|
|
547
697
|
"job:cancel",
|
|
548
698
|
{ name: jobName, executionId },
|
|
549
699
|
{ catch: true },
|
|
550
700
|
);
|
|
701
|
+
return;
|
|
551
702
|
}
|
|
552
|
-
} else {
|
|
553
|
-
await this.handleFailure(executionId, jobName, err, context);
|
|
554
703
|
}
|
|
704
|
+
|
|
705
|
+
await this.handleFailure(
|
|
706
|
+
executionId,
|
|
707
|
+
registration,
|
|
708
|
+
execution.attempt,
|
|
709
|
+
err,
|
|
710
|
+
contextId,
|
|
711
|
+
);
|
|
555
712
|
} finally {
|
|
556
713
|
if (timeoutId) clearTimeout(timeoutId);
|
|
557
714
|
this.abortControllers.delete(executionId);
|
|
558
|
-
|
|
559
715
|
await this.alepha.events.emit(
|
|
560
716
|
"job:end",
|
|
561
717
|
{ name: jobName, executionId },
|
|
562
718
|
{ catch: true },
|
|
563
719
|
);
|
|
564
|
-
|
|
565
|
-
// A slot just opened — dispatch next pending job if any
|
|
566
|
-
await this.dispatchNextPending(jobName);
|
|
567
720
|
}
|
|
568
721
|
},
|
|
569
|
-
{ context },
|
|
722
|
+
{ context: contextId },
|
|
570
723
|
);
|
|
571
724
|
} finally {
|
|
572
|
-
this.
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
|
|
576
|
-
/**
|
|
577
|
-
* After a job finishes (success, failure, or cancel), dispatch any pending
|
|
578
|
-
* jobs that were deferred due to the concurrency limit.
|
|
579
|
-
*/
|
|
580
|
-
protected async dispatchNextPending(jobName: string): Promise<void> {
|
|
581
|
-
if (this.stopping || this.pausedJobs.has(jobName)) return;
|
|
582
|
-
|
|
583
|
-
const registration = this.jobs.get(jobName);
|
|
584
|
-
if (!registration) return;
|
|
585
|
-
|
|
586
|
-
const maxConcurrency = registration.options.concurrency ?? 1;
|
|
587
|
-
const runningCount = await this.executions.count({
|
|
588
|
-
jobName: { eq: jobName },
|
|
589
|
-
status: { eq: "running" },
|
|
590
|
-
});
|
|
591
|
-
|
|
592
|
-
const available = maxConcurrency - runningCount;
|
|
593
|
-
if (available <= 0) return;
|
|
594
|
-
|
|
595
|
-
const pending = await this.executions.findMany({
|
|
596
|
-
where: { jobName: { eq: jobName }, status: { eq: "pending" } },
|
|
597
|
-
orderBy: { column: "priority", direction: "asc" },
|
|
598
|
-
limit: available,
|
|
599
|
-
});
|
|
600
|
-
|
|
601
|
-
for (const exec of pending) {
|
|
602
|
-
await this.scheduleProcessing(jobName, exec.id);
|
|
725
|
+
this.perExecutionLogs.delete(contextId);
|
|
603
726
|
}
|
|
604
727
|
}
|
|
605
728
|
|
|
606
729
|
protected async claim(executionId: string): Promise<boolean> {
|
|
607
730
|
const execution = await this.executions.findById(executionId);
|
|
608
731
|
if (!execution) return false;
|
|
609
|
-
|
|
610
732
|
try {
|
|
611
733
|
await this.executions.updateOne(
|
|
612
734
|
{ id: { eq: executionId }, status: { eq: "pending" } },
|
|
@@ -614,7 +736,6 @@ export class JobProvider {
|
|
|
614
736
|
status: "running",
|
|
615
737
|
attempt: execution.attempt + 1,
|
|
616
738
|
startedAt: this.dt.nowISOString(),
|
|
617
|
-
workerId: this.workerId,
|
|
618
739
|
},
|
|
619
740
|
);
|
|
620
741
|
return true;
|
|
@@ -625,64 +746,56 @@ export class JobProvider {
|
|
|
625
746
|
|
|
626
747
|
protected async handleFailure(
|
|
627
748
|
executionId: string,
|
|
628
|
-
|
|
749
|
+
registration: JobRuntimeRegistration,
|
|
750
|
+
currentAttempt: number,
|
|
629
751
|
error: Error,
|
|
630
|
-
|
|
752
|
+
contextId: string,
|
|
631
753
|
): Promise<void> {
|
|
632
|
-
const
|
|
633
|
-
if (!execution) return;
|
|
634
|
-
|
|
635
|
-
const registration = this.getRegistration(jobName);
|
|
754
|
+
const jobName = registration.name;
|
|
636
755
|
const opts = registration.options;
|
|
637
|
-
const
|
|
756
|
+
const retry = opts.retry;
|
|
757
|
+
const maxAttempts = (retry?.retries ?? 0) + 1;
|
|
638
758
|
|
|
639
759
|
const canRetry =
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
(
|
|
760
|
+
retry &&
|
|
761
|
+
currentAttempt + 1 < maxAttempts &&
|
|
762
|
+
(retry.when ? retry.when(error) : true);
|
|
643
763
|
|
|
644
764
|
if (canRetry) {
|
|
645
|
-
|
|
646
|
-
const nextScheduledAt = this.computeBackoff(retryOpts, execution.attempt);
|
|
647
|
-
|
|
765
|
+
const nextScheduledAt = this.computeBackoff(retry, currentAttempt + 1);
|
|
648
766
|
this.log.info(
|
|
649
|
-
`Job '${jobName}' failed, scheduling retry ${
|
|
767
|
+
`Job '${jobName}' failed, scheduling retry ${currentAttempt + 1}/${maxAttempts}`,
|
|
650
768
|
{ executionId, error: error.message, nextScheduledAt },
|
|
651
769
|
);
|
|
652
|
-
|
|
653
770
|
await this.executions.updateById(executionId, {
|
|
654
|
-
status: "
|
|
771
|
+
status: "scheduled",
|
|
655
772
|
error: error.message,
|
|
656
773
|
scheduledAt: nextScheduledAt,
|
|
774
|
+
logs: this.snapshotLogs(contextId),
|
|
657
775
|
});
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
//
|
|
662
|
-
// The delayed dispatch sweep is the safety net in case of crash.
|
|
776
|
+
// Optimistic dispatch: fire a local timer so the retry runs as close to
|
|
777
|
+
// `scheduledAt` as possible. The sweep is the safety net for worker
|
|
778
|
+
// crashes and stateless runtimes (CF Workers, where setTimeout won't
|
|
779
|
+
// survive across invocations anyway).
|
|
663
780
|
const delayMs = Math.max(
|
|
664
781
|
0,
|
|
665
782
|
new Date(nextScheduledAt).getTime() - this.dt.nowMillis(),
|
|
666
783
|
);
|
|
667
|
-
this.dt.createTimeout(
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
);
|
|
784
|
+
this.dt.createTimeout(() => {
|
|
785
|
+
void this.dispatchScheduled(jobName, executionId);
|
|
786
|
+
}, delayMs);
|
|
671
787
|
} else {
|
|
672
|
-
// Dead — all retries exhausted or predicate returned false
|
|
673
788
|
this.log.info(
|
|
674
|
-
`Job '${jobName}'
|
|
789
|
+
`Job '${jobName}' dead after ${currentAttempt} attempt(s)`,
|
|
675
790
|
{ executionId, error: error.message },
|
|
676
791
|
);
|
|
677
|
-
|
|
678
792
|
await this.executions.updateById(executionId, {
|
|
679
|
-
status: "
|
|
793
|
+
status: "error",
|
|
680
794
|
error: error.message,
|
|
681
795
|
completedAt: this.dt.nowISOString(),
|
|
682
796
|
key: null,
|
|
797
|
+
logs: this.snapshotLogs(contextId),
|
|
683
798
|
});
|
|
684
|
-
|
|
685
|
-
await this.writeLogs(executionId, context);
|
|
686
799
|
}
|
|
687
800
|
|
|
688
801
|
await this.alepha.events.emit(
|
|
@@ -692,342 +805,226 @@ export class JobProvider {
|
|
|
692
805
|
);
|
|
693
806
|
}
|
|
694
807
|
|
|
695
|
-
protected computeBackoff(
|
|
696
|
-
retryOpts: JobRetryOptions,
|
|
697
|
-
attempt: number,
|
|
698
|
-
): string {
|
|
808
|
+
protected computeBackoff(retry: JobRetryOptions, attempt: number): string {
|
|
699
809
|
const now = this.dt.now();
|
|
700
|
-
|
|
701
|
-
if (!retryOpts.backoff) {
|
|
702
|
-
// Default: 1 second fixed
|
|
810
|
+
if (!retry.backoff) {
|
|
703
811
|
return now.add(1, "second").toISOString();
|
|
704
812
|
}
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
if (Array.isArray(retryOpts.backoff)) {
|
|
708
|
-
const delay = this.dt.duration(retryOpts.backoff);
|
|
709
|
-
return now.add(delay).toISOString();
|
|
813
|
+
if (Array.isArray(retry.backoff)) {
|
|
814
|
+
return now.add(this.dt.duration(retry.backoff)).toISOString();
|
|
710
815
|
}
|
|
711
|
-
|
|
712
|
-
// Exponential backoff
|
|
713
|
-
const backoff = retryOpts.backoff as JobRetryBackoff;
|
|
816
|
+
const backoff = retry.backoff as JobRetryBackoff;
|
|
714
817
|
const initial = this.dt.duration(backoff.initial).as("milliseconds");
|
|
715
818
|
const factor = backoff.factor ?? 2;
|
|
716
819
|
let delayMs = initial * factor ** (attempt - 1);
|
|
717
|
-
|
|
718
820
|
if (backoff.max) {
|
|
719
|
-
|
|
720
|
-
|
|
821
|
+
delayMs = Math.min(
|
|
822
|
+
delayMs,
|
|
823
|
+
this.dt.duration(backoff.max).as("milliseconds"),
|
|
824
|
+
);
|
|
721
825
|
}
|
|
722
|
-
|
|
723
826
|
if (backoff.jitter) {
|
|
724
|
-
// Add up to 25% random jitter
|
|
725
827
|
delayMs = delayMs * (0.75 + Math.random() * 0.5);
|
|
726
828
|
}
|
|
727
|
-
|
|
728
829
|
return now.add(delayMs, "millisecond").toISOString();
|
|
729
830
|
}
|
|
730
831
|
|
|
731
|
-
protected
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
if (
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
message: `Log entries truncated at ${maxEntries}`,
|
|
747
|
-
timestamp: this.dt.nowMillis(),
|
|
748
|
-
service: "alepha.jobs",
|
|
749
|
-
module: "JobProvider",
|
|
750
|
-
} as LogEntry);
|
|
751
|
-
}
|
|
752
|
-
|
|
753
|
-
try {
|
|
754
|
-
await this.executionLogs.create({
|
|
755
|
-
id: executionId,
|
|
756
|
-
logs,
|
|
757
|
-
});
|
|
758
|
-
} catch {
|
|
759
|
-
// Log write failure is not critical
|
|
760
|
-
this.log.warn(`Failed to write logs for execution ${executionId}`);
|
|
761
|
-
}
|
|
832
|
+
protected snapshotLogs(contextId: string): LogEntry[] | undefined {
|
|
833
|
+
const entries = this.perExecutionLogs.get(contextId);
|
|
834
|
+
if (!entries || entries.length === 0) return undefined;
|
|
835
|
+
const max = this.config.logMaxEntries;
|
|
836
|
+
if (max === 0) return undefined;
|
|
837
|
+
if (entries.length <= max) return [...entries];
|
|
838
|
+
const truncated = entries.slice(0, max);
|
|
839
|
+
truncated.push({
|
|
840
|
+
level: "WARN",
|
|
841
|
+
message: `Log entries truncated at ${max}`,
|
|
842
|
+
timestamp: this.dt.nowMillis(),
|
|
843
|
+
service: "alepha.jobs",
|
|
844
|
+
module: "JobProvider",
|
|
845
|
+
} as LogEntry);
|
|
846
|
+
return truncated;
|
|
762
847
|
}
|
|
763
848
|
|
|
764
|
-
|
|
765
|
-
jobName: string,
|
|
766
|
-
executionId: string,
|
|
767
|
-
): Promise<void> {
|
|
768
|
-
if (this.stopping) return;
|
|
769
|
-
try {
|
|
770
|
-
await this.executions.updateOne(
|
|
771
|
-
{ id: { eq: executionId }, status: { eq: "retrying" } },
|
|
772
|
-
{ status: "pending" },
|
|
773
|
-
);
|
|
774
|
-
await this.scheduleProcessing(jobName, executionId);
|
|
775
|
-
} catch {
|
|
776
|
-
// Already transitioned by another worker or sweep
|
|
777
|
-
}
|
|
778
|
-
}
|
|
779
|
-
|
|
780
|
-
// --- Internal system sweeps (Section 5 of spec) ---
|
|
849
|
+
// --- Sweep ----------------------------------------------------------------------------------------------------
|
|
781
850
|
|
|
782
|
-
|
|
783
|
-
* Recovery Sweep (Section 5.1)
|
|
784
|
-
*
|
|
785
|
-
* Runs every `recovery.interval` (default: 1 minute).
|
|
786
|
-
* - Stale `pending` jobs older than `staleThreshold` → re-dispatch.
|
|
787
|
-
* - Crashed `running` jobs older than `max(job.timeout * 2, recovery.runTimeout)` → mark failed, apply retry policy.
|
|
788
|
-
*/
|
|
789
|
-
protected async recoverySweep(): Promise<void> {
|
|
790
|
-
this.log.trace("Starting recovery sweep");
|
|
851
|
+
protected async sweep(): Promise<void> {
|
|
791
852
|
if (this.stopping) return;
|
|
792
|
-
|
|
793
|
-
const
|
|
794
|
-
|
|
853
|
+
this.log.trace("Starting job sweep");
|
|
854
|
+
const now = this.dt.now();
|
|
855
|
+
const nowIso = now.toISOString();
|
|
795
856
|
|
|
796
857
|
try {
|
|
797
|
-
|
|
858
|
+
// 1. Due scheduled rows → pending + dispatch
|
|
859
|
+
const dueWhere = this.executions.createQueryWhere();
|
|
860
|
+
dueWhere.status = { eq: "scheduled" };
|
|
861
|
+
dueWhere.scheduledAt = { lte: nowIso };
|
|
862
|
+
const due = await this.executions.findMany({
|
|
863
|
+
where: dueWhere,
|
|
864
|
+
orderBy: { column: "priority", direction: "asc" },
|
|
865
|
+
});
|
|
866
|
+
for (const exec of due) {
|
|
867
|
+
if (!this.jobs.has(exec.jobName)) continue;
|
|
868
|
+
await this.executions.updateById(exec.id, { status: "pending" });
|
|
869
|
+
await this.dispatchToQueueSafe(exec.jobName, exec.id);
|
|
870
|
+
}
|
|
798
871
|
|
|
799
|
-
//
|
|
800
|
-
const
|
|
801
|
-
.subtract(this.config.
|
|
872
|
+
// 2. Stale pending rows → re-dispatch
|
|
873
|
+
const staleIso = now
|
|
874
|
+
.subtract(this.config.staleThreshold, "millisecond")
|
|
802
875
|
.toISOString();
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
const stalePending = await this.executions.findMany({
|
|
809
|
-
where: pendingWhere,
|
|
876
|
+
const staleWhere = this.executions.createQueryWhere();
|
|
877
|
+
staleWhere.status = { eq: "pending" };
|
|
878
|
+
staleWhere.createdAt = { lte: staleIso };
|
|
879
|
+
const stale = await this.executions.findMany({
|
|
880
|
+
where: staleWhere,
|
|
810
881
|
orderBy: { column: "priority", direction: "asc" },
|
|
811
882
|
});
|
|
812
|
-
|
|
813
|
-
for (const exec of stalePending) {
|
|
883
|
+
for (const exec of stale) {
|
|
814
884
|
if (!this.jobs.has(exec.jobName)) continue;
|
|
815
|
-
this.
|
|
816
|
-
`Recovery sweep: re-dispatching stale pending job ${exec.jobName} (${exec.id})`,
|
|
817
|
-
);
|
|
818
|
-
await this.scheduleProcessing(exec.jobName, exec.id);
|
|
885
|
+
await this.dispatchToQueueSafe(exec.jobName, exec.id);
|
|
819
886
|
}
|
|
820
887
|
|
|
821
|
-
//
|
|
888
|
+
// 3. Crashed running rows → mark as failed + apply retry
|
|
822
889
|
const runningWhere = this.executions.createQueryWhere();
|
|
823
890
|
runningWhere.status = { eq: "running" };
|
|
824
|
-
|
|
825
891
|
const running = await this.executions.findMany({ where: runningWhere });
|
|
826
892
|
const nowMs = now.valueOf();
|
|
827
|
-
|
|
828
893
|
for (const exec of running) {
|
|
829
|
-
const
|
|
830
|
-
if (!
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
const
|
|
836
|
-
let crashThresholdMs: number;
|
|
837
|
-
if (opts.timeout) {
|
|
838
|
-
crashThresholdMs =
|
|
839
|
-
this.dt.duration(opts.timeout).as("milliseconds") * 2;
|
|
840
|
-
} else {
|
|
841
|
-
crashThresholdMs = this.config.recovery.runTimeout;
|
|
842
|
-
}
|
|
843
|
-
|
|
844
|
-
const startedAt = exec.startedAt
|
|
894
|
+
const reg = this.jobs.get(exec.jobName);
|
|
895
|
+
if (!reg) continue;
|
|
896
|
+
if (this.abortControllers.has(exec.id)) continue; // still alive locally
|
|
897
|
+
const crashThresholdMs = reg.options.timeout
|
|
898
|
+
? this.dt.duration(reg.options.timeout).as("milliseconds") * 2
|
|
899
|
+
: this.config.runTimeout;
|
|
900
|
+
const startedAtMs = exec.startedAt
|
|
845
901
|
? new Date(exec.startedAt).getTime()
|
|
846
902
|
: 0;
|
|
847
|
-
if (
|
|
903
|
+
if (startedAtMs > 0 && nowMs - startedAtMs > crashThresholdMs) {
|
|
848
904
|
this.log.warn(
|
|
849
|
-
`
|
|
905
|
+
`Sweep: marking crashed ${exec.jobName} (${exec.id}) as failed`,
|
|
850
906
|
);
|
|
851
|
-
const
|
|
907
|
+
const err = new Error(
|
|
852
908
|
"Execution assumed crashed (recovered by sweep)",
|
|
853
909
|
);
|
|
854
|
-
await this.handleFailure(exec.id, exec.
|
|
910
|
+
await this.handleFailure(exec.id, reg, exec.attempt, err, "");
|
|
855
911
|
}
|
|
856
912
|
}
|
|
913
|
+
|
|
914
|
+
// 4. Trim ring buffer per job
|
|
915
|
+
await this.trimRingBuffers();
|
|
857
916
|
} catch (e) {
|
|
858
|
-
this.log.error("
|
|
859
|
-
} finally {
|
|
860
|
-
await this.releaseLock("_alepha:jobs:recovery-lock");
|
|
917
|
+
this.log.error("Sweep failed", { error: e });
|
|
861
918
|
}
|
|
862
919
|
}
|
|
863
920
|
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
* Scans for `scheduled` and `retrying` jobs where `scheduledAt <= now`,
|
|
869
|
-
* moves them to `pending`, and dispatches to the queue layer.
|
|
870
|
-
*/
|
|
871
|
-
protected async delayedDispatchSweep(): Promise<void> {
|
|
872
|
-
this.log.trace("Starting delayed dispatch sweep");
|
|
873
|
-
if (this.stopping) return;
|
|
874
|
-
|
|
875
|
-
const acquired = await this.tryLock("_alepha:jobs:dispatch-lock", 60_000);
|
|
876
|
-
if (!acquired) return;
|
|
877
|
-
|
|
921
|
+
protected async dispatchToQueueSafe(
|
|
922
|
+
jobName: string,
|
|
923
|
+
executionId: string,
|
|
924
|
+
): Promise<void> {
|
|
878
925
|
try {
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
const where = this.executions.createQueryWhere();
|
|
882
|
-
where.status = { inArray: ["scheduled", "retrying"] };
|
|
883
|
-
where.scheduledAt = { lte: now };
|
|
884
|
-
|
|
885
|
-
const ready = await this.executions.findMany({
|
|
886
|
-
where,
|
|
887
|
-
orderBy: { column: "priority", direction: "asc" },
|
|
888
|
-
});
|
|
889
|
-
|
|
890
|
-
for (const exec of ready) {
|
|
891
|
-
if (!this.jobs.has(exec.jobName)) continue;
|
|
892
|
-
await this.executions.updateById(exec.id, { status: "pending" });
|
|
893
|
-
await this.scheduleProcessing(exec.jobName, exec.id);
|
|
894
|
-
}
|
|
926
|
+
await this.dispatchToQueue(jobName, executionId);
|
|
895
927
|
} catch (e) {
|
|
896
|
-
this.log.
|
|
897
|
-
} finally {
|
|
898
|
-
await this.releaseLock("_alepha:jobs:dispatch-lock");
|
|
928
|
+
this.log.warn(`Sweep failed to dispatch ${jobName} (${executionId})`, e);
|
|
899
929
|
}
|
|
900
930
|
}
|
|
901
931
|
|
|
902
932
|
/**
|
|
903
|
-
*
|
|
904
|
-
*
|
|
905
|
-
*
|
|
906
|
-
* Deletes completed/dead/cancelled execution records older than `logRetentionDays`.
|
|
933
|
+
* Move a row from `scheduled` → `pending` and dispatch it.
|
|
934
|
+
* Used by the optimistic retry/delay timer. If the sweep has already moved
|
|
935
|
+
* the row, or another worker has claimed it, the UPDATE guard fails silently.
|
|
907
936
|
*/
|
|
908
|
-
protected async
|
|
937
|
+
protected async dispatchScheduled(
|
|
938
|
+
jobName: string,
|
|
939
|
+
executionId: string,
|
|
940
|
+
): Promise<void> {
|
|
909
941
|
if (this.stopping) return;
|
|
910
942
|
try {
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
where.completedAt = { lte: cutoff };
|
|
919
|
-
|
|
920
|
-
// Bulk-delete logs first (FK-safe), then executions
|
|
921
|
-
const expiredIds = await this.executions.findMany({
|
|
922
|
-
where,
|
|
923
|
-
columns: ["id"] as any,
|
|
924
|
-
});
|
|
925
|
-
if (expiredIds.length > 0) {
|
|
926
|
-
const ids = expiredIds.map((e) => e.id);
|
|
927
|
-
await this.executionLogs.deleteMany({ id: { inArray: ids } });
|
|
928
|
-
await this.executions.deleteMany({ id: { inArray: ids } });
|
|
929
|
-
this.log.info(`Log purge: deleted ${ids.length} old execution records`);
|
|
930
|
-
}
|
|
931
|
-
} catch (e) {
|
|
932
|
-
this.log.error("Log purge failed", { error: e });
|
|
943
|
+
await this.executions.updateOne(
|
|
944
|
+
{ id: { eq: executionId }, status: { eq: "scheduled" } },
|
|
945
|
+
{ status: "pending" },
|
|
946
|
+
);
|
|
947
|
+
await this.dispatchToQueueSafe(jobName, executionId);
|
|
948
|
+
} catch {
|
|
949
|
+
// Row already transitioned (sweep ran, another worker claimed, etc.)
|
|
933
950
|
}
|
|
934
951
|
}
|
|
935
952
|
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
this.pausedJobs.delete(name);
|
|
947
|
-
this.log.info(`Resumed job '${name}'`);
|
|
948
|
-
|
|
949
|
-
// Dispatch any pending items for this job
|
|
950
|
-
const pending = await this.executions.findMany({
|
|
951
|
-
where: { jobName: { eq: name }, status: { eq: "pending" } },
|
|
952
|
-
orderBy: { column: "priority", direction: "asc" },
|
|
953
|
-
});
|
|
954
|
-
for (const exec of pending) {
|
|
955
|
-
await this.scheduleProcessing(name, exec.id);
|
|
953
|
+
protected async trimRingBuffers(): Promise<void> {
|
|
954
|
+
for (const [jobName, reg] of this.jobs) {
|
|
955
|
+
const okLimit = reg.options.keep?.ok ?? this.config.keepLastSuccess;
|
|
956
|
+
const errLimit = reg.options.keep?.error ?? this.config.keepLastError;
|
|
957
|
+
if (okLimit > 0) {
|
|
958
|
+
await this.trimByStatus(jobName, "ok", okLimit);
|
|
959
|
+
}
|
|
960
|
+
if (errLimit > 0) {
|
|
961
|
+
await this.trimByStatus(jobName, "error", errLimit);
|
|
962
|
+
}
|
|
956
963
|
}
|
|
957
964
|
}
|
|
958
965
|
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
966
|
+
protected async trimByStatus(
|
|
967
|
+
jobName: string,
|
|
968
|
+
status: "ok" | "error",
|
|
969
|
+
keep: number,
|
|
970
|
+
): Promise<void> {
|
|
971
|
+
try {
|
|
972
|
+
const rows = await this.executions.findMany({
|
|
973
|
+
where: { jobName: { eq: jobName }, status: { eq: status } },
|
|
974
|
+
orderBy: { column: "startedAt", direction: "desc" },
|
|
975
|
+
limit: keep + 50,
|
|
976
|
+
});
|
|
977
|
+
if (rows.length <= keep) return;
|
|
978
|
+
const toDelete = rows.slice(keep).map((r) => r.id);
|
|
979
|
+
if (toDelete.length > 0) {
|
|
980
|
+
await this.executions.deleteMany({ id: { inArray: toDelete } });
|
|
981
|
+
this.log.debug(
|
|
982
|
+
`Trimmed ${toDelete.length} ${status} rows for '${jobName}'`,
|
|
983
|
+
);
|
|
984
|
+
}
|
|
985
|
+
} catch (e) {
|
|
986
|
+
this.log.warn(`Failed to trim ${status} rows for '${jobName}'`, e);
|
|
987
|
+
}
|
|
978
988
|
}
|
|
979
989
|
|
|
980
|
-
// --- Lifecycle
|
|
990
|
+
// --- Lifecycle -----------------------------------------------------------------------------------------------
|
|
981
991
|
|
|
982
992
|
protected readonly onStart = $hook({
|
|
983
993
|
on: "start",
|
|
984
994
|
handler: async () => {
|
|
985
|
-
|
|
995
|
+
// Validate that queue-mode jobs have a dispatcher registered.
|
|
996
|
+
const needsQueue = [...this.jobs.values()].some(
|
|
997
|
+
(j) => j.type === "queue",
|
|
998
|
+
);
|
|
999
|
+
if (needsQueue && !this.queueDispatch) {
|
|
1000
|
+
throw new AlephaError(
|
|
1001
|
+
`Queue-mode jobs are registered but no queue dispatcher is available. Add '.with(AlephaApiJobsQueue)' to your app.`,
|
|
1002
|
+
);
|
|
1003
|
+
}
|
|
1004
|
+
|
|
986
1005
|
this.log.info(`Job system OK`, {
|
|
987
|
-
|
|
988
|
-
|
|
1006
|
+
dispatch: this.queueDispatch ? "queue" : "inline-only",
|
|
1007
|
+
jobs: this.jobs.size,
|
|
989
1008
|
});
|
|
990
1009
|
|
|
991
|
-
//
|
|
1010
|
+
// Capture logs per execution context.
|
|
992
1011
|
this.alepha.events.on("log", ({ entry }) => {
|
|
993
1012
|
const ctx = entry.context;
|
|
994
1013
|
if (!ctx) return;
|
|
995
|
-
const entries = this.
|
|
1014
|
+
const entries = this.perExecutionLogs.get(ctx);
|
|
996
1015
|
if (!entries) return;
|
|
997
1016
|
entries.push(entry);
|
|
998
1017
|
});
|
|
999
1018
|
|
|
1000
|
-
// Run initial sweeps to recover from previous crashes.
|
|
1001
|
-
// Skipped on serverless — cron triggers handle periodic sweeps instead.
|
|
1002
1019
|
if (!this.alepha.isServerless()) {
|
|
1003
|
-
await this.
|
|
1004
|
-
await this.recoverySweep();
|
|
1020
|
+
await this.sweep();
|
|
1005
1021
|
}
|
|
1006
1022
|
|
|
1007
|
-
// Periodic sweeps via cron (works in serverless environments like Cloudflare Workers)
|
|
1008
1023
|
this.cronProvider.createCronJob(
|
|
1009
|
-
"
|
|
1010
|
-
|
|
1024
|
+
"api:jobs:sweep",
|
|
1025
|
+
SWEEP_CRON,
|
|
1011
1026
|
async () => {
|
|
1012
|
-
await this.
|
|
1013
|
-
},
|
|
1014
|
-
true,
|
|
1015
|
-
);
|
|
1016
|
-
this.cronProvider.createCronJob(
|
|
1017
|
-
"_alepha:jobs:dispatch",
|
|
1018
|
-
JobProvider.SWEEP_CRON,
|
|
1019
|
-
async () => {
|
|
1020
|
-
await this.delayedDispatchSweep();
|
|
1021
|
-
},
|
|
1022
|
-
true,
|
|
1023
|
-
);
|
|
1024
|
-
|
|
1025
|
-
// Daily log purge
|
|
1026
|
-
this.cronProvider.createCronJob(
|
|
1027
|
-
"_alepha:jobs:log-purge",
|
|
1028
|
-
"0 0 * * *",
|
|
1029
|
-
async () => {
|
|
1030
|
-
await this.logPurge();
|
|
1027
|
+
await this.sweep();
|
|
1031
1028
|
},
|
|
1032
1029
|
true,
|
|
1033
1030
|
);
|
|
@@ -1038,8 +1035,6 @@ export class JobProvider {
|
|
|
1038
1035
|
on: "stop",
|
|
1039
1036
|
handler: async () => {
|
|
1040
1037
|
this.stopping = true;
|
|
1041
|
-
|
|
1042
|
-
// Drain: wait for in-flight jobs to finish before aborting
|
|
1043
1038
|
if (this.inFlight.size > 0) {
|
|
1044
1039
|
this.log.info(`Draining ${this.inFlight.size} in-flight job(s)...`);
|
|
1045
1040
|
await Promise.race([
|
|
@@ -1047,8 +1042,6 @@ export class JobProvider {
|
|
|
1047
1042
|
this.dt.wait([this.config.drainTimeout, "millisecond"]),
|
|
1048
1043
|
]);
|
|
1049
1044
|
}
|
|
1050
|
-
|
|
1051
|
-
// Abort any still-running executions after drain timeout
|
|
1052
1045
|
if (this.abortControllers.size > 0) {
|
|
1053
1046
|
this.log.warn(
|
|
1054
1047
|
`Aborting ${this.abortControllers.size} remaining job(s) after drain timeout`,
|
|
@@ -1060,9 +1053,9 @@ export class JobProvider {
|
|
|
1060
1053
|
},
|
|
1061
1054
|
});
|
|
1062
1055
|
|
|
1063
|
-
// --- Helpers
|
|
1056
|
+
// --- Helpers -------------------------------------------------------------------------------------------------
|
|
1064
1057
|
|
|
1065
|
-
protected getRegistration(name: string):
|
|
1058
|
+
protected getRegistration(name: string): JobRuntimeRegistration {
|
|
1066
1059
|
const registration = this.jobs.get(name);
|
|
1067
1060
|
if (!registration) {
|
|
1068
1061
|
throw new AlephaError(`Job not registered: ${name}`);
|
|
@@ -1070,3 +1063,5 @@ export class JobProvider {
|
|
|
1070
1063
|
return registration;
|
|
1071
1064
|
}
|
|
1072
1065
|
}
|
|
1066
|
+
|
|
1067
|
+
export { PRIORITY_MAP, PRIORITY_REVERSE };
|