@fabrika/app 0.9.0-dev.1 → 0.9.0-dev.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +32 -26
- package/skill/SKILL.md +46 -15
- package/skill/references/app-setup.md +43 -12
- package/skill/references/dev.md +5 -2
- package/skill/references/endpoint.md +39 -28
- package/skill/references/errors.md +50 -37
- package/skill/references/files.md +8 -4
- package/skill/references/jobs.md +39 -7
- package/skill/references/table-and-migration.md +47 -3
- package/skill/references/tests.md +73 -16
- package/skill/references/webhooks-links-connectors.md +1 -1
- package/src/build-recipe.ts +198 -0
- package/src/build.ts +60 -4
- package/src/bun.ts +181 -13
- package/src/cloudflare/application.ts +822 -0
- package/src/cloudflare/bindings.ts +159 -0
- package/src/cloudflare/export.ts +621 -0
- package/src/cloudflare/import.ts +888 -0
- package/src/cloudflare/maintenance.ts +518 -0
- package/src/cloudflare/names.ts +16 -0
- package/src/cloudflare/ops.ts +375 -0
- package/src/cloudflare/r2-files.ts +172 -0
- package/src/cloudflare/restore.ts +167 -0
- package/src/cloudflare/tracker.ts +192 -0
- package/src/cloudflare.ts +100 -146
- package/src/contract/app-id.ts +5 -0
- package/src/contract/artifact-schema.ts +9 -1
- package/src/contract/build.ts +40 -2
- package/src/contract/capability-cases.ts +30 -0
- package/src/contract/compile.ts +1 -0
- package/src/contract/containers.ts +19 -0
- package/src/contract/context.ts +35 -9
- package/src/contract/declaration.ts +2 -0
- package/src/contract/defaults.ts +27 -4
- package/src/contract/define.ts +29 -3
- package/src/contract/index.ts +1 -0
- package/src/contract/kit.ts +4 -1
- package/src/contract/platform.ts +3 -2
- package/src/contract/vocabulary.ts +26 -8
- package/src/dev/changes.ts +5 -43
- package/src/dev/contract-digest.ts +17 -0
- package/src/dev/index.ts +1 -1
- package/src/dev/launcher.ts +197 -41
- package/src/dev/output.ts +13 -3
- package/src/dev/wire.ts +11 -1
- package/src/disk-assets.ts +8 -3
- package/src/files/archive.ts +2 -2
- package/src/files/handle.ts +2 -1
- package/src/files/job-names.ts +12 -0
- package/src/files/kinds.ts +74 -32
- package/src/files/scan.ts +3 -4
- package/src/files/streams.ts +3 -1
- package/src/implement/anonymous.ts +12 -11
- package/src/implement/boot.ts +84 -27
- package/src/implement/bounded.ts +21 -2
- package/src/implement/capability.ts +16 -4
- package/src/implement/dispatch.ts +423 -178
- package/src/implement/failures.ts +59 -4
- package/src/implement/faults.ts +5 -1
- package/src/implement/index.ts +1 -0
- package/src/implement/invocation.ts +62 -11
- package/src/implement/principal.ts +25 -2
- package/src/implement/terminals.ts +93 -0
- package/src/implement/tokens.ts +4 -2
- package/src/implement/types.ts +22 -4
- package/src/implement/workload.ts +42 -16
- package/src/jobs/mail.ts +8 -0
- package/src/jobs/report.ts +5 -1
- package/src/jobs/runner.ts +702 -126
- package/src/jobs/schedule.ts +7 -1
- package/src/jobs/scrub.ts +14 -1
- package/src/jobs/types.ts +6 -0
- package/src/spa-fallback.ts +12 -0
- package/src/testing/endpoint-access.ts +493 -202
- package/src/testing/index.ts +1 -1
- package/src/testing/isolation.ts +8 -1
- package/src/testing/kit.ts +86 -4
- package/src/testing/policy-isolation.ts +2 -2
- package/src/testing/proxy.ts +15 -10
- package/types/src/agent-skill.d.ts +29 -0
- package/types/src/build-recipe.d.ts +70 -0
- package/types/src/build.d.ts +29 -0
- package/types/src/bun.d.ts +99 -0
- package/types/src/client/caller.d.ts +19 -0
- package/types/src/client/contract.d.ts +131 -0
- package/types/src/client/error-reporting.d.ts +66 -0
- package/types/src/client/index.d.ts +4 -0
- package/types/src/client/rpc.d.ts +14 -0
- package/types/src/client/share.d.ts +2 -0
- package/types/src/client/transport.d.ts +42 -0
- package/types/src/cloudflare/application.d.ts +73 -0
- package/types/src/cloudflare/bindings.d.ts +33 -0
- package/types/src/cloudflare/export.d.ts +16 -0
- package/types/src/cloudflare/import.d.ts +47 -0
- package/types/src/cloudflare/maintenance.d.ts +117 -0
- package/types/src/cloudflare/names.d.ts +11 -0
- package/types/src/cloudflare/ops.d.ts +81 -0
- package/types/src/cloudflare/r2-files.d.ts +9 -0
- package/types/src/cloudflare/restore.d.ts +9 -0
- package/types/src/cloudflare/tracker.d.ts +29 -0
- package/types/src/cloudflare.d.ts +43 -0
- package/types/src/contract/app-id.d.ts +2 -0
- package/types/src/contract/artifact-schema.d.ts +10 -0
- package/types/src/contract/attachment.d.ts +41 -0
- package/types/src/contract/build.d.ts +12 -0
- package/types/src/contract/capability-cases.d.ts +9 -0
- package/types/src/contract/compile.d.ts +11 -0
- package/types/src/contract/connect.d.ts +51 -0
- package/types/src/contract/containers.d.ts +3 -0
- package/types/src/contract/context.d.ts +525 -0
- package/types/src/contract/declaration.d.ts +307 -0
- package/types/src/contract/defaults.d.ts +228 -0
- package/types/src/contract/define.d.ts +376 -0
- package/types/src/contract/file.d.ts +44 -0
- package/types/src/contract/group.d.ts +274 -0
- package/types/src/contract/handled.d.ts +18 -0
- package/types/src/contract/index.d.ts +20 -0
- package/types/src/contract/kit.d.ts +211 -0
- package/types/src/contract/platform.d.ts +9 -0
- package/types/src/contract/problems.d.ts +14 -0
- package/types/src/contract/redirect.d.ts +25 -0
- package/types/src/contract/schema-from-shape.d.ts +10 -0
- package/types/src/contract/schema-shape.d.ts +5 -0
- package/types/src/contract/snapshot-source.d.ts +15 -0
- package/types/src/contract/token.d.ts +58 -0
- package/types/src/contract/vocabulary.d.ts +32 -0
- package/types/src/contract/webhook-binding.d.ts +24 -0
- package/types/src/contract/webhook-scheme.d.ts +23 -0
- package/types/src/dev/changes.d.ts +7 -0
- package/types/src/dev/contract-digest.d.ts +1 -0
- package/types/src/dev/index.d.ts +2 -0
- package/types/src/dev/launcher.d.ts +32 -0
- package/types/src/dev/output.d.ts +28 -0
- package/types/src/dev/wire.d.ts +126 -0
- package/types/src/disk-assets.d.ts +6 -0
- package/types/src/errors.d.ts +46 -0
- package/types/src/fetch.d.ts +7 -0
- package/types/src/files/archive.d.ts +26 -0
- package/types/src/files/clamscan.d.ts +32 -0
- package/types/src/files/handle.d.ts +107 -0
- package/types/src/files/job-names.d.ts +9 -0
- package/types/src/files/kinds.d.ts +112 -0
- package/types/src/files/scan.d.ts +78 -0
- package/types/src/files/sniff.d.ts +7 -0
- package/types/src/files/streams.d.ts +17 -0
- package/types/src/files/treatment.d.ts +18 -0
- package/types/src/from-row.d.ts +20 -0
- package/types/src/honeypot.d.ts +2 -0
- package/types/src/implement/anonymous.d.ts +24 -0
- package/types/src/implement/attachment.d.ts +14 -0
- package/types/src/implement/boot.d.ts +128 -0
- package/types/src/implement/bounded.d.ts +32 -0
- package/types/src/implement/browser-config.d.ts +19 -0
- package/types/src/implement/capability.d.ts +142 -0
- package/types/src/implement/challenge.d.ts +75 -0
- package/types/src/implement/client-address.d.ts +11 -0
- package/types/src/implement/connect.d.ts +67 -0
- package/types/src/implement/crashes.d.ts +31 -0
- package/types/src/implement/dispatch.d.ts +251 -0
- package/types/src/implement/egress.d.ts +14 -0
- package/types/src/implement/environment.d.ts +14 -0
- package/types/src/implement/failures.d.ts +58 -0
- package/types/src/implement/faults.d.ts +48 -0
- package/types/src/implement/headers.d.ts +7 -0
- package/types/src/implement/implement.d.ts +58 -0
- package/types/src/implement/index.d.ts +25 -0
- package/types/src/implement/invocation.d.ts +136 -0
- package/types/src/implement/membership.d.ts +201 -0
- package/types/src/implement/multipart.d.ts +36 -0
- package/types/src/implement/params.d.ts +7 -0
- package/types/src/implement/principal.d.ts +57 -0
- package/types/src/implement/project.d.ts +11 -0
- package/types/src/implement/public.d.ts +61 -0
- package/types/src/implement/serve.d.ts +25 -0
- package/types/src/implement/terminals.d.ts +33 -0
- package/types/src/implement/tokens.d.ts +49 -0
- package/types/src/implement/types.d.ts +263 -0
- package/types/src/implement/urls.d.ts +15 -0
- package/types/src/implement/verifiers.d.ts +19 -0
- package/types/src/implement/webhook.d.ts +55 -0
- package/types/src/implement/workload.d.ts +61 -0
- package/types/src/implement/xlsx.d.ts +6 -0
- package/types/src/index.d.ts +12 -0
- package/types/src/jobs/exception.d.ts +40 -0
- package/types/src/jobs/mail.d.ts +67 -0
- package/types/src/jobs/report.d.ts +56 -0
- package/types/src/jobs/runner.d.ts +218 -0
- package/types/src/jobs/schedule.d.ts +17 -0
- package/types/src/jobs/scrub.d.ts +12 -0
- package/types/src/jobs/types.d.ts +98 -0
- package/types/src/react/caller.d.ts +30 -0
- package/types/src/react/errors.d.ts +40 -0
- package/types/src/react/hooks.d.ts +110 -0
- package/types/src/react/index.d.ts +5 -0
- package/types/src/react/provider.d.ts +58 -0
- package/types/src/react/store.d.ts +68 -0
- package/types/src/react/testing.d.ts +28 -0
- package/types/src/react/types.d.ts +33 -0
- package/types/src/rpc/builder.d.ts +52 -0
- package/types/src/rpc/contract.d.ts +23 -0
- package/types/src/rpc/dispatcher.d.ts +11 -0
- package/types/src/rpc/types.d.ts +62 -0
- package/types/src/service/define.d.ts +42 -0
- package/types/src/service/index.d.ts +3 -0
- package/types/src/service/serve.d.ts +28 -0
- package/types/src/spa-fallback.d.ts +2 -0
- package/types/src/standard-schema.d.ts +55 -0
- package/types/src/testing/caller.d.ts +95 -0
- package/types/src/testing/challenge.d.ts +11 -0
- package/types/src/testing/dev.d.ts +97 -0
- package/types/src/testing/endpoint-access.d.ts +132 -0
- package/types/src/testing/index.d.ts +12 -0
- package/types/src/testing/isolation.d.ts +3 -0
- package/types/src/testing/kit.d.ts +207 -0
- package/types/src/testing/membership.d.ts +132 -0
- package/types/src/testing/policy-isolation.d.ts +113 -0
- package/types/src/testing/preload.d.ts +1 -0
- package/types/src/testing/proxy.d.ts +40 -0
- package/types/src/testing/runner.d.ts +8 -0
- package/types/src/testing/sample-file.d.ts +5 -0
- package/types/src/testing/status.d.ts +4 -0
- package/types/src/undumpable.d.ts +5 -0
package/src/jobs/runner.ts
CHANGED
|
@@ -3,13 +3,25 @@
|
|
|
3
3
|
// that timed out, failed or lost the lease never commits (`@fabrika/db`'s fence). A site's `ctx.kick` sends a run in
|
|
4
4
|
// the background after its commit, without waiting for any lock; a kick that finds the lease held leaves the shared pending flag, and the run
|
|
5
5
|
// holding it runs once more after it succeeded. Failures, crashed runs and a job that stopped succeeding are reported
|
|
6
|
-
// to Operations, each under a fixed failure code.
|
|
6
|
+
// to Operations, each under a fixed failure code. On Cloudflare the object's alarm drives the jobs through `scheduler`:
|
|
7
|
+
// a run it starts owns the job's due wakes, and when it fails they move to a durable retry instead of an inline one.
|
|
7
8
|
|
|
8
9
|
import type { IamMembership } from '@fabrika/auth'
|
|
9
10
|
import { type ContractArtifact, JOB_SITE_PREFIX, type JobArtifact, type JobMailer, type PermissionEntry, permits } from '@fabrika/auth-core'
|
|
10
11
|
import type { CapabilityAuthority } from '@fabrika/db/capability'
|
|
11
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
type DataAccess,
|
|
14
|
+
type FinishWakes,
|
|
15
|
+
type FrameworkTables,
|
|
16
|
+
type JobLeases,
|
|
17
|
+
type JobWakes,
|
|
18
|
+
type LeaseFence,
|
|
19
|
+
LeaseLostError,
|
|
20
|
+
type Session,
|
|
21
|
+
type Wake,
|
|
22
|
+
} from '@fabrika/db/framework'
|
|
12
23
|
import { EmailDeliveryError, type EmailSenderCapabilities, mayRetryDelivery } from '@fabrika/email'
|
|
24
|
+
import { uuidv7 } from '@fabrika/platform'
|
|
13
25
|
import { NotFoundError } from '../errors'
|
|
14
26
|
import { FileWork, type StoredFileKinds, storedFilesHandle } from '../files/kinds'
|
|
15
27
|
import {
|
|
@@ -21,8 +33,9 @@ import {
|
|
|
21
33
|
runInRequest,
|
|
22
34
|
type Within,
|
|
23
35
|
} from '../implement/capability'
|
|
36
|
+
import { admitInvocation, type InvocationAdmission } from '../implement/dispatch'
|
|
24
37
|
import { runAsSite } from '../implement/egress'
|
|
25
|
-
import { failureCode, type FailureReason, failureReason } from '../implement/failures'
|
|
38
|
+
import { failureCode, type FailureReason, failureReason, type HandlerErrorSink, reveal } from '../implement/failures'
|
|
26
39
|
import { ServerFault } from '../implement/faults'
|
|
27
40
|
import type { ContractDatabase } from '../implement/implement'
|
|
28
41
|
import {
|
|
@@ -33,6 +46,7 @@ import {
|
|
|
33
46
|
jobHandles,
|
|
34
47
|
jobSql,
|
|
35
48
|
jobSqlOutsideTransaction,
|
|
49
|
+
type KickRecorder,
|
|
36
50
|
kickRecorder,
|
|
37
51
|
siteData,
|
|
38
52
|
type SiteDeclaration,
|
|
@@ -41,10 +55,10 @@ import {
|
|
|
41
55
|
import { jobLinkIssuing, type JobLinkMount } from '../implement/membership'
|
|
42
56
|
import type { SignedTokens } from '../implement/tokens'
|
|
43
57
|
import { type AppUrl, appUrl } from '../implement/urls'
|
|
44
|
-
import { type WorkloadCredential, workloadSite } from '../implement/workload'
|
|
58
|
+
import { type WorkloadAuthority, type WorkloadCredential, workloadSite } from '../implement/workload'
|
|
45
59
|
import { jobMail, type MailTransports } from './mail'
|
|
46
60
|
import type { OperationsReporter } from './report'
|
|
47
|
-
import { type CronSchedule, parseCron, periodAt } from './schedule'
|
|
61
|
+
import { type CronSchedule, isDueAt, parseCron, periodAt } from './schedule'
|
|
48
62
|
import type { AppMail, JobContext, JobPrincipal } from './types'
|
|
49
63
|
|
|
50
64
|
/**
|
|
@@ -54,6 +68,11 @@ import type { AppMail, JobContext, JobPrincipal } from './types'
|
|
|
54
68
|
export const LEASE_GRACE_MS = 60_000
|
|
55
69
|
/** How often the serving process checks every job for a crashed run and for a schedule that stopped. */
|
|
56
70
|
export const JOB_CHECK_INTERVAL_MS = 60_000
|
|
71
|
+
/**
|
|
72
|
+
* A run the Cloudflare alarm started has one segment (`CLOUDFLARE_MAX_JOB_SEGMENT_MS`): its attempt ends this much
|
|
73
|
+
* before the segment, at most a tenth of it, so the finish runs inside the segment too.
|
|
74
|
+
*/
|
|
75
|
+
export const SEGMENT_FINISH_RESERVE_MS = 30_000
|
|
57
76
|
const BACKOFF_BASE_MS = 1_000
|
|
58
77
|
const BACKOFF_MAX_MS = 5 * 60_000
|
|
59
78
|
|
|
@@ -69,20 +88,30 @@ export type JobFailureCode =
|
|
|
69
88
|
| 'query_refused'
|
|
70
89
|
/** A capability or a handle was refused: forged, ended, of other work, or of another kind than the call takes. */
|
|
71
90
|
| 'capability_refused'
|
|
91
|
+
/** The framework failed the attempt (`ServerFault`): its reason names why, as a request's 500 names it. */
|
|
92
|
+
| 'framework_fault'
|
|
93
|
+
/**
|
|
94
|
+
* The framework could not reach a service it needs (a `ServerFault` with status 503), as a request's 503 names it. IAM
|
|
95
|
+
* is the exception: met inside the handler as at the attempt's start, it is `iam_unavailable` or `unauthorized`.
|
|
96
|
+
*/
|
|
97
|
+
| 'unavailable'
|
|
72
98
|
| 'driver_failed'
|
|
73
99
|
| 'crashed'
|
|
74
100
|
/** IAM refused the job's key: the attempt mints no capability. */
|
|
75
101
|
| 'unauthorized'
|
|
76
102
|
/** IAM could not be asked for the job's token. */
|
|
77
103
|
| 'iam_unavailable'
|
|
78
|
-
/** The process stopped a kicked run while shutting down; the schedule runs
|
|
104
|
+
/** The process stopped a kicked run while shutting down, or the object's maintenance stopped a run; the schedule runs it again. */
|
|
79
105
|
| 'stopped'
|
|
80
106
|
/** The transport refused a message the handler did not catch; `reason` is the `EmailDeliveryError` code. */
|
|
81
107
|
| 'mail_failed'
|
|
82
108
|
|
|
83
109
|
export type JobRunResult =
|
|
84
110
|
| { readonly status: 'succeeded'; readonly attempts: number }
|
|
85
|
-
/**
|
|
111
|
+
/**
|
|
112
|
+
* `reason` is a framework failure's catalog key (`query_refused`, `capability_refused`, `framework_fault`,
|
|
113
|
+
* `unavailable`) or the `EmailDeliveryError` code (`mail_failed`), present only for one of those.
|
|
114
|
+
*/
|
|
86
115
|
| { readonly status: 'failed'; readonly attempts: number; readonly code: JobFailureCode; readonly reason?: string }
|
|
87
116
|
/** Another run holds the lease: an overlapping run does not start. */
|
|
88
117
|
| { readonly status: 'skipped' }
|
|
@@ -93,6 +122,8 @@ export interface JobClock {
|
|
|
93
122
|
wait(ms: number, signal: AbortSignal): Promise<void>
|
|
94
123
|
/** In [0, 1): the jitter of a retry's backoff. */
|
|
95
124
|
random(): number
|
|
125
|
+
/** Epoch milliseconds: what an attempt's deadline is measured on; `Date.now()` when absent. */
|
|
126
|
+
now?(): number
|
|
96
127
|
}
|
|
97
128
|
|
|
98
129
|
export interface JobRunOptions {
|
|
@@ -117,12 +148,67 @@ export interface FrameworkJob {
|
|
|
117
148
|
export interface ContractJobs {
|
|
118
149
|
/** The contract's `c.cron` jobs; `run` also takes the name of a framework job the contract needs. */
|
|
119
150
|
readonly names: readonly string[]
|
|
151
|
+
/** Every job bound to run, the framework's included, with its schedule: what a minute tick runs when due. */
|
|
152
|
+
readonly scheduled: readonly { readonly name: string; readonly schedule: CronSchedule }[]
|
|
120
153
|
run(name: string, options?: JobRunOptions): Promise<JobRunResult>
|
|
121
154
|
/**
|
|
122
155
|
* Registers every job (its first-seen time), reports a run that crashed and a job that has not succeeded within
|
|
123
156
|
* twice its period. The serving process calls it every `JOB_CHECK_INTERVAL_MS`; it never throws.
|
|
124
157
|
*/
|
|
125
158
|
check(options?: Pick<JobRunOptions, 'reporter'>): Promise<void>
|
|
159
|
+
/**
|
|
160
|
+
* The serving process's wake timer on Bun, where crontab runs the schedule: reaps a crashed holder of every job and
|
|
161
|
+
* kicks each contract job with a due wake through the kick path, handing each kick to `track` until it ended. Answers
|
|
162
|
+
* how long until a wake is due next on the database's clock (0 while one still is), `null` when none is pending. The
|
|
163
|
+
* first call deletes the wakes of jobs the contract no longer declares.
|
|
164
|
+
*/
|
|
165
|
+
wake(track: (kick: Promise<void>) => void): Promise<number | null>
|
|
166
|
+
/** What the Cloudflare object's alarm drives the jobs through; `null` without `@fabrika/db`. */
|
|
167
|
+
readonly scheduler: JobScheduler | null
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** What the scheduler's `take` did for one job in the alarm's transaction. */
|
|
171
|
+
export interface TakenLease {
|
|
172
|
+
/** The lease a run of the job now holds, for `run`; `null` when nothing was due or another run holds it. */
|
|
173
|
+
readonly fence: LeaseFence | null
|
|
174
|
+
/** Whether a crashed holder's lease was freed: `crashed` reports it once the transaction committed. */
|
|
175
|
+
readonly reaped: boolean
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* The jobs as the Cloudflare object's alarm drives them (ADR-0065 §5): over the wakes and leases of the object's
|
|
180
|
+
* storage, every time on the database's clock. The object holds the transactions and arms the alarm.
|
|
181
|
+
*/
|
|
182
|
+
export interface JobScheduler {
|
|
183
|
+
/** Every job bound to run, the framework's included. */
|
|
184
|
+
readonly jobs: readonly string[]
|
|
185
|
+
/**
|
|
186
|
+
* Startup repair: reaps and reports every crashed run, of a job the contract no longer declares too; registers every
|
|
187
|
+
* job and surveils it (as `check`), deletes the wakes of jobs the contract no longer declares and the cron wakes of a
|
|
188
|
+
* changed expression, and keeps one cron wake per job at its next fire; an overdue one stays, so missed runs coalesce
|
|
189
|
+
* into one. `false` when the object's barrier admits no job.
|
|
190
|
+
*/
|
|
191
|
+
repair(options?: Pick<JobRunOptions, 'reporter'>): Promise<boolean>
|
|
192
|
+
/** The jobs the contract no longer declares whose lease a crashed run still holds: `take` reaps them. */
|
|
193
|
+
orphans(): Promise<readonly string[]>
|
|
194
|
+
/** How long until every wake due within `withinMs` of the database's clock is due; 0 when none is early. */
|
|
195
|
+
earlyBy(withinMs: number): Promise<number>
|
|
196
|
+
/**
|
|
197
|
+
* Reaps a crashed holder of `job`'s lease, then, when one of its wakes is due, takes the lease, or sets the pending
|
|
198
|
+
* flag for the run holding it; of a job no longer declared (an orphan), only reaps. Statements only, for the caller's
|
|
199
|
+
* one transaction.
|
|
200
|
+
*/
|
|
201
|
+
take(job: string): Promise<TakenLease>
|
|
202
|
+
/** Runs `job` under the lease `take` took. A failed run retries the wakes it owns durably. */
|
|
203
|
+
run(job: string, fence: LeaseFence, options?: JobRunOptions): Promise<JobRunResult>
|
|
204
|
+
/** Reports the crashed run `take` reaped. */
|
|
205
|
+
crashed(job: string, options?: Pick<JobRunOptions, 'reporter'>): Promise<void>
|
|
206
|
+
/** When the scheduler wakes next: `JobWakes.next`, read in `using` when given. */
|
|
207
|
+
next(using?: Session): Promise<Date | null>
|
|
208
|
+
/** How far the next wake is from the database's now, in milliseconds: `JobWakes.untilNext`, read in `using` when given. */
|
|
209
|
+
untilNext(using?: Session): Promise<number | null>
|
|
210
|
+
/** Reports a failure of the scheduler itself: the alarm's or the repair's. Never rejects. */
|
|
211
|
+
failed(step: 'alarm' | 'repair', error: unknown): Promise<void>
|
|
126
212
|
}
|
|
127
213
|
|
|
128
214
|
/** A run that cannot start: an unknown job. Its message is framework-authored. */
|
|
@@ -131,6 +217,7 @@ export class JobError extends Error {
|
|
|
131
217
|
}
|
|
132
218
|
|
|
133
219
|
export const systemClock: JobClock = Object.freeze({
|
|
220
|
+
now: () => Date.now(),
|
|
134
221
|
wait: (ms: number, signal: AbortSignal) =>
|
|
135
222
|
new Promise<void>((resolve) => {
|
|
136
223
|
if (signal.aborted) {
|
|
@@ -171,6 +258,8 @@ interface RunnerContext {
|
|
|
171
258
|
readonly database: ContractDatabase | null
|
|
172
259
|
readonly ownedDimensions: ReadonlySet<string>
|
|
173
260
|
readonly leases: JobLeases
|
|
261
|
+
/** The jobs' pending wakes, which a run's finish consumes, retries or schedules. */
|
|
262
|
+
readonly wakes: JobWakes
|
|
174
263
|
readonly reporter: OperationsReporter
|
|
175
264
|
readonly mint: CapabilityMint
|
|
176
265
|
readonly data: DataAccess | null
|
|
@@ -187,9 +276,6 @@ interface RunnerContext {
|
|
|
187
276
|
readonly url: AppUrl
|
|
188
277
|
}
|
|
189
278
|
|
|
190
|
-
/** Where `fabrika app dev` shows what a handler threw, on the developer's own machine; nothing else sets one. */
|
|
191
|
-
export type HandlerErrorSink = (where: string, error: unknown) => void
|
|
192
|
-
|
|
193
279
|
export interface JobBindingInput {
|
|
194
280
|
readonly artifact: ContractArtifact
|
|
195
281
|
/** The handler tree `boot` binds; a job's handler is `handlers.jobs.<name>`. */
|
|
@@ -216,7 +302,7 @@ export interface JobBindingInput {
|
|
|
216
302
|
readonly capabilities: CapabilityAuthority
|
|
217
303
|
/** Each dimension's containers, nearest first: a link's chain. */
|
|
218
304
|
readonly containers: ReadonlyMap<string, readonly string[]>
|
|
219
|
-
/** What a job
|
|
305
|
+
/** What a job attempt threw, for the author's own run; without it the error is only its fixed code. */
|
|
220
306
|
readonly revealHandlerErrors?: HandlerErrorSink
|
|
221
307
|
/** The transports of the jobs that declare `mailer`, which `boot` resolved. */
|
|
222
308
|
readonly mail: MailTransports
|
|
@@ -307,11 +393,22 @@ export function bindJobs(input: JobBindingInput, problems: string[]): BoundJobs
|
|
|
307
393
|
}
|
|
308
394
|
const names = Object.freeze(entries.map(([name]) => name))
|
|
309
395
|
const framework = input.framework
|
|
310
|
-
const
|
|
396
|
+
const declaredJobs: readonly (Scheduled & { readonly timeoutMs: number })[] = [
|
|
397
|
+
...[...bound.values()].map((job) => ({ name: job.name, expression: job.artifact.schedule, schedule: job.schedule, timeoutMs: job.timeoutMs })),
|
|
398
|
+
...[...frameworkJobs.values()].map((own) => ({
|
|
399
|
+
name: own.name,
|
|
400
|
+
expression: own.job.schedule,
|
|
401
|
+
schedule: own.schedule,
|
|
402
|
+
timeoutMs: own.job.timeoutMs,
|
|
403
|
+
})),
|
|
404
|
+
]
|
|
405
|
+
let reconciled = false
|
|
406
|
+
const contextFor = (tables: FrameworkTables, chosen: OperationsReporter | undefined): RunnerContext => ({
|
|
311
407
|
artifact: input.artifact,
|
|
312
408
|
database: input.database,
|
|
313
409
|
ownedDimensions: input.ownedDimensions,
|
|
314
|
-
leases,
|
|
410
|
+
leases: tables.jobs,
|
|
411
|
+
wakes: tables.wakes,
|
|
315
412
|
reporter: chosen ?? input.reporter,
|
|
316
413
|
mint: input.mint,
|
|
317
414
|
data: input.data,
|
|
@@ -328,60 +425,151 @@ export function bindJobs(input: JobBindingInput, problems: string[]): BoundJobs
|
|
|
328
425
|
})
|
|
329
426
|
const jobs: ContractJobs = Object.freeze({
|
|
330
427
|
names,
|
|
428
|
+
scheduled: Object.freeze([...bound.values(), ...frameworkJobs.values()].map(({ name, schedule }) => ({ name, schedule }))),
|
|
331
429
|
run(name: string, options: JobRunOptions = {}): Promise<JobRunResult> {
|
|
332
430
|
const own = frameworkJobs.get(name)
|
|
333
431
|
if (own !== undefined && framework !== null) {
|
|
334
|
-
return
|
|
432
|
+
return admitted(
|
|
433
|
+
'framework-job',
|
|
434
|
+
name,
|
|
435
|
+
(control) => runFrameworkJob(own, contextFor(framework, options.reporter), options.clock ?? systemClock, control, null),
|
|
436
|
+
)
|
|
335
437
|
}
|
|
336
438
|
const job = bound.get(name)
|
|
337
439
|
if (job === undefined || framework === null) {
|
|
338
440
|
throw new JobError(`no job '${name}' in the contract (jobs: ${names.length === 0 ? 'none' : names.join(', ')})`)
|
|
339
441
|
}
|
|
340
|
-
return
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
442
|
+
return admitted(
|
|
443
|
+
'job',
|
|
444
|
+
name,
|
|
445
|
+
(control) => runJob(job, contextFor(framework, options.reporter), options.clock ?? systemClock, 'wait', control, 'inline'),
|
|
446
|
+
)
|
|
447
|
+
.then((result) => {
|
|
448
|
+
if (result.status === 'skipped') {
|
|
449
|
+
console.error(`fabrika: a previous run of job '${name}' still holds its lease; this run does not start`)
|
|
450
|
+
}
|
|
451
|
+
return result
|
|
452
|
+
})
|
|
346
453
|
},
|
|
347
454
|
async check(options: Pick<JobRunOptions, 'reporter'> = {}): Promise<void> {
|
|
348
|
-
|
|
455
|
+
const admission = framework === null ? null : admitInvocation('framework-job')
|
|
456
|
+
if (framework === null || admission === null) {
|
|
349
457
|
return
|
|
350
458
|
}
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
459
|
+
try {
|
|
460
|
+
const context = contextFor(framework, options.reporter)
|
|
461
|
+
for (const job of [...bound.values(), ...frameworkJobs.values()]) {
|
|
462
|
+
try {
|
|
463
|
+
await context.leases.register(job.name)
|
|
464
|
+
await surveil(job, context)
|
|
465
|
+
} catch {
|
|
466
|
+
// A driver error may quote the database URL.
|
|
467
|
+
console.error(`fabrika: the check of job '${job.name}' failed: driver_failed`)
|
|
468
|
+
}
|
|
359
469
|
}
|
|
470
|
+
} finally {
|
|
471
|
+
admission.release()
|
|
360
472
|
}
|
|
361
473
|
},
|
|
474
|
+
async wake(track: (kick: Promise<void>) => void): Promise<number | null> {
|
|
475
|
+
if (framework === null) {
|
|
476
|
+
return null
|
|
477
|
+
}
|
|
478
|
+
if (!reconciled) {
|
|
479
|
+
await framework.wakes.reconcile(declaredJobs.map((job) => ({ job: job.name, expression: job.expression })))
|
|
480
|
+
reconciled = true
|
|
481
|
+
}
|
|
482
|
+
// A crashed holder's expiry is a wake too, and a kick never takes a lease over.
|
|
483
|
+
for (const job of declaredJobs) {
|
|
484
|
+
if (await framework.jobs.reapCrashed(job.name)) {
|
|
485
|
+
await reportCrashed(job.name, input.reporter)
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
const now = (await framework.jobs.now()).getTime()
|
|
489
|
+
for (const job of bound.values()) {
|
|
490
|
+
const due = (await framework.wakes.pending(job.name)).filter((wake) => wake.dueAt.getTime() <= now)
|
|
491
|
+
if (due.length > 0 && !(await ownedByHolder(job.name, now, framework.jobs, framework.wakes))) {
|
|
492
|
+
track(kick(job.name))
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
const next = await framework.wakes.next()
|
|
496
|
+
return next === null ? null : Math.max(0, next.getTime() - now)
|
|
497
|
+
},
|
|
498
|
+
scheduler: framework === null ? null : jobScheduler({
|
|
499
|
+
framework,
|
|
500
|
+
declared: declaredJobs,
|
|
501
|
+
contextFor: (reporter) => contextFor(framework, reporter),
|
|
502
|
+
run(name: string, fence: LeaseFence, options: JobRunOptions): Promise<JobRunResult> {
|
|
503
|
+
const clock = options.clock ?? systemClock
|
|
504
|
+
const context = contextFor(framework, options.reporter)
|
|
505
|
+
const own = frameworkJobs.get(name)
|
|
506
|
+
if (own !== undefined) {
|
|
507
|
+
return admitted('framework-job', name, (control) => runFrameworkJob(own, context, clock, control, fence))
|
|
508
|
+
}
|
|
509
|
+
const job = bound.get(name)
|
|
510
|
+
if (job === undefined) {
|
|
511
|
+
throw new JobError(`no job '${name}' in the contract (jobs: ${names.length === 0 ? 'none' : names.join(', ')})`)
|
|
512
|
+
}
|
|
513
|
+
return admitted('job', name, (control) => runJob(job, context, clock, { fence }, control, 'durable'))
|
|
514
|
+
},
|
|
515
|
+
}),
|
|
362
516
|
})
|
|
363
517
|
const stopping = new AbortController()
|
|
364
518
|
const kick = async (name: string): Promise<void> => {
|
|
365
519
|
const job = bound.get(name)
|
|
366
|
-
if (job
|
|
367
|
-
|
|
520
|
+
if (job === undefined || framework === null) {
|
|
521
|
+
return
|
|
522
|
+
}
|
|
523
|
+
const admission = admitInvocation('kick')
|
|
524
|
+
if (admission === null) {
|
|
525
|
+
// A kick hurries a run; the schedule still runs the job once the object opens.
|
|
526
|
+
console.error(`fabrika: job '${name}' was not kicked: the object is in maintenance`)
|
|
527
|
+
return
|
|
528
|
+
}
|
|
529
|
+
try {
|
|
530
|
+
await kickOnce(job, contextFor(framework, undefined), {
|
|
531
|
+
signal: AbortSignal.any([admission.signal, stopping.signal]),
|
|
532
|
+
track: admission.track,
|
|
533
|
+
})
|
|
534
|
+
} finally {
|
|
535
|
+
admission.release()
|
|
368
536
|
}
|
|
369
537
|
}
|
|
370
538
|
return { jobs, declared, kick, stopKicks: () => stopping.abort() }
|
|
371
539
|
}
|
|
372
540
|
|
|
541
|
+
/**
|
|
542
|
+
* What stops one run, and where the handler work that outlives its attempt is counted: the run's admission at the
|
|
543
|
+
* object's barrier, which a drain aborts, and for a kick also the process's stop.
|
|
544
|
+
*/
|
|
545
|
+
type RunControl = Pick<InvocationAdmission, 'signal' | 'track'>
|
|
546
|
+
|
|
547
|
+
/** A run admitted at the current event's gate, released once it ended; a refused one is `stopped` and starts nothing. */
|
|
548
|
+
async function admitted(kind: 'job' | 'framework-job', name: string, run: (control: RunControl) => Promise<JobRunResult>): Promise<JobRunResult> {
|
|
549
|
+
const admission = admitInvocation(kind)
|
|
550
|
+
if (admission === null) {
|
|
551
|
+
console.error(`fabrika: job '${name}' did not start: the object is in maintenance; its schedule runs it again`)
|
|
552
|
+
return { status: 'failed', attempts: 0, code: 'stopped' }
|
|
553
|
+
}
|
|
554
|
+
try {
|
|
555
|
+
return await run(admission)
|
|
556
|
+
} finally {
|
|
557
|
+
admission.release()
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
|
|
373
561
|
/**
|
|
374
562
|
* One kick: a run while the lease is free, else the shared pending flag. Neither waits for a lock, so a kick never
|
|
375
563
|
* holds a connection requests share while a run writes. After a failed run the flag stays for the schedule.
|
|
376
564
|
*/
|
|
377
|
-
async function kickOnce(job: BoundJob, context: RunnerContext,
|
|
378
|
-
if (
|
|
565
|
+
async function kickOnce(job: BoundJob, context: RunnerContext, control: RunControl): Promise<void> {
|
|
566
|
+
if (control.signal.aborted) {
|
|
379
567
|
// The flag waits for no lock, and another process that is not stopping runs the job.
|
|
380
568
|
await markPending(job, context)
|
|
381
569
|
return
|
|
382
570
|
}
|
|
383
|
-
if ((await runJob(job, context, systemClock, 'try',
|
|
384
|
-
await handOver(job, context, systemClock,
|
|
571
|
+
if ((await runJob(job, context, systemClock, 'try', control, 'inline')).status === 'skipped') {
|
|
572
|
+
await handOver(job, context, systemClock, control, 'inline')
|
|
385
573
|
}
|
|
386
574
|
}
|
|
387
575
|
|
|
@@ -389,11 +577,11 @@ async function kickOnce(job: BoundJob, context: RunnerContext, stop: AbortSignal
|
|
|
389
577
|
* Leaves the flag to the run holding the lease, which takes it after it succeeded. That run may have finished before
|
|
390
578
|
* the flag was set, so the lease is tried once more; `null` when another run still holds it.
|
|
391
579
|
*/
|
|
392
|
-
async function handOver(job: BoundJob, context: RunnerContext, clock: JobClock,
|
|
580
|
+
async function handOver(job: BoundJob, context: RunnerContext, clock: JobClock, control: RunControl, retries: Retries): Promise<JobRunResult | null> {
|
|
393
581
|
if (!(await markPending(job, context))) {
|
|
394
582
|
return null
|
|
395
583
|
}
|
|
396
|
-
const result = await runJob(job, context, clock, 'try',
|
|
584
|
+
const result = await runJob(job, context, clock, 'try', control, retries)
|
|
397
585
|
return result.status === 'skipped' ? null : result
|
|
398
586
|
}
|
|
399
587
|
|
|
@@ -432,7 +620,7 @@ function jobLinks(where: string, site: string, declaration: SiteDeclaration | nu
|
|
|
432
620
|
return { name, prefix: mount.prefix, dimension: mount.dimension, issuedBy: mount.issuedBy }
|
|
433
621
|
}
|
|
434
622
|
|
|
435
|
-
const attemptId = (): string =>
|
|
623
|
+
const attemptId = (): string => uuidv7()
|
|
436
624
|
|
|
437
625
|
/** Exponential from one second, capped at five minutes, with half of it jittered. */
|
|
438
626
|
function backoffMs(attempt: number, random: number): number {
|
|
@@ -440,13 +628,123 @@ function backoffMs(attempt: number, random: number): number {
|
|
|
440
628
|
return Math.round(ceiling / 2 + (ceiling / 2) * random)
|
|
441
629
|
}
|
|
442
630
|
|
|
631
|
+
const nowOf = (clock: JobClock): number => clock.now?.() ?? Date.now()
|
|
632
|
+
|
|
633
|
+
/** How much of a segment that ends at `deadline` is left, never below zero. */
|
|
634
|
+
const remainingMs = (clock: JobClock, deadline: number): number => Math.max(0, deadline - nowOf(clock))
|
|
635
|
+
|
|
636
|
+
const finishReserveMs = (timeoutMs: number): number => Math.min(SEGMENT_FINISH_RESERVE_MS, Math.floor(timeoutMs / 10))
|
|
637
|
+
|
|
638
|
+
const OUT_OF_SEGMENT: unique symbol = Symbol('fabrika.job.outOfSegment')
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* `step` while the segment that ends at `end` lasts (always, without one). Past its end the run stops waiting for it;
|
|
642
|
+
* only a step that is safe to abandon goes here: a read, or the refusal of the attempt's writes, never a finish.
|
|
643
|
+
*/
|
|
644
|
+
async function withinSegment<T>(clock: JobClock, end: number | null, step: Promise<T>): Promise<T | typeof OUT_OF_SEGMENT> {
|
|
645
|
+
if (end === null) {
|
|
646
|
+
return await step
|
|
647
|
+
}
|
|
648
|
+
const timer = new AbortController()
|
|
649
|
+
// Abandoned past the segment, its rejection is nobody's: the race below still answers it while it is awaited.
|
|
650
|
+
step.catch(() => undefined)
|
|
651
|
+
try {
|
|
652
|
+
return await Promise.race([step, clock.wait(remainingMs(clock, end), timer.signal).then((): typeof OUT_OF_SEGMENT => OUT_OF_SEGMENT)])
|
|
653
|
+
} finally {
|
|
654
|
+
timer.abort()
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
/** A run whose segment ended before its finish started: its lease expires, the reap reports the crash, and the alarm runs it again. */
|
|
659
|
+
function outOfSegment(job: { readonly name: string }, attempts: number): JobRunResult {
|
|
660
|
+
console.error(`fabrika: job '${job.name}' reached the end of its segment before its finish; its lease expires and the alarm runs it again`)
|
|
661
|
+
return { status: 'failed', attempts, code: 'timeout' }
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
const MINUTE_MS = 60_000
|
|
665
|
+
/** How far ahead a next fire is looked for, as far as `previousDue` looks back. */
|
|
666
|
+
const NEXT_FIRE_DAYS = 366 * 5
|
|
667
|
+
|
|
668
|
+
/** The first due minute of `schedule` after `after`, or `null` when none falls in the window. */
|
|
669
|
+
function nextFire(schedule: CronSchedule, after: Date): Date | null {
|
|
670
|
+
const hours = [...schedule.hours].sort((left, right) => left - right)
|
|
671
|
+
const minutes = [...schedule.minutes].sort((left, right) => left - right)
|
|
672
|
+
const [firstHour, firstMinute] = [hours[0], minutes[0]]
|
|
673
|
+
if (firstHour === undefined || firstMinute === undefined) {
|
|
674
|
+
return null
|
|
675
|
+
}
|
|
676
|
+
const today = Date.UTC(after.getUTCFullYear(), after.getUTCMonth(), after.getUTCDate())
|
|
677
|
+
for (let ahead = 0; ahead <= NEXT_FIRE_DAYS; ahead++) {
|
|
678
|
+
const day = today + ahead * 24 * 60 * MINUTE_MS
|
|
679
|
+
// The day's first scheduled minute is due exactly when the day matches.
|
|
680
|
+
if (!isDueAt(schedule, new Date(day + (firstHour * 60 + firstMinute) * MINUTE_MS))) {
|
|
681
|
+
continue
|
|
682
|
+
}
|
|
683
|
+
for (const hour of hours) {
|
|
684
|
+
for (const minute of minutes) {
|
|
685
|
+
const time = day + (hour * 60 + minute) * MINUTE_MS
|
|
686
|
+
if (time > after.getTime()) {
|
|
687
|
+
return new Date(time)
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
return null
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
/** A cron wake's dedupe key: one pending wake per job and expression. */
|
|
696
|
+
const cronKey = (job: string, expression: string): string => `cron:${job}:${expression}`
|
|
697
|
+
|
|
698
|
+
/** A job's schedule as the scheduler and a finish read it. */
|
|
699
|
+
interface Scheduled {
|
|
700
|
+
readonly name: string
|
|
701
|
+
/** The cron expression as declared: its wake's key. */
|
|
702
|
+
readonly expression: string
|
|
703
|
+
readonly schedule: CronSchedule
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
/**
|
|
707
|
+
* Whether a run holds `job`'s lease and owns every wake due by `now`, as it took them with the lease: its finish consumes
|
|
708
|
+
* them, so a kick or a pending run for them would only repeat that run. A post-commit kick's run usually owns its wake.
|
|
709
|
+
*/
|
|
710
|
+
async function ownedByHolder(job: string, now: number, leases: JobLeases, wakes: JobWakes): Promise<boolean> {
|
|
711
|
+
const state = await leases.state(job)
|
|
712
|
+
if (state === null || state.holder === null) {
|
|
713
|
+
return false
|
|
714
|
+
}
|
|
715
|
+
return (await wakes.pending(job)).filter((wake) => wake.dueAt.getTime() <= now).every((wake) => wake.owned)
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
/** The wakes the run holding `job`'s lease owns: those it took with the lease. */
|
|
719
|
+
async function ownedWakes(job: string, context: RunnerContext): Promise<readonly Wake[]> {
|
|
720
|
+
return (await context.wakes.pending(job)).filter((wake) => wake.owned)
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
/**
|
|
724
|
+
* What a finish does with the owned wakes when the run is done with them, succeeded or failed past its retries: they
|
|
725
|
+
* are consumed, and when the job keeps a cron wake (always, for a run the alarm started) its next fire is scheduled
|
|
726
|
+
* under its key. On Bun, where crontab runs the schedule, a job keeps none.
|
|
727
|
+
*/
|
|
728
|
+
async function completion(job: Scheduled, context: RunnerContext, durable: boolean): Promise<FinishWakes> {
|
|
729
|
+
if (!durable && !(await context.wakes.pending(job.name)).some((wake) => wake.reason === 'cron')) {
|
|
730
|
+
return { kind: 'completed' }
|
|
731
|
+
}
|
|
732
|
+
const dueAt = nextFire(job.schedule, await context.leases.now())
|
|
733
|
+
return dueAt === null ? { kind: 'completed' } : { kind: 'completed', nextCron: { expression: job.expression, dueAt } }
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
/** A crashed run (its lease expired before it finished): one report. */
|
|
737
|
+
async function reportCrashed(job: string, reporter: OperationsReporter): Promise<void> {
|
|
738
|
+
const message = `job '${job}' failed: crashed (its run's lease expired before the run finished)`
|
|
739
|
+
console.error(`fabrika: ${message}`)
|
|
740
|
+
await reporter.report({ fingerprint: ['fabrika-job-failed', job], type: 'JobFailed', message, tags: { job, code: 'crashed' } })
|
|
741
|
+
}
|
|
742
|
+
|
|
443
743
|
/** A crashed run (its lease expired before it finished) and a job overdue by twice its period: one report each. */
|
|
444
744
|
async function surveil(job: { readonly name: string; readonly schedule: CronSchedule }, context: RunnerContext): Promise<void> {
|
|
445
745
|
const tags = { job: job.name }
|
|
446
746
|
if (await context.leases.reapCrashed(job.name)) {
|
|
447
|
-
|
|
448
|
-
console.error(`fabrika: ${message}`)
|
|
449
|
-
await context.reporter.report({ fingerprint: ['fabrika-job-failed', job.name], type: 'JobFailed', message, tags: { ...tags, code: 'crashed' } })
|
|
747
|
+
await reportCrashed(job.name, context.reporter)
|
|
450
748
|
}
|
|
451
749
|
const period = periodAt(job.schedule, await context.leases.now())
|
|
452
750
|
const claim = period === null ? null : await context.leases.claimOverdue(job.name, period)
|
|
@@ -464,30 +762,156 @@ async function surveil(job: { readonly name: string; readonly schedule: CronSche
|
|
|
464
762
|
})
|
|
465
763
|
}
|
|
466
764
|
|
|
765
|
+
interface SchedulerInput {
|
|
766
|
+
readonly framework: FrameworkTables
|
|
767
|
+
/** Every job bound to run, with the attempt time its lease is taken for. */
|
|
768
|
+
readonly declared: readonly (Scheduled & { readonly timeoutMs: number })[]
|
|
769
|
+
readonly contextFor: (reporter: OperationsReporter | undefined) => RunnerContext
|
|
770
|
+
readonly run: (job: string, fence: LeaseFence, options: JobRunOptions) => Promise<JobRunResult>
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
function jobScheduler(input: SchedulerInput): JobScheduler {
|
|
774
|
+
const { framework, declared } = input
|
|
775
|
+
const byName = new Map(declared.map((job) => [job.name, job]))
|
|
776
|
+
return Object.freeze({
|
|
777
|
+
jobs: Object.freeze(declared.map((job) => job.name)),
|
|
778
|
+
async repair(options: Pick<JobRunOptions, 'reporter'> = {}): Promise<boolean> {
|
|
779
|
+
const admission = admitInvocation('framework-job')
|
|
780
|
+
if (admission === null) {
|
|
781
|
+
return false
|
|
782
|
+
}
|
|
783
|
+
try {
|
|
784
|
+
const context = input.contextFor(options.reporter)
|
|
785
|
+
for (const job of await framework.wakes.expiredHolders()) {
|
|
786
|
+
if (await context.leases.reapCrashed(job)) {
|
|
787
|
+
await reportCrashed(job, context.reporter)
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
for (const job of declared) {
|
|
791
|
+
await context.leases.register(job.name)
|
|
792
|
+
await surveil(job, context)
|
|
793
|
+
}
|
|
794
|
+
await framework.wakes.reconcile(declared.map((job) => ({ job: job.name, expression: job.expression })))
|
|
795
|
+
const now = await context.leases.now()
|
|
796
|
+
for (const job of declared) {
|
|
797
|
+
const dueAt = nextFire(job.schedule, now)
|
|
798
|
+
if (dueAt !== null) {
|
|
799
|
+
await framework.wakes.schedule({ job: job.name, reason: 'cron', dueAt, dedupe: cronKey(job.name, job.expression) })
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
return true
|
|
803
|
+
} finally {
|
|
804
|
+
admission.release()
|
|
805
|
+
}
|
|
806
|
+
},
|
|
807
|
+
async orphans(): Promise<readonly string[]> {
|
|
808
|
+
return (await framework.wakes.expiredHolders()).filter((job) => !byName.has(job))
|
|
809
|
+
},
|
|
810
|
+
async earlyBy(withinMs: number): Promise<number> {
|
|
811
|
+
const now = (await framework.jobs.now()).getTime()
|
|
812
|
+
let early = 0
|
|
813
|
+
for (const job of declared) {
|
|
814
|
+
for (const wake of await framework.wakes.pending(job.name)) {
|
|
815
|
+
const ahead = wake.dueAt.getTime() - now
|
|
816
|
+
if (ahead <= withinMs) {
|
|
817
|
+
early = Math.max(early, ahead)
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
return early
|
|
822
|
+
},
|
|
823
|
+
async take(name: string): Promise<TakenLease> {
|
|
824
|
+
const job = byName.get(name)
|
|
825
|
+
const leases = framework.jobs
|
|
826
|
+
if (job === undefined) {
|
|
827
|
+
// A removed job runs no more; only a crashed run's lease is freed, so it stops arming the alarm.
|
|
828
|
+
return { fence: null, reaped: await leases.reapCrashed(name) }
|
|
829
|
+
}
|
|
830
|
+
await leases.register(name)
|
|
831
|
+
const reaped = await leases.reapCrashed(name)
|
|
832
|
+
const now = (await leases.now()).getTime()
|
|
833
|
+
const due = (await framework.wakes.pending(name)).filter((wake) => wake.dueAt.getTime() <= now)
|
|
834
|
+
if (due.length === 0) {
|
|
835
|
+
return { fence: null, reaped }
|
|
836
|
+
}
|
|
837
|
+
const fence = await leases.tryAcquire(name, attemptId(), job.timeoutMs, job.timeoutMs + LEASE_GRACE_MS)
|
|
838
|
+
if (fence === null && !(await ownedByHolder(name, now, leases, framework.wakes))) {
|
|
839
|
+
// The run holding the lease runs once more after it succeeded; a wake it did not own stays pending.
|
|
840
|
+
await leases.markPending(name)
|
|
841
|
+
}
|
|
842
|
+
return { fence, reaped }
|
|
843
|
+
},
|
|
844
|
+
run(job: string, fence: LeaseFence, options: JobRunOptions = {}): Promise<JobRunResult> {
|
|
845
|
+
return input.run(job, fence, options)
|
|
846
|
+
},
|
|
847
|
+
async crashed(job: string, options: Pick<JobRunOptions, 'reporter'> = {}): Promise<void> {
|
|
848
|
+
await reportCrashed(job, input.contextFor(options.reporter).reporter)
|
|
849
|
+
},
|
|
850
|
+
next(using?: Session): Promise<Date | null> {
|
|
851
|
+
return framework.wakes.next(using)
|
|
852
|
+
},
|
|
853
|
+
untilNext(using?: Session): Promise<number | null> {
|
|
854
|
+
return framework.wakes.untilNext(using)
|
|
855
|
+
},
|
|
856
|
+
async failed(step: 'alarm' | 'repair', error: unknown): Promise<void> {
|
|
857
|
+
try {
|
|
858
|
+
await input.contextFor(undefined).reporter.report({
|
|
859
|
+
fingerprint: ['fabrika-alarm-failed', step],
|
|
860
|
+
type: 'AlarmFailed',
|
|
861
|
+
message: `the job scheduler's ${step} failed`,
|
|
862
|
+
tags: { step },
|
|
863
|
+
error,
|
|
864
|
+
})
|
|
865
|
+
} catch {
|
|
866
|
+
console.error(`fabrika: the failure of the job scheduler's ${step} could not be reported`)
|
|
867
|
+
}
|
|
868
|
+
},
|
|
869
|
+
})
|
|
870
|
+
}
|
|
871
|
+
|
|
467
872
|
interface Progress {
|
|
468
873
|
attempts: number
|
|
469
874
|
}
|
|
470
875
|
|
|
876
|
+
/** What the attempt that succeeded kicked: `null` until a handler returned. */
|
|
877
|
+
interface Kicked {
|
|
878
|
+
recorder: KickRecorder | null
|
|
879
|
+
}
|
|
880
|
+
|
|
471
881
|
/**
|
|
472
882
|
* How a run takes the lease: `wait` is `fabrika job run`'s, in a process of its own; `try` is a kick's and a follow-up
|
|
473
|
-
* run's, which never wait for a lock.
|
|
883
|
+
* run's, which never wait for a lock; a fence is one the alarm's scheduler already took.
|
|
474
884
|
*/
|
|
475
|
-
type Acquisition = 'wait' | 'try'
|
|
885
|
+
type Acquisition = 'wait' | 'try' | { readonly fence: LeaseFence }
|
|
476
886
|
|
|
477
887
|
/**
|
|
478
|
-
*
|
|
479
|
-
*
|
|
888
|
+
* How a failed attempt is retried: `inline` in the same run, after a backoff, under a renewed lease; `durable` (a run the
|
|
889
|
+
* Cloudflare alarm started) by moving the wakes it owns to a later due time, so the alarm runs it again. A durable run
|
|
890
|
+
* that owns no wake retries inline: there is nothing to move.
|
|
480
891
|
*/
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
892
|
+
type Retries = 'inline' | 'durable'
|
|
893
|
+
|
|
894
|
+
/**
|
|
895
|
+
* A run of the job, then one more while a kick arrived after the last one started and it succeeded. Once the run's
|
|
896
|
+
* control aborted, no follow-up run starts: the pending flag stays for the schedule.
|
|
897
|
+
*/
|
|
898
|
+
async function runJob(
|
|
899
|
+
job: BoundJob,
|
|
900
|
+
context: RunnerContext,
|
|
901
|
+
clock: JobClock,
|
|
902
|
+
acquisition: Acquisition,
|
|
903
|
+
control: RunControl,
|
|
904
|
+
retries: Retries,
|
|
905
|
+
): Promise<JobRunResult> {
|
|
906
|
+
let result = await runOnce(job, context, clock, acquisition, control, retries)
|
|
907
|
+
while (result.status === 'succeeded' && !control.signal.aborted && (await pendingTaken(job, context))) {
|
|
908
|
+
const next = await runOnce(job, context, clock, 'try', control, retries)
|
|
485
909
|
if (next.status !== 'skipped') {
|
|
486
910
|
result = next
|
|
487
911
|
continue
|
|
488
912
|
}
|
|
489
913
|
// A newer run holds the lease and may have read before the kick's row: the flag goes back to it.
|
|
490
|
-
const handed = await handOver(job, context, clock,
|
|
914
|
+
const handed = await handOver(job, context, clock, control, retries)
|
|
491
915
|
return handed ?? result
|
|
492
916
|
}
|
|
493
917
|
return result
|
|
@@ -503,11 +927,23 @@ async function pendingTaken(job: BoundJob, context: RunnerContext): Promise<bool
|
|
|
503
927
|
}
|
|
504
928
|
}
|
|
505
929
|
|
|
506
|
-
async function runOnce(
|
|
930
|
+
async function runOnce(
|
|
931
|
+
job: BoundJob,
|
|
932
|
+
context: RunnerContext,
|
|
933
|
+
clock: JobClock,
|
|
934
|
+
acquisition: Acquisition,
|
|
935
|
+
control: RunControl,
|
|
936
|
+
retries: Retries,
|
|
937
|
+
): Promise<JobRunResult> {
|
|
507
938
|
const progress: Progress = { attempts: 0 }
|
|
508
939
|
try {
|
|
509
|
-
return await runLeased(job, context, clock, progress, acquisition,
|
|
940
|
+
return await runLeased(job, context, clock, progress, acquisition, control, retries)
|
|
510
941
|
} catch {
|
|
942
|
+
if (control.signal.aborted) {
|
|
943
|
+
// A stopped run's lease write is refused by the object's write fence; the lease expires and the schedule runs it.
|
|
944
|
+
console.error(`fabrika: job '${job.name}' stopped before it released its lease; its schedule runs it again`)
|
|
945
|
+
return { status: 'failed', attempts: progress.attempts, code: 'stopped' }
|
|
946
|
+
}
|
|
511
947
|
// Only the lease store throws here: the handler's errors stay inside its attempt. Its message may quote the URL.
|
|
512
948
|
return failed(job, progress.attempts, failure('driver_failed'), context.reporter)
|
|
513
949
|
}
|
|
@@ -519,32 +955,47 @@ async function runLeased(
|
|
|
519
955
|
clock: JobClock,
|
|
520
956
|
progress: Progress,
|
|
521
957
|
acquisition: Acquisition,
|
|
522
|
-
|
|
958
|
+
control: RunControl,
|
|
959
|
+
retries: Retries,
|
|
523
960
|
): Promise<JobRunResult> {
|
|
524
961
|
const { leases } = context
|
|
525
|
-
await leases.register(job.name)
|
|
526
962
|
const leaseMs = job.timeoutMs + LEASE_GRACE_MS
|
|
527
|
-
if (stop?.aborted === true) {
|
|
528
|
-
return { status: 'skipped' }
|
|
529
|
-
}
|
|
530
963
|
let fence: LeaseFence | null
|
|
531
|
-
if (acquisition === '
|
|
532
|
-
|
|
533
|
-
fence = await leases.acquire(job.name, attemptId(), job.timeoutMs, leaseMs)
|
|
964
|
+
if (typeof acquisition === 'object') {
|
|
965
|
+
fence = acquisition.fence
|
|
534
966
|
} else {
|
|
535
|
-
|
|
967
|
+
await leases.register(job.name)
|
|
968
|
+
if (control.signal.aborted) {
|
|
969
|
+
return { status: 'skipped' }
|
|
970
|
+
}
|
|
971
|
+
if (acquisition === 'wait') {
|
|
972
|
+
await surveil(job, context)
|
|
973
|
+
fence = await leases.acquire(job.name, attemptId(), job.timeoutMs, leaseMs)
|
|
974
|
+
} else {
|
|
975
|
+
fence = await leases.tryAcquire(job.name, attemptId(), job.timeoutMs, leaseMs)
|
|
976
|
+
}
|
|
536
977
|
}
|
|
537
978
|
if (fence === null) {
|
|
538
979
|
return { status: 'skipped' }
|
|
539
980
|
}
|
|
981
|
+
// The segment is measured from the lease, as the database's attempt deadline is. A run the alarm started has one
|
|
982
|
+
// segment: its attempts end before it by the finish's reserve, and no finish starts past it.
|
|
983
|
+
const segmentEnd = retries === 'durable' ? nowOf(clock) + job.timeoutMs : null
|
|
984
|
+
const attemptEnd = (
|
|
985
|
+
start: number,
|
|
986
|
+
): number => (segmentEnd === null ? start + job.timeoutMs : Math.min(start + job.timeoutMs, segmentEnd - finishReserveMs(job.timeoutMs)))
|
|
987
|
+
let deadline = attemptEnd(nowOf(clock))
|
|
540
988
|
// This run covers every kick before it: their rows committed before it reads.
|
|
541
989
|
await pendingTaken(job, context)
|
|
990
|
+
const owned = retries === 'durable' ? await ownedWakes(job.name, context) : []
|
|
991
|
+
const durable = owned.length > 0
|
|
992
|
+
const scheduled: Scheduled = { name: job.name, expression: job.artifact.schedule, schedule: job.schedule }
|
|
542
993
|
const attempts = job.artifact.retries + 1
|
|
543
|
-
progress.attempts = 1
|
|
544
|
-
// The
|
|
545
|
-
const kicked:
|
|
546
|
-
let outcome = await attemptOnce(job, fence, progress.attempts, clock, context, kicked,
|
|
547
|
-
while (outcome !== null && outcome.code !== 'stopped' && !outcome.final && progress.attempts < attempts) {
|
|
994
|
+
progress.attempts = durable ? Math.max(...owned.map((wake) => wake.attempt)) + 1 : 1
|
|
995
|
+
// The kicks of the succeeding attempt; a failed attempt's kicks are dropped with its writes.
|
|
996
|
+
const kicked: Kicked = { recorder: null }
|
|
997
|
+
let outcome = await attemptOnce(job, fence, progress.attempts, deadline, segmentEnd, clock, context, kicked, control)
|
|
998
|
+
while (!durable && outcome !== null && outcome.code !== 'stopped' && !outcome.final && progress.attempts < attempts) {
|
|
548
999
|
console.error(`fabrika: job '${job.name}' attempt ${progress.attempts} of ${attempts} failed: ${describeFailure(outcome)}`)
|
|
549
1000
|
const delay = backoffMs(progress.attempts, clock.random())
|
|
550
1001
|
// A new generation at once: nothing the failed attempt still has in flight commits from here on.
|
|
@@ -553,24 +1004,63 @@ async function runLeased(
|
|
|
553
1004
|
return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
|
|
554
1005
|
}
|
|
555
1006
|
fence = next
|
|
556
|
-
|
|
1007
|
+
deadline = attemptEnd(nowOf(clock) + delay)
|
|
1008
|
+
await clock.wait(delay, control.signal)
|
|
557
1009
|
progress.attempts++
|
|
558
|
-
outcome = await attemptOnce(job, fence, progress.attempts, clock, context, kicked,
|
|
1010
|
+
outcome = await attemptOnce(job, fence, progress.attempts, deadline, segmentEnd, clock, context, kicked, control)
|
|
1011
|
+
}
|
|
1012
|
+
const held = fence
|
|
1013
|
+
// The finish, unless the segment ended first; a finish once started is awaited, as it may be committing.
|
|
1014
|
+
const finish = async (status: 'succeeded' | 'failed', wakes: Promise<FinishWakes | undefined>): Promise<boolean | typeof OUT_OF_SEGMENT> => {
|
|
1015
|
+
const applied = await withinSegment(clock, segmentEnd, wakes)
|
|
1016
|
+
if (applied === OUT_OF_SEGMENT || (segmentEnd !== null && remainingMs(clock, segmentEnd) === 0)) {
|
|
1017
|
+
return OUT_OF_SEGMENT
|
|
1018
|
+
}
|
|
1019
|
+
return await leases.finish(held, status, applied)
|
|
559
1020
|
}
|
|
560
1021
|
if (outcome === null) {
|
|
561
|
-
|
|
1022
|
+
// Before the finish, so a crash between it and a kick's start leaves the kick's wake to the scheduler.
|
|
1023
|
+
const kicks = kicked.recorder
|
|
1024
|
+
for (const wake of kicks?.wakes(await leases.now()) ?? []) {
|
|
1025
|
+
await context.wakes.schedule(wake)
|
|
1026
|
+
}
|
|
1027
|
+
const finished = await finish('succeeded', completion(scheduled, context, durable))
|
|
1028
|
+
if (finished === OUT_OF_SEGMENT) {
|
|
1029
|
+
return outOfSegment(job, progress.attempts)
|
|
1030
|
+
}
|
|
1031
|
+
if (finished) {
|
|
562
1032
|
// After the lease is released, so a job that kicks another never waits on its own lease; a kick never rejects.
|
|
563
|
-
await Promise.all(kicked.map((name) => context.kick(name)))
|
|
1033
|
+
await Promise.all((kicks?.kicked() ?? []).map((name) => context.kick(name)))
|
|
564
1034
|
return { status: 'succeeded', attempts: progress.attempts }
|
|
565
1035
|
}
|
|
566
1036
|
return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
|
|
567
1037
|
}
|
|
568
|
-
await leases.finish(fence, 'failed')
|
|
569
1038
|
if (outcome.code === 'stopped') {
|
|
570
|
-
//
|
|
1039
|
+
// Its wakes stay, so the alarm or the schedule runs it again. Not a failure of the job: Operations hears nothing.
|
|
1040
|
+
await leases.finish(fence, 'failed')
|
|
571
1041
|
console.error(`fabrika: job '${job.name}' stopped: the process is shutting down; its schedule runs it again`)
|
|
572
1042
|
return { status: 'failed', attempts: progress.attempts, code: 'stopped' }
|
|
573
1043
|
}
|
|
1044
|
+
if (durable && !outcome.final && progress.attempts < attempts) {
|
|
1045
|
+
const at = new Date(nowOf(clock) + backoffMs(progress.attempts, clock.random()))
|
|
1046
|
+
console.error(
|
|
1047
|
+
`fabrika: job '${job.name}' attempt ${progress.attempts} of ${attempts} failed: ${describeFailure(outcome)}; it runs again at ${at.toISOString()}`,
|
|
1048
|
+
)
|
|
1049
|
+
const finished = await finish('failed', Promise.resolve({ kind: 'retry', at }))
|
|
1050
|
+
if (finished === OUT_OF_SEGMENT) {
|
|
1051
|
+
return outOfSegment(job, progress.attempts)
|
|
1052
|
+
}
|
|
1053
|
+
if (!finished) {
|
|
1054
|
+
return failed(job, progress.attempts, failure('lease_lost'), context.reporter)
|
|
1055
|
+
}
|
|
1056
|
+
return outcome.why === null
|
|
1057
|
+
? { status: 'failed', attempts: progress.attempts, code: outcome.code }
|
|
1058
|
+
: { status: 'failed', attempts: progress.attempts, code: outcome.code, reason: outcome.why.reason }
|
|
1059
|
+
}
|
|
1060
|
+
// Past its retries a run is done with its wakes: the next cron fire is scheduled, a kick's wake removed.
|
|
1061
|
+
if ((await finish('failed', completion(scheduled, context, durable))) === OUT_OF_SEGMENT) {
|
|
1062
|
+
return outOfSegment(job, progress.attempts)
|
|
1063
|
+
}
|
|
574
1064
|
return failed(job, progress.attempts, outcome, context.reporter)
|
|
575
1065
|
}
|
|
576
1066
|
|
|
@@ -609,14 +1099,27 @@ async function failed(job: { readonly name: string }, attempts: number, outcome:
|
|
|
609
1099
|
|
|
610
1100
|
const TIMED_OUT: unique symbol = Symbol('fabrika.job.timedOut')
|
|
611
1101
|
|
|
612
|
-
/**
|
|
613
|
-
|
|
1102
|
+
/**
|
|
1103
|
+
* One run of a framework job: its lease, one attempt in a frame fenced by it, and the outcome reported like a job's.
|
|
1104
|
+
* A run its control stopped is not a failure: the schedule runs it again. `acquired` is the lease the alarm's scheduler
|
|
1105
|
+
* took: such a run is done with the wakes it owns whether it succeeded or failed, as a framework job has no retries.
|
|
1106
|
+
*/
|
|
1107
|
+
async function runFrameworkJob(
|
|
1108
|
+
bound: BoundFrameworkJob,
|
|
1109
|
+
context: RunnerContext,
|
|
1110
|
+
clock: JobClock,
|
|
1111
|
+
control: RunControl,
|
|
1112
|
+
acquired: LeaseFence | null,
|
|
1113
|
+
): Promise<JobRunResult> {
|
|
614
1114
|
const { leases } = context
|
|
615
|
-
|
|
1115
|
+
const scheduled: Scheduled = { name: bound.name, expression: bound.job.schedule, schedule: bound.schedule }
|
|
1116
|
+
let fence: LeaseFence | null = acquired
|
|
616
1117
|
try {
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
1118
|
+
if (fence === null) {
|
|
1119
|
+
await leases.register(bound.name)
|
|
1120
|
+
await surveil(bound, context)
|
|
1121
|
+
fence = await leases.acquire(bound.name, attemptId(), bound.job.timeoutMs, bound.job.timeoutMs + LEASE_GRACE_MS)
|
|
1122
|
+
}
|
|
620
1123
|
} catch {
|
|
621
1124
|
return failed(bound, 0, failure('driver_failed'), context.reporter)
|
|
622
1125
|
}
|
|
@@ -624,14 +1127,18 @@ async function runFrameworkJob(bound: BoundFrameworkJob, context: RunnerContext,
|
|
|
624
1127
|
console.error(`fabrika: a previous run of job '${bound.name}' still holds its lease; this run does not start`)
|
|
625
1128
|
return { status: 'skipped' }
|
|
626
1129
|
}
|
|
1130
|
+
// A run the alarm took the lease for has one segment, whose end is kept for the finish, as a contract job's.
|
|
1131
|
+
const segmentEnd = acquired === null ? null : nowOf(clock) + bound.job.timeoutMs
|
|
627
1132
|
const frame = new RequestFrame(fence.holder, { fence })
|
|
628
1133
|
const cancel = new AbortController()
|
|
629
1134
|
const timer = new AbortController()
|
|
630
1135
|
let outcome: JobFailure | null
|
|
631
1136
|
try {
|
|
632
1137
|
const ran = runInRequest(frame, () => bound.job.run(context.reporter, cancel.signal)).then(() => null, (error: unknown) => failureOf(error, null))
|
|
633
|
-
|
|
634
|
-
const
|
|
1138
|
+
control.track(ran)
|
|
1139
|
+
const budget = segmentEnd === null ? bound.job.timeoutMs : bound.job.timeoutMs - finishReserveMs(bound.job.timeoutMs)
|
|
1140
|
+
const timeout = clock.wait(budget, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
|
|
1141
|
+
const result = await Promise.race([ran, timeout, stoppedBy(control, timer.signal)])
|
|
635
1142
|
outcome = result === TIMED_OUT ? failure('timeout') : result
|
|
636
1143
|
} finally {
|
|
637
1144
|
frame.end()
|
|
@@ -640,52 +1147,103 @@ async function runFrameworkJob(bound: BoundFrameworkJob, context: RunnerContext,
|
|
|
640
1147
|
try {
|
|
641
1148
|
// Writes are refused before the run hears it was stopped: a scanner that answers on abort commits nothing.
|
|
642
1149
|
try {
|
|
643
|
-
await leases.cancelInFlight(fence)
|
|
1150
|
+
await withinSegment(clock, segmentEnd, leases.cancelInFlight(fence))
|
|
644
1151
|
} finally {
|
|
645
1152
|
cancel.abort()
|
|
646
1153
|
}
|
|
647
|
-
|
|
1154
|
+
// A stopped run keeps its wakes, so the alarm runs it again once the object opens.
|
|
1155
|
+
const done = outcome === null || (acquired !== null && outcome.code !== 'stopped')
|
|
1156
|
+
const wakes = await withinSegment(clock, segmentEnd, done ? completion(scheduled, context, acquired !== null) : Promise.resolve(undefined))
|
|
1157
|
+
if (wakes === OUT_OF_SEGMENT || (segmentEnd !== null && remainingMs(clock, segmentEnd) === 0)) {
|
|
1158
|
+
return outOfSegment(bound, 1)
|
|
1159
|
+
}
|
|
1160
|
+
// Awaited once started: a finish may be committing.
|
|
1161
|
+
const finished = await leases.finish(fence, outcome === null ? 'succeeded' : 'failed', wakes)
|
|
648
1162
|
if (outcome === null) {
|
|
649
1163
|
return finished ? { status: 'succeeded', attempts: 1 } : failed(bound, 1, failure('lease_lost'), context.reporter)
|
|
650
1164
|
}
|
|
651
1165
|
} catch {
|
|
652
|
-
|
|
1166
|
+
// A stopped run's lease write is refused by the object's write fence; the lease expires and the schedule runs it.
|
|
1167
|
+
if (!control.signal.aborted) {
|
|
1168
|
+
return failed(bound, 1, failure('driver_failed'), context.reporter)
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
// Here a run without a failure of its own is one whose stop refused its lease write.
|
|
1172
|
+
if (outcome === null || outcome.code === 'stopped' || control.signal.aborted) {
|
|
1173
|
+
console.error(`fabrika: job '${bound.name}' stopped: the object entered maintenance; its schedule runs it again`)
|
|
1174
|
+
return { status: 'failed', attempts: 1, code: 'stopped' }
|
|
653
1175
|
}
|
|
654
1176
|
return failed(bound, 1, outcome, context.reporter)
|
|
655
1177
|
}
|
|
656
1178
|
|
|
657
1179
|
/** Read afresh at each call: the signal may abort while the caller awaits. */
|
|
658
|
-
const isStopped = (
|
|
1180
|
+
const isStopped = (control: RunControl): boolean => control.signal.aborted
|
|
1181
|
+
|
|
1182
|
+
/** Resolves `stopped` once the run's control aborts; the listener goes when `done` aborts. */
|
|
1183
|
+
function stoppedBy(control: RunControl, done: AbortSignal): Promise<JobFailure> {
|
|
1184
|
+
return new Promise<JobFailure>((resolve) => {
|
|
1185
|
+
if (control.signal.aborted) {
|
|
1186
|
+
resolve(failure('stopped'))
|
|
1187
|
+
}
|
|
1188
|
+
control.signal.addEventListener('abort', () => resolve(failure('stopped')), { once: true, signal: done })
|
|
1189
|
+
})
|
|
1190
|
+
}
|
|
659
1191
|
|
|
660
1192
|
/**
|
|
661
|
-
* `null` when the handler finished, else the attempt's failure code.
|
|
662
|
-
*
|
|
1193
|
+
* `null` when the handler finished, else the attempt's failure code. One deadline bounds the attempt: IAM's answer for
|
|
1194
|
+
* the job's key, then the handler with only what is left of it. Either way the attempt's frame ends and the statements
|
|
1195
|
+
* it still has running are cancelled before the runner touches the lease again, waited for only while the segment lasts.
|
|
663
1196
|
*/
|
|
664
1197
|
async function attemptOnce(
|
|
665
1198
|
job: BoundJob,
|
|
666
1199
|
fence: LeaseFence,
|
|
667
1200
|
attempt: number,
|
|
1201
|
+
deadline: number,
|
|
1202
|
+
segmentEnd: number | null,
|
|
668
1203
|
clock: JobClock,
|
|
669
1204
|
context: RunnerContext,
|
|
670
|
-
kicked:
|
|
671
|
-
|
|
1205
|
+
kicked: Kicked,
|
|
1206
|
+
control: RunControl,
|
|
672
1207
|
): Promise<JobFailure | null> {
|
|
673
|
-
kicked.
|
|
674
|
-
if (isStopped(
|
|
1208
|
+
kicked.recorder = null
|
|
1209
|
+
if (isStopped(control)) {
|
|
675
1210
|
return failure('stopped')
|
|
676
1211
|
}
|
|
677
1212
|
// The job's authority is what IAM signs for its key at the start of the attempt; a revoked key stops the next one.
|
|
678
|
-
const
|
|
679
|
-
|
|
1213
|
+
const setup = new AbortController()
|
|
1214
|
+
const exchange = job.credential.authority()
|
|
1215
|
+
// Its settlement, not only the attempt's: a drain waits for an exchange the attempt stopped waiting for.
|
|
1216
|
+
control.track(exchange)
|
|
1217
|
+
let authority: WorkloadAuthority
|
|
1218
|
+
try {
|
|
1219
|
+
const answered = await Promise.race([
|
|
1220
|
+
exchange,
|
|
1221
|
+
clock.wait(remainingMs(clock, deadline), setup.signal).then((): typeof TIMED_OUT => TIMED_OUT),
|
|
1222
|
+
stoppedBy(control, setup.signal),
|
|
1223
|
+
])
|
|
1224
|
+
if (answered === TIMED_OUT || 'code' in answered) {
|
|
1225
|
+
// The next attempt asks IAM afresh rather than wait on an exchange that outlived this one.
|
|
1226
|
+
job.credential.abandon(exchange)
|
|
1227
|
+
return answered === TIMED_OUT ? failure('timeout') : answered
|
|
1228
|
+
}
|
|
1229
|
+
authority = answered
|
|
1230
|
+
} finally {
|
|
1231
|
+
setup.abort()
|
|
1232
|
+
}
|
|
1233
|
+
if (isStopped(control)) {
|
|
680
1234
|
return failure('stopped')
|
|
681
1235
|
}
|
|
682
1236
|
if (!authority.ok) {
|
|
683
1237
|
return failure(authority.reason === 'unavailable' ? 'iam_unavailable' : 'unauthorized')
|
|
684
1238
|
}
|
|
1239
|
+
const budget = remainingMs(clock, deadline)
|
|
1240
|
+
if (budget === 0) {
|
|
1241
|
+
return failure('timeout')
|
|
1242
|
+
}
|
|
685
1243
|
const frame = new RequestFrame(fence.holder, { fence })
|
|
686
1244
|
const cancel = new AbortController()
|
|
687
1245
|
const timer = new AbortController()
|
|
688
|
-
const signal =
|
|
1246
|
+
const signal = AbortSignal.any([cancel.signal, control.signal])
|
|
689
1247
|
const kicks = kickRecorder(`job '${job.name}'`, job.site?.kicks ?? [])
|
|
690
1248
|
const ctx = { ...jobContext(job, frame, attempt, signal, context, authority.perms), kick: kicks.kick }
|
|
691
1249
|
try {
|
|
@@ -696,26 +1254,23 @@ async function attemptOnce(
|
|
|
696
1254
|
} else {
|
|
697
1255
|
await runAsSite(`job '${job.name}'`, site.egress, async () => await job.handler({ ctx }))
|
|
698
1256
|
}
|
|
699
|
-
kicked.
|
|
1257
|
+
kicked.recorder = kicks
|
|
700
1258
|
}).then(() => null, (error: unknown) => {
|
|
701
1259
|
const failed = failureOf(error, job.mailer === null ? null : (context.mail[job.mailer]?.sender.capabilities ?? null))
|
|
702
|
-
|
|
1260
|
+
// What the attempt threw: the handler's own error, or a framework failure whose message the log line withholds.
|
|
1261
|
+
if ('thrown' in failed) reveal(context.revealHandlerErrors, `job '${job.name}' attempt ${attempt}`, error)
|
|
703
1262
|
return failed
|
|
704
1263
|
})
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
}
|
|
710
|
-
stop?.addEventListener('abort', () => resolve(failure('stopped')), { once: true, signal: timer.signal })
|
|
711
|
-
})
|
|
712
|
-
const result = await Promise.race([ran, timeout, stopped])
|
|
1264
|
+
// Its settlement, not only the end of its frame: a drain waits for a handler that outlived the attempt.
|
|
1265
|
+
control.track(ran)
|
|
1266
|
+
const timeout = clock.wait(budget, timer.signal).then((): typeof TIMED_OUT => TIMED_OUT)
|
|
1267
|
+
const result = await Promise.race([ran, timeout, stoppedBy(control, timer.signal)])
|
|
713
1268
|
return result === TIMED_OUT ? failure('timeout') : result
|
|
714
1269
|
} finally {
|
|
715
1270
|
frame.end()
|
|
716
1271
|
cancel.abort()
|
|
717
1272
|
timer.abort()
|
|
718
|
-
await context.leases.cancelInFlight(fence)
|
|
1273
|
+
await withinSegment(clock, segmentEnd, context.leases.cancelInFlight(fence))
|
|
719
1274
|
}
|
|
720
1275
|
}
|
|
721
1276
|
|
|
@@ -723,8 +1278,9 @@ async function attemptOnce(
|
|
|
723
1278
|
const UNKNOWN_TRANSPORT: EmailSenderCapabilities = Object.freeze({ idempotent: false, confirmation: 'queued', maxBatch: 1 })
|
|
724
1279
|
|
|
725
1280
|
/**
|
|
726
|
-
* By class, never by name: an application error can call itself anything. A
|
|
727
|
-
* (`failureCode`), so both name it alike; every other
|
|
1281
|
+
* By class, never by name: an application error can call itself anything. A framework failure (a refusal or a
|
|
1282
|
+
* `ServerFault`) is classified as a request's is (`failureCode`), so both name it alike, with its reason; every other
|
|
1283
|
+
* thrown value, an `HttpError` of a 5xx status included, is `handler_failed`.
|
|
728
1284
|
*/
|
|
729
1285
|
function failureOf(error: unknown, transport: EmailSenderCapabilities | null): JobFailure {
|
|
730
1286
|
if (error instanceof LeaseLostError) {
|
|
@@ -735,10 +1291,14 @@ function failureOf(error: unknown, transport: EmailSenderCapabilities | null): J
|
|
|
735
1291
|
// ADR-0019's one retry rule: a repeat over a non-idempotent transport may deliver the message twice.
|
|
736
1292
|
return { code: 'mail_failed', why: { reason: error.code, text }, final: !mayRetryDelivery(error, transport ?? UNKNOWN_TRANSPORT) }
|
|
737
1293
|
}
|
|
1294
|
+
if (error instanceof ServerFault && (error.reason === 'iam_unavailable' || error.reason === 'workload_unauthorized')) {
|
|
1295
|
+
// IAM met inside the handler is named as at the attempt's start, so an alert on the code sees both.
|
|
1296
|
+
return { code: error.reason === 'iam_unavailable' ? 'iam_unavailable' : 'unauthorized', why: null, thrown: error, final: false }
|
|
1297
|
+
}
|
|
738
1298
|
const code = failureCode(error)
|
|
739
|
-
return code === '
|
|
740
|
-
? { code, why:
|
|
741
|
-
: { code
|
|
1299
|
+
return code === 'handler_failed' || code === 'handler_status'
|
|
1300
|
+
? { code: 'handler_failed', why: null, thrown: error, final: false }
|
|
1301
|
+
: { code, why: failureReason(error), thrown: error, final: false }
|
|
742
1302
|
}
|
|
743
1303
|
|
|
744
1304
|
function jobContext(
|
|
@@ -792,6 +1352,8 @@ function jobContext(
|
|
|
792
1352
|
checkAction(action)
|
|
793
1353
|
return context.mint.app(frame, action, principal.id)
|
|
794
1354
|
}
|
|
1355
|
+
// What `checkAction` admits: a job mints each of these over the whole application.
|
|
1356
|
+
const holds = job.artifact.actions.filter((action) => permits([...perms], action))
|
|
795
1357
|
return Object.freeze({
|
|
796
1358
|
principal,
|
|
797
1359
|
runId: frame.id,
|
|
@@ -810,6 +1372,7 @@ function jobContext(
|
|
|
810
1372
|
ref,
|
|
811
1373
|
app,
|
|
812
1374
|
periodAt(job.schedule, new Date()) ?? undefined,
|
|
1375
|
+
holds,
|
|
813
1376
|
)),
|
|
814
1377
|
...(job.links === null || context.membership === null ? {} : { links: jobLinksOf(job, job.links, context.membership, frame, context) }),
|
|
815
1378
|
...(job.mailer === null ? {} : { mail: jobMailOf(job, job.mailer, frame, signal, context) }),
|
|
@@ -887,36 +1450,49 @@ function kitJobContext(
|
|
|
887
1450
|
ref: (action: string, dimension: string, id: string, using: Session) => Promise<Ref<string, string>>,
|
|
888
1451
|
app: (action: string) => AppCapability<string>,
|
|
889
1452
|
tickMs: number | undefined,
|
|
1453
|
+
holds: readonly string[],
|
|
890
1454
|
): Readonly<Record<string, unknown>> {
|
|
891
1455
|
// One attempt's files: what it reads, puts and writes, inside its transactions and outside them, belongs to it.
|
|
892
1456
|
const work = new FileWork(frame, frame.id)
|
|
893
1457
|
const binding = kinds?.binding(work)
|
|
894
|
-
const reads = jobHandles(frame, data, site, principal, 'job', async () => data.direct(), () => false, tickMs, binding)
|
|
1458
|
+
const reads = jobHandles(frame, data, site, principal, 'job', async () => data.direct(), () => false, tickMs, binding, holds)
|
|
895
1459
|
const job = `job '${name}'`
|
|
896
1460
|
const putAndOpen = (check: () => void) => storedFilesHandle(kinds, work, { where: job, check, writes: true })
|
|
897
1461
|
return {
|
|
898
1462
|
db: siteData(reads, (action) => reads.app(app(action))),
|
|
899
1463
|
sql: site.sql === null ? undeclaredSql(job, 'job') : jobSqlOutsideTransaction(job),
|
|
900
1464
|
files: putAndOpen(boundTo(frame, () => false, 'ctx.files')),
|
|
901
|
-
transaction: async <R>(
|
|
1465
|
+
transaction: async <R>(body: (tx: object) => Promise<R>): Promise<R> => {
|
|
902
1466
|
// Outside its attempt there is no lease to fence the transaction with, so it never opens.
|
|
903
1467
|
boundTo(frame, () => false)()
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
const
|
|
911
|
-
|
|
912
|
-
|
|
1468
|
+
// Its own files: a rollback releases only what this transaction staged, never a file of a concurrent one.
|
|
1469
|
+
const own = new FileWork(frame, frame.id)
|
|
1470
|
+
let rolledBack = false
|
|
1471
|
+
try {
|
|
1472
|
+
return await data.transaction(async (session) => {
|
|
1473
|
+
let open = true
|
|
1474
|
+
const writes = jobHandles(frame, data, site, principal, 'mutation', async () => session, () => !open, undefined, binding)
|
|
1475
|
+
try {
|
|
1476
|
+
const sql = site.sql === null ? undeclaredSql(job, 'job', 'tx.sql') : jobSql(frame, data, site.sql, session, () => !open)
|
|
1477
|
+
// Located in this transaction, so an object it wrote is found before the commit.
|
|
1478
|
+
const located = (action: string, dimension: string, id: string) => {
|
|
1479
|
+
boundTo(frame, () => !open, 'tx.ref')()
|
|
1480
|
+
return ref(action, dimension, id, session)
|
|
1481
|
+
}
|
|
1482
|
+
const files = storedFilesHandle(kinds, own, { where: job, check: boundTo(frame, () => !open, 'tx.files'), writes: true })
|
|
1483
|
+
return await body(Object.freeze({ db: siteData(writes, (action) => writes.app(app(action))), sql, files, ref: located }))
|
|
1484
|
+
} catch (error) {
|
|
1485
|
+
rolledBack = true
|
|
1486
|
+
throw error
|
|
1487
|
+
} finally {
|
|
1488
|
+
open = false
|
|
913
1489
|
}
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
}
|
|
1490
|
+
})
|
|
1491
|
+
} catch (error) {
|
|
1492
|
+
// A transaction the work threw out of is a known rollback; a failed commit is not, and the sweep decides.
|
|
1493
|
+
if (rolledBack) await kinds?.release(own)
|
|
1494
|
+
throw error
|
|
1495
|
+
}
|
|
920
1496
|
},
|
|
921
1497
|
}
|
|
922
1498
|
}
|