@happyvertical/smrt-jobs 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +71 -0
- package/CLAUDE.md +1 -0
- package/LICENSE +7 -0
- package/README.md +151 -0
- package/dist/__smrt-register__.d.ts +2 -0
- package/dist/__smrt-register__.d.ts.map +1 -0
- package/dist/background-policy.d.ts +121 -0
- package/dist/background-policy.d.ts.map +1 -0
- package/dist/chunks/runner-DV8FBO0y.js +1642 -0
- package/dist/chunks/runner-DV8FBO0y.js.map +1 -0
- package/dist/chunks/worker-liveness-DOTjoIjr.js +65 -0
- package/dist/chunks/worker-liveness-DOTjoIjr.js.map +1 -0
- package/dist/error-redaction.d.ts +48 -0
- package/dist/error-redaction.d.ts.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +926 -0
- package/dist/index.js.map +1 -0
- package/dist/job-builder.d.ts +94 -0
- package/dist/job-builder.d.ts.map +1 -0
- package/dist/job-handle.d.ts +71 -0
- package/dist/job-handle.d.ts.map +1 -0
- package/dist/logger-extension.d.ts +58 -0
- package/dist/logger-extension.d.ts.map +1 -0
- package/dist/manifest.json +1327 -0
- package/dist/object-extension.d.ts +68 -0
- package/dist/object-extension.d.ts.map +1 -0
- package/dist/playground.d.ts +2 -0
- package/dist/playground.d.ts.map +1 -0
- package/dist/playground.js +179 -0
- package/dist/playground.js.map +1 -0
- package/dist/runner.d.ts +189 -0
- package/dist/runner.d.ts.map +1 -0
- package/dist/runner.js +15 -0
- package/dist/runner.js.map +1 -0
- package/dist/schedule-runner.d.ts +151 -0
- package/dist/schedule-runner.d.ts.map +1 -0
- package/dist/smrt-job-event.d.ts +54 -0
- package/dist/smrt-job-event.d.ts.map +1 -0
- package/dist/smrt-job.d.ts +215 -0
- package/dist/smrt-job.d.ts.map +1 -0
- package/dist/smrt-knowledge.json +508 -0
- package/dist/smrt-worker.d.ts +72 -0
- package/dist/smrt-worker.d.ts.map +1 -0
- package/dist/stale-recovery.d.ts +34 -0
- package/dist/stale-recovery.d.ts.map +1 -0
- package/dist/svelte/components/JobActions.svelte +103 -0
- package/dist/svelte/components/JobActions.svelte.d.ts +23 -0
- package/dist/svelte/components/JobActions.svelte.d.ts.map +1 -0
- package/dist/svelte/components/JobDashboard.svelte +199 -0
- package/dist/svelte/components/JobDashboard.svelte.d.ts +27 -0
- package/dist/svelte/components/JobDashboard.svelte.d.ts.map +1 -0
- package/dist/svelte/components/JobDetail.svelte +256 -0
- package/dist/svelte/components/JobDetail.svelte.d.ts +17 -0
- package/dist/svelte/components/JobDetail.svelte.d.ts.map +1 -0
- package/dist/svelte/components/JobList.svelte +360 -0
- package/dist/svelte/components/JobList.svelte.d.ts +28 -0
- package/dist/svelte/components/JobList.svelte.d.ts.map +1 -0
- package/dist/svelte/components/JobStats.svelte +242 -0
- package/dist/svelte/components/JobStats.svelte.d.ts +15 -0
- package/dist/svelte/components/JobStats.svelte.d.ts.map +1 -0
- package/dist/svelte/components/JobStatusBadge.svelte +23 -0
- package/dist/svelte/components/JobStatusBadge.svelte.d.ts +9 -0
- package/dist/svelte/components/JobStatusBadge.svelte.d.ts.map +1 -0
- package/dist/svelte/components/types.d.ts +9 -0
- package/dist/svelte/components/types.d.ts.map +1 -0
- package/dist/svelte/components/types.js +8 -0
- package/dist/svelte/i18n.d.ts +22 -0
- package/dist/svelte/i18n.d.ts.map +1 -0
- package/dist/svelte/i18n.js +22 -0
- package/dist/svelte/index.d.ts +25 -0
- package/dist/svelte/index.d.ts.map +1 -0
- package/dist/svelte/index.js +28 -0
- package/dist/svelte/playground.d.ts +329 -0
- package/dist/svelte/playground.d.ts.map +1 -0
- package/dist/svelte/playground.js +174 -0
- package/dist/svelte/types.d.ts +191 -0
- package/dist/svelte/types.d.ts.map +1 -0
- package/dist/svelte/types.js +87 -0
- package/dist/ui.d.ts +10 -0
- package/dist/ui.d.ts.map +1 -0
- package/dist/ui.js +69 -0
- package/dist/ui.js.map +1 -0
- package/dist/worker-liveness-thread.d.ts +2 -0
- package/dist/worker-liveness-thread.d.ts.map +1 -0
- package/dist/worker-liveness-thread.js +66 -0
- package/dist/worker-liveness-thread.js.map +1 -0
- package/dist/worker-liveness-ticker.d.ts +30 -0
- package/dist/worker-liveness-ticker.d.ts.map +1 -0
- package/dist/worker-liveness.d.ts +71 -0
- package/dist/worker-liveness.d.ts.map +1 -0
- package/package.json +93 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner-DV8FBO0y.js","sources":["../../src/__smrt-register__.ts","../../src/background-policy.ts","../../src/error-redaction.ts","../../src/logger-extension.ts","../../src/smrt-job.ts","../../src/smrt-job-event.ts","../../src/smrt-worker.ts","../../src/stale-recovery.ts","../../src/runner.ts"],"sourcesContent":["/**\n * Self-registers this package's build-time manifest before any @smrt() decorator\n * in the package fires. Fixes issue #1132: in consumer runtimes (tsx, SvelteKit\n * SSR, plain `vite dev`) the decorator's synchronous manifest lookup previously\n * missed because no step populated the global manifest cache — classes got\n * registered with zero fields and `save()` / `toJSON()` silently dropped every\n * declared property.\n *\n * Import this module as the first statement in `src/index.ts` so its top-level\n * side effect runs ahead of any class module's @smrt() decorator.\n *\n * Silent no-op in dev/test, where the vitest plugin already populates manifests\n * via a different path. Only needs to succeed in the published dist output.\n *\n * @see https://github.com/happyvertical/smrt/issues/1132\n */\nimport { ObjectRegistry } from '@happyvertical/smrt-core';\n\n// `new URL('./manifest.json', import.meta.url)` resolves at runtime to the\n// manifest sitting next to this module's compiled output. Vite warns at build\n// time that it cannot pre-resolve the URL; that is the intended behavior —\n// the URL must resolve to dist/manifest.json at runtime, not be inlined.\nObjectRegistry.registerPackageManifest(\n new URL('./manifest.json', import.meta.url),\n);\n","/**\n * Opt-in policy controls for background job creation and dispatch.\n *\n * @remarks\n * These guards harden the background-jobs surface flagged by the S5 audit\n * (#1402):\n *\n * - {@link MAX_JOB_RETRIES} caps the retry count a caller can request so a\n * misconfigured `.retries(n)` cannot pin a worker on a poison job forever.\n * - {@link assertWithinTenantCreationCap} bounds how many jobs a single tenant\n * may hold in the queue at once, so one tenant cannot exhaust the shared\n * worker pool (a cross-tenant denial of service).\n * - {@link isBackgroundEligibleMethod} / {@link backgroundEligible} provide an\n * opt-in allowlist of methods that may be invoked by the runner. The runner's\n * dispatch is already bounded to existing prototype methods (no eval / dynamic\n * import), but a class can further restrict which of its methods are reachable\n * from a persisted job row. This same marker is intended to be consumed by the\n * agents package, which dispatches methods through an equivalent path.\n */\n\n/**\n * Hard ceiling on retry attempts a caller may request via `.retries(n)` /\n * `bg(..., { retries })`. Requests above this are clamped (not rejected) so\n * existing callers keep working while the worst case stays bounded.\n */\nexport const MAX_JOB_RETRIES = 25;\n\n/**\n * Default maximum number of non-terminal (pending/running) jobs a single\n * tenant may hold in the queue at once. Configurable per call; `0` / negative\n * disables the cap.\n */\nexport const DEFAULT_TENANT_JOB_CAP = 10_000;\n\n/**\n * Clamp a requested retry count to {@link MAX_JOB_RETRIES}.\n *\n * @param requested - The retry count supplied by the caller.\n * @returns A non-negative integer no greater than {@link MAX_JOB_RETRIES}.\n */\nexport function clampRetries(requested: number): number {\n if (Number.isNaN(requested) || requested < 0) {\n return 0;\n }\n if (requested === Number.POSITIVE_INFINITY) {\n return MAX_JOB_RETRIES;\n }\n return Math.min(Math.floor(requested), MAX_JOB_RETRIES);\n}\n\n/**\n * Error thrown when a tenant exceeds its allowed in-flight job count.\n */\nexport class TenantJobCapExceededError extends Error {\n constructor(\n public readonly tenantId: string,\n public readonly cap: number,\n public readonly current: number,\n ) {\n super(\n `Tenant \"${tenantId}\" has reached its background-job cap ` +\n `(${current}/${cap} in-flight). Refusing to enqueue another job.`,\n );\n this.name = 'TenantJobCapExceededError';\n }\n}\n\n/**\n * Throw {@link TenantJobCapExceededError} when a tenant is at or above its cap.\n *\n * @param tenantId - Tenant the new job would belong to (`null` = global; not\n * subject to the per-tenant cap).\n * @param current - Current count of non-terminal jobs for the tenant.\n * @param cap - Maximum allowed; `<= 0` disables the check.\n */\nexport function assertWithinTenantCreationCap(\n tenantId: string | null | undefined,\n current: number,\n cap: number,\n): void {\n if (!tenantId || cap <= 0) return;\n if (current >= cap) {\n throw new TenantJobCapExceededError(tenantId, cap, current);\n }\n}\n\n/**\n * Class shape that opts into a background-method allowlist by declaring a\n * static set/array of method names.\n */\nexport interface BackgroundEligibleClass {\n /**\n * Method names that may be invoked by the job/agent runner. When present\n * (even if empty), it is treated as an exhaustive allowlist. When absent,\n * the runner falls back to its default behaviour (any existing method).\n */\n backgroundEligibleMethods?: ReadonlyArray<string> | ReadonlySet<string>;\n}\n\n/**\n * Add method names to a class's background-eligible allowlist.\n *\n * Installs/extends a static `backgroundEligibleMethods` set on the constructor.\n * Once any method is marked, the runner refuses to dispatch a job whose\n * `method` is not in the set — turning the dispatch surface from \"any prototype\n * method\" into an explicit contract. Use this when applying the\n * {@link backgroundEligible} decorator is inconvenient (or in non-decorator\n * code, including the agents package).\n *\n * @param ctor - The class constructor to annotate.\n * @param methods - Method names to allow.\n */\nexport function markBackgroundEligible(\n ctor: object,\n ...methods: string[]\n): void {\n const target = ctor as { backgroundEligibleMethods?: ReadonlySet<string> };\n const existing = target.backgroundEligibleMethods;\n const set =\n existing instanceof Set\n ? new Set<string>(existing)\n : new Set<string>(existing ?? []);\n for (const method of methods) set.add(method);\n target.backgroundEligibleMethods = set;\n}\n\n/**\n * Decorator: mark a method as background-eligible.\n *\n * This is a legacy (`experimentalDecorators`) method decorator — the mode the\n * SMRT monorepo compiles with. Applying it (one or more times) builds up the\n * static `backgroundEligibleMethods` allowlist on the owning class. Once any\n * method is marked, the runner will refuse to dispatch a job whose `method` is\n * not in the set.\n *\n * @example\n * ```ts\n * class Report extends SmrtObject {\n * \\@backgroundEligible()\n * async regenerate() {} // reachable from a job\n *\n * async deleteEverything() {} // NOT reachable from a job\n * }\n * ```\n */\nexport function backgroundEligible() {\n return (\n target: object,\n propertyKey: string | symbol,\n descriptor?: PropertyDescriptor,\n ): PropertyDescriptor | undefined => {\n // Legacy method decorators receive the prototype as `target`; the class\n // constructor (where we keep the static allowlist) is `target.constructor`.\n const ctor = (target as { constructor: object }).constructor;\n markBackgroundEligible(ctor, String(propertyKey));\n return descriptor;\n };\n}\n\n/**\n * Resolve the declared allowlist for a class, if any.\n *\n * @param ctor - The target object's constructor.\n * @returns A `Set` of allowed method names, or `null` when the class did not\n * opt in (runner should fall back to default behaviour).\n */\nexport function getBackgroundEligibleMethods(\n ctor: unknown,\n): ReadonlySet<string> | null {\n const declared = (ctor as BackgroundEligibleClass | undefined)\n ?.backgroundEligibleMethods;\n if (declared == null) return null;\n return declared instanceof Set ? declared : new Set(declared);\n}\n\n/**\n * Whether a method may be invoked by the runner for a given target class.\n *\n * @param ctor - Constructor of the resolved target class.\n * @param method - Method name from the persisted job row.\n * @returns `true` when the class declared no allowlist (default) or when the\n * method is on the allowlist; `false` when an allowlist exists and excludes\n * the method.\n */\nexport function isBackgroundEligibleMethod(\n ctor: unknown,\n method: string,\n): boolean {\n const allow = getBackgroundEligibleMethods(ctor);\n if (allow == null) return true;\n return allow.has(method);\n}\n","/**\n * Redact secret-shaped substrings from a job error message before it is\n * persisted to the `_smrt_jobs.last_error` column.\n *\n * @remarks\n * `last_error` is durable and readable through generated `list`/`get` routes\n * (now tenant-scoped — see {@link \"./smrt-job\".SmrtJob}). Error messages thrown\n * from a failing job frequently echo back the arguments or environment that\n * caused the failure: a database URL with embedded credentials, a `Bearer`\n * token, an `Authorization` header, an API key, or a `key=value` secret pair.\n * Persisting those verbatim turns a transient failure into a durable credential\n * leak (S5 audit #1402).\n *\n * The policy biases toward over-redaction: matching a benign string is a\n * cosmetic loss, while leaking a credential is a security incident. Patterns are\n * intentionally conservative and order-independent — each is applied globally so\n * multiple secrets in one message are all masked.\n */\n\nconst REDACTED = '***REDACTED***';\n\n/**\n * Credentials embedded in a URL userinfo segment:\n * `scheme://user:pass@host` → `scheme://***REDACTED***@host`.\n * The host is preserved so the message stays diagnosable.\n */\nconst CREDENTIAL_URL_RE = /([a-z][a-z0-9+.-]*:\\/\\/)[^/?#@\\s]+@/gi;\n\n/**\n * `Bearer <token>` / `token <token>` style authorization values.\n */\nconst BEARER_TOKEN_RE = /\\b(bearer|token)\\s+[A-Za-z0-9._\\-+/=]{8,}/gi;\n\n/**\n * An `Authorization:` header value (covers Basic/Bearer/etc.). Captures the\n * scheme word plus the credential token that follows it so the whole value is\n * masked, not just the scheme.\n */\nconst AUTHORIZATION_HEADER_RE =\n /\\bauthorization\\s*[:=]\\s*(?:[A-Za-z]+\\s+)?[^\\s,;)]+/gi;\n\n/**\n * Common secret-bearing `key=value` / `key: value` pairs. The key portion is\n * preserved so the shape of the failure remains legible; only the value is\n * masked. Matches camelCase / snake_case / kebab / UPPER variants.\n */\nconst SECRET_KEY_VALUE_RE =\n /\\b([A-Za-z0-9_-]*(?:password|passwd|pwd|secret|api[_-]?key|access[_-]?key|secret[_-]?key|private[_-]?key|client[_-]?secret|token|credential|auth)[A-Za-z0-9_-]*)\\s*([:=])\\s*(\"[^\"]*\"|'[^']*'|[^\\s,;)]+)/gi;\n\n/**\n * Secret-bearing keys written as JSON object members, e.g.\n * `{\"password\":\"x\"}` or `{ \"apiKey\" : \"y\" }`. The generic key=value rule above\n * cannot see these because the key is wrapped in quotes (so the separator does\n * not immediately follow the key word). The quoted key is preserved; only the\n * quoted value is masked (S5 audit #1402).\n */\nconst JSON_SECRET_KEY_VALUE_RE =\n /(\"(?:[A-Za-z0-9_-]*(?:password|passwd|pwd|secret|api[_-]?key|access[_-]?key|secret[_-]?key|private[_-]?key|client[_-]?secret|token|credential|auth)[A-Za-z0-9_-]*)\")\\s*:\\s*\"[^\"]*\"/gi;\n\n/**\n * Provider-recognizable standalone secrets that may appear without a key:\n * OpenAI `sk-...` (incl. hyphenated project keys like `sk-proj-...`) /\n * `sk_live_...` / `sk_test_...`, AWS access key id (`AKIA...`), GitHub tokens\n * (`ghp_`/`gho_`/`ghu_`/`ghs_`/`ghr_`), Google API keys (`AIza...`), and Slack\n * bot/user tokens (`xoxb-`/`xoxp-`) (S5 audit #1402).\n *\n * The OpenAI patterns allow internal hyphens/underscores in the token *body*\n * (after the `sk-`/`sk_` prefix) so segmented keys such as `sk-proj-ABC123...`\n * are matched whole rather than truncated at the first separator. The body must\n * end on an alphanumeric so a trailing separator is not swallowed, and a length\n * floor keeps short benign `sk-foo` tokens from over-matching.\n */\nconst STANDALONE_SECRET_RE =\n /\\b(sk-[A-Za-z0-9_-]{14,}[A-Za-z0-9]|sk_(?:live|test)_[A-Za-z0-9_-]{14,}[A-Za-z0-9]|AKIA[0-9A-Z]{12,}|gh[pousr]_[A-Za-z0-9]{20,}|AIza[A-Za-z0-9_-]{20,}|xox[bp]-[A-Za-z0-9-]{10,})\\b/g;\n\n/**\n * Mask secret-shaped substrings in an error message.\n *\n * Strictly `string => string`: pass a known message here. For an arbitrary\n * throwable of unknown shape (e.g. a caught `unknown`), call\n * {@link redactErrorForPersistence} instead — it coerces non-`Error` values to\n * a string first. The runtime non-string guard below is defensive depth only;\n * the type contract is that callers supply a string.\n *\n * @param message - Raw error message (e.g. `error.message`).\n * @returns The message with credential-like substrings replaced by\n * `***REDACTED***`. Empty input is returned unchanged.\n *\n * @example\n * ```ts\n * redactErrorMessage('connect failed: postgres://u:p@db/app')\n * // => 'connect failed: postgres://***REDACTED***@db/app'\n * redactErrorMessage('401 from api: apiKey=<the-key>')\n * // => '401 from api: apiKey=***REDACTED***'\n * ```\n */\nexport function redactErrorMessage(message: string): string {\n if (typeof message !== 'string' || message.length === 0) {\n return message;\n }\n\n return (\n message\n .replace(CREDENTIAL_URL_RE, `$1${REDACTED}@`)\n // Authorization headers run before the generic key=value rule so the\n // full `<scheme> <token>` value is masked rather than just the scheme.\n .replace(AUTHORIZATION_HEADER_RE, `authorization=${REDACTED}`)\n .replace(BEARER_TOKEN_RE, `$1 ${REDACTED}`)\n // JSON-shaped secrets run before the generic key=value rule (which cannot\n // match a quoted key) so `{\"password\":\"x\"}` is masked to a valid shape.\n .replace(JSON_SECRET_KEY_VALUE_RE, `$1:\"${REDACTED}\"`)\n .replace(SECRET_KEY_VALUE_RE, `$1$2${REDACTED}`)\n .replace(STANDALONE_SECRET_RE, REDACTED)\n );\n}\n\n/**\n * Redact an arbitrary error's message, tolerating non-Error throwables.\n *\n * @param error - Anything thrown (`Error`, string, or unknown).\n * @returns A redacted message string suitable for persistence.\n */\nexport function redactErrorForPersistence(error: unknown): string {\n if (error instanceof Error) {\n return redactErrorMessage(error.message);\n }\n return redactErrorMessage(String(error));\n}\n","import type { Logger } from '@happyvertical/logger';\n\n/**\n * Job context for logging\n */\nexport interface JobContext {\n jobId: string;\n attempt: number;\n queue: string;\n objectType: string;\n method: string;\n}\n\nexport interface JobEventInput {\n type?: string;\n level?: 'debug' | 'info' | 'warn' | 'error';\n stage?: string | null;\n progress?: number | null;\n message?: string;\n data?: Record<string, unknown>;\n}\n\nexport interface JobProgressInput {\n stage: string;\n progress: number;\n message?: string;\n detail?: string;\n source?: string;\n data?: Record<string, unknown>;\n}\n\nexport interface JobExecutionContext {\n job: JobContext & { tenantId?: string | null };\n logger: Logger;\n event(input: JobEventInput): Promise<void>;\n progress(input: JobProgressInput): Promise<void>;\n log(\n level: 'debug' | 'info' | 'warn' | 'error',\n message: string,\n data?: Record<string, unknown>,\n ): Promise<void>;\n}\n\n/**\n * Logger extension that auto-injects job context into all log entries\n *\n * During job execution, all logs automatically include:\n * - jobId: The job's unique identifier\n * - attempt: Which attempt this is (1, 2, 3...)\n * - queue: Which queue the job is in\n * - objectType: The type of object being operated on\n * - method: The method being invoked\n */\nexport class JobContextLogger implements Logger {\n constructor(\n private readonly baseLogger: Logger,\n private readonly jobContext: JobContext,\n ) {}\n\n private addContext(data?: Record<string, unknown>): Record<string, unknown> {\n return {\n ...data,\n _job: {\n id: this.jobContext.jobId,\n attempt: this.jobContext.attempt,\n queue: this.jobContext.queue,\n objectType: this.jobContext.objectType,\n method: this.jobContext.method,\n },\n };\n }\n\n debug(message: string, data?: Record<string, unknown>): void {\n this.baseLogger.debug(message, this.addContext(data));\n }\n\n info(message: string, data?: Record<string, unknown>): void {\n this.baseLogger.info(message, this.addContext(data));\n }\n\n warn(message: string, data?: Record<string, unknown>): void {\n this.baseLogger.warn(message, this.addContext(data));\n }\n\n error(message: string, data?: Record<string, unknown>): void {\n this.baseLogger.error(message, this.addContext(data));\n }\n}\n\nexport default JobContextLogger;\n","import type { RetryStrategyConfig } from '@happyvertical/jobs';\nimport {\n detectEngine,\n ensureJobsSystemTableCompatibility,\n field,\n SmrtCollection,\n SmrtObject,\n smrt,\n} from '@happyvertical/smrt-core';\nimport {\n getTenantId,\n TenantScoped,\n tenantId,\n} from '@happyvertical/smrt-tenancy';\nimport type { DatabaseInterface } from '@happyvertical/sql';\nimport {\n assertWithinTenantCreationCap,\n clampRetries,\n DEFAULT_TENANT_JOB_CAP,\n} from './background-policy.js';\n\n/**\n * Job status type\n */\nexport type JobStatus =\n | 'pending'\n | 'running'\n | 'completed'\n | 'failed'\n | 'cancelled';\n\n/**\n * Timeout behavior type\n */\nexport type TimeoutBehavior = 'fail' | 'kill' | 'warn';\n\n/**\n * Persistent job record stored in the `_smrt_jobs` system table.\n *\n * @remarks\n * Each SmrtJob represents a deferred method call on a SmrtObject. The TaskRunner polls for\n * pending jobs, resolves the target class via ObjectRegistry, and invokes the method. Jobs\n * track status (`pending -> running -> completed/failed/cancelled`), retry attempts with\n * configurable strategies, worker heartbeats for stale-job detection, and optional result pointers.\n * Priority ordering is `higher = sooner`; the default timeout is 5 minutes (300000ms).\n */\n@smrt({\n tableName: '_smrt_jobs',\n // Fail closed: `_smrt_jobs` is an internal operational queue table. Generated\n // REST/MCP list/get only filter by tenant when a tenant context is active;\n // reached without context (a tenant-less/admin principal, or any non-SvelteKit\n // surface) an `optional`-mode class returns UNFILTERED rows, leaking every\n // tenant's jobs. Workers read this table via the collection directly\n // (allowRawOnTenantScoped), never through generated routes, so nothing\n // internal needs the read surface — so we do not generate one (S5 audit #1402).\n api: false,\n // retry/cancel are operator commands invoked in-process via the CLI;\n // they intentionally aren't exposed over HTTP.\n cli: {\n include: ['list', 'get', 'retry', 'cancel'],\n skipApiCheck: true,\n http: false,\n },\n mcp: false,\n})\n// Keep the data model tenant-scoped (defense in depth): even without a generated\n// read route, the @tenantId() field alone does NOT make collection reads filter\n// by tenant. `optional` mode preserves global (NULL tenant) jobs while scoping\n// tenant-owned rows for any future tenant-context-required path (S5 audit #1402).\n@TenantScoped({ mode: 'optional' })\nexport class SmrtJob extends SmrtObject {\n /** Tenant context captured for this job, if any */\n @tenantId({ nullable: true })\n tenantId: string | null | undefined = undefined;\n\n /** Queue name for the job */\n @field({ type: 'text', required: true, default: 'default' })\n queue: string = 'default';\n\n /** Type of object to invoke method on */\n @field({ type: 'text', required: true })\n objectType: string = '';\n\n /** ID of the specific object (null for static methods) */\n @field({ type: 'text', nullable: true })\n objectId: string | null = null;\n\n /** Method name to invoke */\n @field({ type: 'text', required: true })\n method: string = '';\n\n /** Arguments to pass to the method (JSON) */\n @field({ type: 'json' })\n args: Record<string, unknown> = {};\n\n /** When to run the job */\n @field({ type: 'datetime', required: true })\n runAt: Date = new Date();\n\n /** Priority (higher = sooner) */\n @field({ type: 'integer', required: true, default: 50 })\n priority: number = 50;\n\n /** Current status */\n @field({ type: 'text', required: true, default: 'pending' })\n status: JobStatus = 'pending';\n\n /** Number of execution attempts */\n @field({ type: 'integer', required: true, default: 0 })\n attempts: number = 0;\n\n /** Maximum retry attempts */\n @field({ type: 'integer', required: true, default: 3 })\n maxAttempts: number = 3;\n\n /** Timeout in milliseconds */\n @field({ type: 'integer', required: true, default: 300000 })\n timeout: number = 300000;\n\n /** What to do on timeout */\n @field({ type: 'text', required: true, default: 'fail' })\n timeoutBehavior: TimeoutBehavior = 'fail';\n\n /** When execution started */\n @field({ type: 'datetime', nullable: true })\n startedAt: Date | null = null;\n\n /** When execution completed */\n @field({ type: 'datetime', nullable: true })\n completedAt: Date | null = null;\n\n /** Last error message */\n @field({ type: 'text', nullable: true })\n lastError: string | null = null;\n\n /** Pointer to where result is stored */\n @field({ type: 'text', nullable: true })\n resultPointer: string | null = null;\n\n /** Retry strategy configuration */\n @field({ type: 'json' })\n retryStrategy: RetryStrategyConfig = {\n type: 'exponential',\n config: { initialDelay: 1000, multiplier: 2, maxDelay: 300000 },\n };\n\n /** ID of the worker processing this job */\n @field({ type: 'text', nullable: true })\n workerId: string | null = null;\n\n /** Last heartbeat from the worker */\n @field({ type: 'datetime', nullable: true })\n workerHeartbeat: Date | null = null;\n\n /**\n * Capture ambient tenant context when a job is saved inside withTenant().\n *\n * Scheduled jobs can also set this explicitly from their owning schedule.\n */\n override async save(): Promise<this> {\n if (this.tenantId === undefined) {\n const contextTenantId = getTenantId();\n if (contextTenantId) {\n this.tenantId = contextTenantId;\n }\n }\n\n return super.save();\n }\n\n /**\n * Mark the job for retry\n */\n async retry(): Promise<void> {\n if (this.status === 'completed') {\n throw new Error('Cannot retry a completed job');\n }\n\n this.status = 'pending';\n this.attempts = 0;\n this.lastError = null;\n this.startedAt = null;\n this.completedAt = null;\n this.workerId = null;\n this.workerHeartbeat = null;\n\n await this.save();\n }\n\n /**\n * Cancel the job\n */\n async cancel(): Promise<void> {\n if (this.status === 'completed' || this.status === 'cancelled') {\n throw new Error(`Cannot cancel job with status: ${this.status}`);\n }\n\n this.status = 'cancelled';\n this.completedAt = new Date();\n\n await this.save();\n }\n\n /**\n * Get a human-readable description of the job\n */\n getDescription(): string {\n const target = this.objectId\n ? `${this.objectType}#${this.objectId}`\n : this.objectType;\n return `${target}.${this.method}()`;\n }\n}\n\n/**\n * Job data type (for create operations)\n */\nexport interface SmrtJobData {\n tenantId?: string | null;\n queue?: string;\n objectType: string;\n objectId?: string | null;\n method: string;\n args?: Record<string, unknown>;\n runAt?: Date;\n priority?: number;\n maxAttempts?: number;\n timeout?: number;\n timeoutBehavior?: TimeoutBehavior;\n retryStrategy?: RetryStrategyConfig;\n}\n\n/**\n * Options controlling a centralized {@link SmrtJobCollection.enqueueJob} call.\n */\nexport interface EnqueueJobOptions {\n /**\n * Per-tenant in-flight cap. Defaults to {@link DEFAULT_TENANT_JOB_CAP}.\n * `0`/negative disables the cap (trusted internal callers). Global\n * (no-context / null tenant) jobs are always exempt.\n */\n tenantJobCap?: number;\n}\n\n/**\n * Options for listReady\n */\nexport interface ListReadyOptions {\n limit?: number;\n queues?: string[];\n}\n\n/**\n * Options for atomically claiming ready jobs.\n */\nexport interface ClaimReadyOptions extends ListReadyOptions {\n workerId: string;\n now?: Date;\n}\n\ntype DatabaseWithConfig = DatabaseInterface & {\n config?: {\n type?: string;\n url?: string;\n };\n type?: string;\n};\n\n/**\n * Collection for managing SmrtJob objects\n */\nexport class SmrtJobCollection extends SmrtCollection<SmrtJob> {\n static readonly _itemClass = SmrtJob;\n\n override async initialize(): Promise<this> {\n await super.initialize();\n await ensureJobsSystemTableCompatibility(this.db);\n return this;\n }\n\n /**\n * List jobs by status\n */\n async listByStatus(\n status: JobStatus | JobStatus[],\n options: { limit?: number; queue?: string } = {},\n ): Promise<SmrtJob[]> {\n const where: Record<string, unknown> = {\n status: Array.isArray(status) ? status : [status],\n };\n\n if (options.queue) {\n where.queue = options.queue;\n }\n\n return this.list({\n where,\n orderBy: ['priority DESC', 'run_at ASC'],\n limit: options.limit,\n });\n }\n\n /**\n * List pending jobs ready to run\n */\n async listReady(\n options: { limit?: number; queues?: string[] } = {},\n ): Promise<SmrtJob[]> {\n const now = new Date().toISOString();\n const whereConditions: string[] = [\"status = 'pending'\", 'run_at <= ?'];\n const params: unknown[] = [now];\n\n if (options.queues?.length) {\n const placeholders = options.queues.map(() => '?').join(', ');\n whereConditions.push(`queue IN (${placeholders})`);\n params.push(...options.queues);\n }\n\n params.push(options.limit || 100);\n\n // Worker-internal scan: the runner intentionally processes ready jobs\n // across all tenants, so it manages tenant context per-job at execution\n // time rather than filtering here (SmrtJob is now @TenantScoped, S5 #1402).\n return this.query(\n `SELECT * FROM _smrt_jobs WHERE ${whereConditions.join(' AND ')} ORDER BY priority DESC, run_at ASC LIMIT ?`,\n params,\n { allowRawOnTenantScoped: true },\n );\n }\n\n /**\n * Atomically claim pending jobs ready to run for a worker.\n *\n * The claim is performed as one conditional UPDATE so concurrent workers\n * cannot receive the same pending row. PostgreSQL additionally skips rows\n * locked by other workers instead of waiting behind them.\n */\n async claimReady(options: ClaimReadyOptions): Promise<SmrtJob[]> {\n const limit = options.limit ?? 100;\n if (limit <= 0) return [];\n\n const now = options.now ?? new Date();\n const nowIso = now.toISOString();\n const whereConditions: string[] = [\"status = 'pending'\", 'run_at <= ?'];\n const whereParams: unknown[] = [nowIso];\n\n if (options.queues?.length) {\n const placeholders = options.queues.map(() => '?').join(', ');\n whereConditions.push(`queue IN (${placeholders})`);\n whereParams.push(...options.queues);\n }\n\n const lockClause =\n getDatabaseEngine(this.db) === 'postgres'\n ? ' FOR UPDATE SKIP LOCKED'\n : '';\n const candidateSelect = `\n SELECT id\n FROM _smrt_jobs\n WHERE ${whereConditions.join(' AND ')}\n ORDER BY priority DESC, run_at ASC, created_at ASC, id ASC\n LIMIT ?${lockClause}\n `;\n\n const claimed = await this.query(\n `UPDATE _smrt_jobs\n SET status = 'running',\n worker_id = ?,\n worker_heartbeat = ?,\n started_at = ?,\n attempts = attempts + 1,\n updated_at = ?\n WHERE id IN (${candidateSelect})\n AND status = 'pending'\n RETURNING *`,\n [options.workerId, nowIso, nowIso, nowIso, ...whereParams, limit],\n // Worker-internal cross-tenant claim; tenant context is restored\n // per-job at execution (SmrtJob is now @TenantScoped, S5 #1402).\n { allowRawOnTenantScoped: true },\n );\n\n return claimed.toSorted(compareClaimOrder);\n }\n\n /**\n * Count non-terminal (pending/running) jobs owned by a tenant.\n *\n * Used to enforce the per-tenant creation cap so one tenant cannot exhaust\n * the shared worker pool (S5 audit #1402). Reads `_smrt_jobs` directly so it\n * works regardless of ambient tenant context.\n *\n * @param tenantId - Tenant to count for. `null` counts global (NULL-tenant)\n * jobs.\n */\n async countInFlightForTenant(tenantId: string | null): Promise<number> {\n const predicate = tenantId === null ? 'tenant_id IS NULL' : 'tenant_id = ?';\n const params = tenantId === null ? [] : [tenantId];\n\n // Use the public `db` accessor (with its init guard), not the protected\n // `_db` internal — consistent with claimReady()/this.db usage above. This\n // is a deliberately cross-tenant count (it must see every tenant's rows to\n // bound a single tenant), so it intentionally bypasses the tenant-scoped\n // query interceptor rather than routing through this.query().\n const result = await this.db.query(\n `SELECT COUNT(*) AS count\n FROM _smrt_jobs\n WHERE status IN ('pending', 'running')\n AND ${predicate}`,\n ...params,\n );\n\n const row = result.rows[0] as { count?: number | string } | undefined;\n return Number(row?.count ?? 0);\n }\n\n /**\n * The single creation path for queued jobs.\n *\n * Centralizes the two creation-time security guards from the S5 audit (#1402)\n * so every enqueue — the fluent {@link \"./job-builder\".JobBuilder} *and* the\n * ScheduleRunner's cron-triggered jobs — goes through one place:\n *\n * 1. `maxAttempts` is clamped to {@link MAX_JOB_RETRIES} so a misconfigured\n * caller cannot pin a worker on a poison job indefinitely.\n * 2. A per-tenant in-flight cap bounds how many non-terminal jobs one tenant\n * may hold, so one tenant cannot exhaust the shared worker pool\n * (cross-tenant denial of service). The cap applies to the row's effective\n * tenant (explicit `data.tenantId` or, when absent, the ambient context);\n * global (null-tenant) jobs are exempt.\n *\n * Atomicity note (best-effort soft cap, by design): the cap is a\n * count-then-insert, NOT a hard transactional invariant. It is intentionally\n * left non-atomic. A plain transaction would not help — under the adapters'\n * default isolation two concurrent same-tenant enqueues would each read the\n * same COUNT and both insert, so serializing them would require either a\n * per-tenant lock row (`SELECT ... FOR UPDATE`) or SERIALIZABLE-isolation\n * retry loops. That cross-process locking is fragile (lock-row contention,\n * adapter-specific isolation behavior, the `transaction` adapter method being\n * optional) and out of proportion to the threat: this cap is defense in depth\n * against runaway/accidental creation exhausting the shared worker pool, not a\n * billing/quota boundary. So under truly simultaneous enqueues a tenant may\n * momentarily overshoot by the number of in-flight creators; the bound still\n * prevents unbounded growth and closes the prior ScheduleRunner bypass. If a\n * hard guarantee is ever needed, enforce it with a DB CHECK/trigger or a\n * dedicated counter row, not an application-level lock.\n */\n async enqueueJob(\n data: SmrtJobData,\n options: EnqueueJobOptions = {},\n ): Promise<SmrtJob> {\n const cap = options.tenantJobCap ?? DEFAULT_TENANT_JOB_CAP;\n\n // Effective tenant: an explicitly provided tenantId wins (ScheduleRunner\n // passes the schedule's tenant even when no ambient context exists);\n // otherwise fall back to the ambient context (JobBuilder path).\n const explicitTenant =\n typeof data.tenantId === 'string' && data.tenantId.length > 0\n ? data.tenantId\n : data.tenantId === null\n ? null\n : undefined;\n const effectiveTenant =\n explicitTenant !== undefined ? explicitTenant : (getTenantId() ?? null);\n\n if (effectiveTenant && cap > 0) {\n const current = await this.countInFlightForTenant(effectiveTenant);\n assertWithinTenantCreationCap(effectiveTenant, current, cap);\n }\n\n const job = await this.create({\n ...data,\n // Clamp here so neither the builder nor the schedule runner can bypass the\n // retry ceiling (S5 audit #1402).\n maxAttempts: clampRetries(data.maxAttempts ?? 3),\n });\n await job.save();\n return job;\n }\n\n /**\n * Get job statistics\n */\n async stats(queue?: string): Promise<{\n pending: number;\n running: number;\n completed: number;\n failed: number;\n cancelled: number;\n }> {\n const query = queue\n ? 'SELECT status, COUNT(*) as count FROM _smrt_jobs WHERE queue = ? GROUP BY status'\n : 'SELECT status, COUNT(*) as count FROM _smrt_jobs GROUP BY status';\n const params = queue ? [queue] : [];\n\n const result = await this._db.query(query, ...params);\n\n const counts: Record<string, number> = {};\n for (const row of result.rows) {\n counts[row.status as string] = row.count as number;\n }\n\n return {\n pending: counts.pending ?? 0,\n running: counts.running ?? 0,\n completed: counts.completed ?? 0,\n failed: counts.failed ?? 0,\n cancelled: counts.cancelled ?? 0,\n };\n }\n\n /**\n * Cleanup old completed/failed jobs\n */\n async cleanup(options: {\n completedBefore?: Date;\n failedBefore?: Date;\n cancelledBefore?: Date;\n limit?: number;\n }): Promise<number> {\n const conditions: string[] = [];\n const params: unknown[] = [];\n\n if (options.completedBefore) {\n conditions.push(\"(status = 'completed' AND completed_at < ?)\");\n params.push(options.completedBefore.toISOString());\n }\n\n if (options.failedBefore) {\n conditions.push(\"(status = 'failed' AND completed_at < ?)\");\n params.push(options.failedBefore.toISOString());\n }\n\n if (options.cancelledBefore) {\n conditions.push(\"(status = 'cancelled' AND completed_at < ?)\");\n params.push(options.cancelledBefore.toISOString());\n }\n\n if (conditions.length === 0) return 0;\n\n let query = `DELETE FROM _smrt_jobs WHERE (${conditions.join(' OR ')})`;\n\n if (options.limit) {\n query = `\n DELETE FROM _smrt_jobs\n WHERE id IN (\n SELECT id FROM _smrt_jobs\n WHERE (${conditions.join(' OR ')})\n LIMIT ?\n )\n `;\n params.push(options.limit);\n }\n\n const result = await this._db.query(query, ...params);\n return result.rowCount ?? 0;\n }\n}\n\nfunction getDatabaseEngine(\n db: DatabaseInterface,\n): ReturnType<typeof detectEngine> {\n const dbWithConfig = db as DatabaseWithConfig;\n return detectEngine(\n db.url || dbWithConfig.config?.url || '',\n dbWithConfig.type || dbWithConfig.config?.type,\n );\n}\n\nfunction compareClaimOrder(left: SmrtJob, right: SmrtJob): number {\n const priority = right.priority - left.priority;\n if (priority !== 0) return priority;\n\n const runAt = left.runAt.getTime() - right.runAt.getTime();\n if (runAt !== 0) return runAt;\n\n const createdAt = timestamp(left.created_at) - timestamp(right.created_at);\n if (createdAt !== 0) return createdAt;\n\n return (left.id ?? '').localeCompare(right.id ?? '');\n}\n\nfunction timestamp(value: Date | null | undefined): number {\n return value?.getTime() ?? 0;\n}\n\nexport default SmrtJob;\n","// Self-register this package's manifest for consumers that import via this\n// subpath without the main entry. See src/__smrt-register__.ts (issue #1132).\nimport './__smrt-register__.js';\n\nimport {\n ensureJobEventsSystemTableCompatibility,\n field,\n foreignKey,\n SmrtCollection,\n SmrtObject,\n smrt,\n} from '@happyvertical/smrt-core';\nimport {\n getTenantId,\n TenantScoped,\n tenantId,\n} from '@happyvertical/smrt-tenancy';\n\nexport type SmrtJobEventType = 'status' | 'progress' | 'log' | 'error' | string;\n\nexport type SmrtJobEventLevel = 'debug' | 'info' | 'warn' | 'error';\n\nexport interface SmrtJobEventData {\n tenantId?: string | null;\n jobId: string;\n type?: SmrtJobEventType;\n level?: SmrtJobEventLevel;\n stage?: string | null;\n progress?: number | null;\n message?: string;\n data?: Record<string, unknown>;\n createdAt?: Date;\n}\n\nexport interface JobEventCursor {\n createdAt: string | Date;\n id: string;\n}\n\nexport interface ListJobEventsOptions {\n tenantId?: string | null;\n limit?: number;\n since?: string | Date;\n afterId?: string;\n cursor?: string | JobEventCursor;\n}\n\nconst JOB_EVENT_STORAGE_COLUMNS = [\n 'id',\n 'slug',\n 'context',\n 'created_at',\n 'updated_at',\n 'tenant_id',\n 'job_id',\n 'type',\n 'level',\n 'stage',\n 'progress',\n 'message',\n 'data',\n].join(', ');\n\n@smrt({\n tableName: '_smrt_job_events',\n // Fail closed: same reasoning as SmrtJob. `_smrt_job_events` carries job\n // progress/log/error payloads for every tenant; an `optional`-mode generated\n // read reached without tenant context returns UNFILTERED rows. Consumers read\n // events through the collection's tenant-aware methods (listByJob /\n // listSinceCursor, which require an explicit tenantId or ambient context), not\n // through generated routes — so we do not generate a read surface here\n // (S5 audit #1402).\n api: false,\n // In-process operator commands only (http: false). skipApiCheck acknowledges\n // that these CLI reads intentionally have no HTTP/API route now that api is\n // disabled (S5 audit #1402).\n cli: { include: ['list', 'get'], http: false, skipApiCheck: true },\n mcp: false,\n})\n// Keep the data model tenant-scoped (defense in depth); the @tenantId() field\n// alone does not make collection reads filter by tenant. `optional` keeps global\n// (NULL tenant) events working (S5 audit #1402).\n@TenantScoped({ mode: 'optional' })\nexport class SmrtJobEvent extends SmrtObject {\n @tenantId({ nullable: true })\n tenantId: string | null | undefined = undefined;\n\n @foreignKey('SmrtJob', { required: true })\n jobId: string = '';\n\n @field({ type: 'text', required: true, default: 'log' })\n type: SmrtJobEventType = 'log';\n\n @field({ type: 'text', required: true, default: 'info' })\n level: SmrtJobEventLevel = 'info';\n\n @field({ type: 'text', nullable: true })\n stage: string | null = null;\n\n @field({ type: 'integer', nullable: true })\n progress: number | null = null;\n\n @field({ type: 'text', required: true, default: '' })\n message: string = '';\n\n @field({ type: 'json' })\n data: Record<string, unknown> = {};\n\n @field({ type: 'datetime', required: true })\n createdAt: Date = new Date();\n\n toCursor(): string {\n const createdAt =\n this.createdAt instanceof Date\n ? this.createdAt.toISOString()\n : String(this.createdAt);\n return `${createdAt}|${this.id ?? ''}`;\n }\n}\n\nfunction normalizeLimit(limit: number | undefined): number {\n const numeric =\n typeof limit === 'number' && Number.isFinite(limit) ? limit : 250;\n return Math.max(1, Math.min(1000, Math.floor(numeric)));\n}\n\nfunction normalizeProgress(progress: unknown): number | null {\n if (typeof progress !== 'number' || !Number.isFinite(progress)) {\n return null;\n }\n\n return Math.max(0, Math.min(100, Math.round(progress)));\n}\n\nfunction parseCursor(cursor: string | JobEventCursor): JobEventCursor {\n if (typeof cursor !== 'string') return cursor;\n const separator = cursor.lastIndexOf('|');\n if (separator === -1) {\n return { createdAt: cursor, id: '' };\n }\n return {\n createdAt: cursor.slice(0, separator),\n id: cursor.slice(separator + 1),\n };\n}\n\nfunction normalizeCursorDate(value: string | Date): string {\n if (value instanceof Date) {\n return value.toISOString();\n }\n\n const parsed = new Date(value);\n if (!Number.isNaN(parsed.getTime())) {\n return parsed.toISOString();\n }\n\n return value;\n}\n\nfunction usesSqliteDateFunctions(dbUrl: string): boolean {\n const normalized = dbUrl.toLowerCase();\n return !(\n normalized.startsWith('postgres:') || normalized.startsWith('postgresql:')\n );\n}\n\nfunction getQueryRows(result: unknown): Record<string, unknown>[] {\n if (Array.isArray(result)) {\n return result as Record<string, unknown>[];\n }\n\n return (result as { rows?: Record<string, unknown>[] }).rows ?? [];\n}\n\nexport class SmrtJobEventCollection extends SmrtCollection<SmrtJobEvent> {\n static readonly _itemClass = SmrtJobEvent;\n\n override async initialize(): Promise<this> {\n await super.initialize();\n await ensureJobEventsSystemTableCompatibility(this.db);\n return this;\n }\n\n async append(input: SmrtJobEventData): Promise<SmrtJobEvent> {\n return this.create({\n tenantId: input.tenantId,\n jobId: input.jobId,\n type: input.type ?? 'log',\n level: input.level ?? 'info',\n stage: input.stage ?? null,\n progress: normalizeProgress(input.progress),\n message: input.message ?? '',\n data: input.data ?? {},\n createdAt: input.createdAt ?? new Date(),\n });\n }\n\n async listByJob(\n jobId: string,\n options: ListJobEventsOptions = {},\n ): Promise<SmrtJobEvent[]> {\n return this.listSinceCursor({\n ...options,\n jobId,\n });\n }\n\n async listSinceCursor(\n options: ListJobEventsOptions & { jobId?: string } = {},\n ): Promise<SmrtJobEvent[]> {\n const where: string[] = [];\n const params: unknown[] = [];\n\n if (options.jobId) {\n where.push('job_id = ?');\n params.push(options.jobId);\n }\n\n this.addTenantPredicate(where, params, options);\n\n if (options.cursor) {\n const cursor = parseCursor(options.cursor);\n const createdAt = await this.resolveCursorCreatedAt(cursor, options);\n const createdAtExpression = this.createdAtComparableExpression();\n where.push(\n `(${createdAtExpression} > ? OR (${createdAtExpression} = ? AND id > ?))`,\n );\n params.push(createdAt, createdAt, cursor.id);\n } else if (options.since) {\n where.push(`${this.createdAtComparableExpression()} > ?`);\n params.push(normalizeCursorDate(options.since));\n }\n\n if (options.afterId) {\n where.push('id > ?');\n params.push(options.afterId);\n }\n\n params.push(normalizeLimit(options.limit));\n\n const whereSql = where.length ? `WHERE ${where.join(' AND ')}` : '';\n return this.query(\n `SELECT ${JOB_EVENT_STORAGE_COLUMNS}\n FROM _smrt_job_events\n ${whereSql}\n ORDER BY ${this.createdAtComparableExpression()} ASC, id ASC\n LIMIT ?`,\n params,\n { allowRawOnTenantScoped: true },\n );\n }\n\n async latestProgressByJobIds(\n jobIds: string[],\n options: { tenantId?: string | null } = {},\n ): Promise<Map<string, SmrtJobEvent>> {\n const uniqueJobIds = [...new Set(jobIds.filter(Boolean))];\n const latestByJobId = new Map<string, SmrtJobEvent>();\n if (uniqueJobIds.length === 0) return latestByJobId;\n\n const placeholders = uniqueJobIds.map(() => '?').join(', ');\n const where: string[] = [\n `job_id IN (${placeholders})`,\n \"type = 'progress'\",\n ];\n const params: unknown[] = [...uniqueJobIds];\n\n this.addTenantPredicate(where, params, options);\n const createdAtExpression = this.createdAtComparableExpression();\n\n const events = await this.query(\n `SELECT ${JOB_EVENT_STORAGE_COLUMNS}\n FROM (\n SELECT ${JOB_EVENT_STORAGE_COLUMNS},\n ${createdAtExpression} AS smrt_created_at_sort,\n ROW_NUMBER() OVER (\n PARTITION BY job_id\n ORDER BY ${createdAtExpression} DESC, id DESC\n ) AS smrt_rank\n FROM _smrt_job_events\n WHERE ${where.join(' AND ')}\n ) ranked\n WHERE smrt_rank = 1\n ORDER BY smrt_created_at_sort DESC, id DESC`,\n params,\n { allowRawOnTenantScoped: true },\n );\n\n for (const event of events) {\n latestByJobId.set(event.jobId, event);\n }\n\n return latestByJobId;\n }\n\n private addTenantPredicate(\n where: string[],\n params: unknown[],\n options: { tenantId?: string | null },\n ): void {\n if (options.tenantId === null) {\n where.push('tenant_id IS NULL');\n return;\n }\n\n const tenantId =\n typeof options.tenantId === 'string' ? options.tenantId : getTenantId();\n\n if (tenantId) {\n where.push('tenant_id = ?');\n params.push(tenantId);\n return;\n }\n\n throw new Error(\n 'Tenant-scoped job event queries require tenantId, tenantId: null, or an ambient tenant context.',\n );\n }\n\n private createdAtComparableExpression(): string {\n if (usesSqliteDateFunctions(this.db.url)) {\n return \"strftime('%Y-%m-%dT%H:%M:%fZ', created_at)\";\n }\n\n return 'created_at';\n }\n\n private async resolveCursorCreatedAt(\n cursor: JobEventCursor,\n options: ListJobEventsOptions & { jobId?: string },\n ): Promise<string> {\n if (!cursor.id) {\n return normalizeCursorDate(cursor.createdAt);\n }\n\n const where = ['id = ?'];\n const params: unknown[] = [cursor.id];\n if (options.jobId) {\n where.push('job_id = ?');\n params.push(options.jobId);\n }\n this.addTenantPredicate(where, params, options);\n\n const result = await this.db.query(\n `SELECT ${this.createdAtComparableExpression()} AS cursor_created_at\n FROM _smrt_job_events\n WHERE ${where.join(' AND ')}\n LIMIT 1`,\n ...params,\n );\n const cursorCreatedAt = getQueryRows(result)[0]?.cursor_created_at;\n\n return typeof cursorCreatedAt === 'string' && cursorCreatedAt.trim()\n ? cursorCreatedAt\n : normalizeCursorDate(cursor.createdAt);\n }\n}\n\nexport default SmrtJobEvent;\n","// Self-register this package's manifest for consumers that import via this\n// subpath without the main entry. See src/__smrt-register__.ts (issue #1132).\nimport './__smrt-register__.js';\n\nimport {\n field,\n SmrtCollection,\n SmrtObject,\n smrt,\n} from '@happyvertical/smrt-core';\n\n/**\n * Liveness record for a single TaskRunner / ScheduleRunner incarnation,\n * stored in the `_smrt_workers` system table.\n *\n * @remarks\n * Job recovery asks \"is this job's owning worker alive?\" rather than \"is this\n * job's heartbeat fresh?\" (issue #1474). Each running worker keeps a row here\n * and renews `leaseExpiresAt` on a fixed cadence; a worker that dies stops\n * renewing and its lease expires, so its `running` jobs are recovered.\n *\n * `workerId` is unique per *incarnation* (a restarted runner gets a new key —\n * see `createWorkerKey`), which is what lets recovery distinguish a crashed\n * worker's orphaned jobs from an identically-configured restart.\n *\n * `leaseExpiresAt` is a `datetime` so it maps to a real timestamp column on\n * every engine (an integer epoch-ms column overflows `int4`/`INT32` on\n * Postgres and DuckDB). Stage 1 writes/compares it against the host clock —\n * the same approach the previous heartbeat recovery used; Stage 2 will move to\n * database-side time once an off-loop writer exists.\n */\n@smrt({\n tableName: '_smrt_workers',\n conflictColumns: ['worker_id'],\n api: false,\n cli: false,\n mcp: false,\n})\nexport class SmrtWorker extends SmrtObject {\n /** Per-incarnation-unique worker key (also stored on owned jobs' workerId). */\n @field({ type: 'text', required: true })\n workerId: string = '';\n\n /** OS process id of the owning runner (diagnostic). */\n @field({ type: 'integer', nullable: true })\n pid: number | null = null;\n\n /** Hostname of the owning runner (diagnostic). */\n @field({ type: 'text', nullable: true })\n hostname: string | null = null;\n\n /** When this incarnation started. */\n @field({ type: 'datetime', nullable: true })\n startedAt: Date | null = null;\n\n /** Last lease renewal time (diagnostic; liveness uses leaseExpiresAt). */\n @field({ type: 'datetime', nullable: true })\n heartbeatAt: Date | null = null;\n\n /** Lease expiry — the worker is alive while this is in the future. */\n @field({ type: 'datetime', nullable: true })\n leaseExpiresAt: Date | null = null;\n\n /** Lifecycle status (`running` while the runner is processing). */\n @field({ type: 'text', required: true, default: 'running' })\n status: string = 'running';\n}\n\nexport interface RegisterWorkerInput {\n workerKey: string;\n pid?: number | null;\n hostname?: string | null;\n leaseTtlMs: number;\n}\n\n/**\n * Collection for managing `_smrt_workers` liveness rows.\n */\nexport class SmrtWorkerCollection extends SmrtCollection<SmrtWorker> {\n static readonly _itemClass = SmrtWorker;\n\n /**\n * Fail fast if the `_smrt_workers` table has not been migrated.\n *\n * The framework never creates application/system tables at runtime; the\n * table is created by `smrt db:migrate` (or `getTestDatabase`). A consumer\n * that upgrades smrt-jobs without migrating must get a clear, actionable\n * error at `start()` rather than a confusing recovery failure later.\n */\n async assertReady(): Promise<void> {\n try {\n await this.db.query('SELECT 1 FROM _smrt_workers LIMIT 1');\n } catch (error) {\n throw new Error(\n 'The _smrt_workers table is missing. Run `smrt db:migrate` to create ' +\n 'job-system tables before starting a TaskRunner/ScheduleRunner. ' +\n `(underlying error: ${(error as Error).message})`,\n );\n }\n }\n\n /** Whether the `_smrt_workers` table exists (recovery skips lease checks if not). */\n async tableReady(): Promise<boolean> {\n try {\n await this.db.query('SELECT 1 FROM _smrt_workers LIMIT 1');\n return true;\n } catch {\n return false;\n }\n }\n\n /** Register a worker incarnation with its lease seeded to `now + ttl`. */\n async registerWorker(input: RegisterWorkerInput): Promise<void> {\n const now = new Date();\n await this.create({\n workerId: input.workerKey,\n pid: input.pid ?? null,\n hostname: input.hostname ?? null,\n startedAt: now,\n heartbeatAt: now,\n // Seed the lease in the same write so the worker is immediately \"alive\",\n // closing the window between registration and the first claimReady().\n leaseExpiresAt: new Date(now.getTime() + input.leaseTtlMs),\n status: 'running',\n });\n }\n\n /** Renew a worker's lease to `now + ttl`. */\n async renewLease(workerKey: string, leaseTtlMs: number): Promise<void> {\n const now = new Date();\n await this.db.query(\n `UPDATE _smrt_workers\n SET lease_expires_at = ?,\n heartbeat_at = ?\n WHERE worker_id = ?`,\n new Date(now.getTime() + leaseTtlMs).toISOString(),\n now.toISOString(),\n workerKey,\n );\n }\n\n /** Remove a worker incarnation (graceful shutdown). */\n async expireWorker(workerKey: string): Promise<void> {\n await this.db.query(\n 'DELETE FROM _smrt_workers WHERE worker_id = ?',\n workerKey,\n );\n }\n\n /** Worker keys whose database lease is still fresh (alive cross-process). */\n async freshLeaseWorkerKeys(): Promise<Set<string>> {\n const result = await this.db.query(\n `SELECT worker_id\n FROM _smrt_workers\n WHERE lease_expires_at IS NOT NULL\n AND lease_expires_at >= ?`,\n new Date().toISOString(),\n );\n const keys = new Set<string>();\n for (const row of result.rows as Array<{ worker_id?: unknown }>) {\n if (typeof row.worker_id === 'string') keys.add(row.worker_id);\n }\n return keys;\n }\n\n /** Delete worker rows whose lease expired more than `graceMs` ago. */\n async pruneExpired(graceMs: number): Promise<void> {\n const cutoff = new Date(Date.now() - Math.max(0, graceMs)).toISOString();\n await this.db.query(\n `DELETE FROM _smrt_workers\n WHERE lease_expires_at IS NOT NULL\n AND lease_expires_at < ?`,\n cutoff,\n );\n }\n}\n\nexport default SmrtWorker;\n","export const DEFAULT_TASK_HEARTBEAT_INTERVAL_MS = 30000;\nconst STALE_HEARTBEAT_GRACE_MULTIPLIER = 3;\n\n/**\n * Default cadence for renewing a worker's liveness lease.\n *\n * Liveness is a per-*worker* lease (see {@link ../smrt-worker.js}), not a\n * per-job heartbeat. The runner renews its lease on this interval; a dead\n * worker stops renewing and its lease expires after {@link DEFAULT_LEASE_TTL_MS}.\n */\nexport const DEFAULT_LEASE_TICK_MS = 10000;\n\n/**\n * Default time-to-live for a worker liveness lease.\n *\n * A `running` job is only recovered when its owning worker is neither live in\n * this process nor holding a fresh lease in the database. The TTL is the\n * cross-process detection latency for a genuinely dead worker.\n */\nexport const DEFAULT_LEASE_TTL_MS = 30000;\n\nconst LEASE_TTL_GRACE_MULTIPLIER = 3;\n\n/**\n * Keep stale-job recovery aligned with the actual heartbeat cadence.\n *\n * @deprecated Recovery no longer keys on per-job heartbeat staleness (#1474);\n * it keys on worker liveness. Retained for one release for any external caller.\n * Use {@link getEffectiveLeaseTtlMs} with the lease tick instead.\n */\nexport function getEffectiveStaleJobThresholdMs(\n staleJobThresholdMs: number,\n heartbeatIntervalMs: number,\n): number {\n return Math.max(\n staleJobThresholdMs,\n heartbeatIntervalMs * STALE_HEARTBEAT_GRACE_MULTIPLIER,\n );\n}\n\n/**\n * Never let a lease expire in fewer than three renewal ticks.\n *\n * A single missed renewal (GC pause, a slow database round-trip) must not be\n * enough to declare a healthy worker dead. Mirrors the floor applied by\n * {@link getEffectiveStaleJobThresholdMs} for the legacy heartbeat path.\n */\nexport function getEffectiveLeaseTtlMs(\n leaseTtlMs: number,\n leaseTickMs: number,\n): number {\n return Math.max(leaseTtlMs, leaseTickMs * LEASE_TTL_GRACE_MULTIPLIER);\n}\n","// Self-register this package's manifest for consumers that import via this\n// subpath without the main entry. See src/__smrt-register__.ts (issue #1132).\nimport './__smrt-register__.js';\n\nimport { EventEmitter } from 'node:events';\nimport { Worker } from 'node:worker_threads';\nimport { fromConfig, type RetryDecision } from '@happyvertical/jobs';\nimport { createLogger } from '@happyvertical/logger';\nimport {\n getClassConfigResolvers,\n ObjectRegistry,\n resolveLazyConfig,\n type SmrtObject,\n} from '@happyvertical/smrt-core';\nimport { TenantContext } from '@happyvertical/smrt-tenancy';\nimport type { DatabaseInterface } from '@happyvertical/sql';\nimport { createId } from '@happyvertical/utils';\nimport { isBackgroundEligibleMethod } from './background-policy.js';\nimport { redactErrorForPersistence } from './error-redaction.js';\nimport {\n JobContextLogger,\n type JobEventInput,\n type JobExecutionContext,\n type JobProgressInput,\n} from './logger-extension.js';\nimport { type SmrtJob, SmrtJobCollection } from './smrt-job.js';\nimport { type SmrtJobEvent, SmrtJobEventCollection } from './smrt-job-event.js';\nimport { SmrtWorkerCollection } from './smrt-worker.js';\nimport {\n DEFAULT_LEASE_TICK_MS,\n DEFAULT_LEASE_TTL_MS,\n DEFAULT_TASK_HEARTBEAT_INTERVAL_MS,\n getEffectiveLeaseTtlMs,\n} from './stale-recovery.js';\nimport {\n createWorkerKey,\n isWorkerAlive,\n offLoopEligible,\n registerLiveWorker,\n resolveEngine,\n resolveUrl,\n tuneSqliteForConcurrency,\n unregisterLiveWorker,\n} from './worker-liveness.js';\n\n/**\n * TaskRunner configuration\n */\nexport interface TaskRunnerConfig {\n /** Worker ID (auto-generated if not provided) */\n id?: string;\n /** Number of concurrent jobs to process */\n concurrency?: number;\n /** Queues to process (default: ['default']) */\n queues?: string[];\n /** Polling interval in milliseconds */\n pollInterval?: number;\n /** Heartbeat interval in milliseconds */\n heartbeatInterval?: number;\n /** Maximum time to wait for jobs to complete on shutdown */\n shutdownTimeout?: number;\n /**\n * @deprecated No longer used. Recovery keys on worker liveness, not per-job\n * heartbeat staleness (#1474). Use {@link leaseTtlMs} / {@link leaseTickMs}.\n */\n staleJobThresholdMs?: number;\n /** Worker liveness lease time-to-live in milliseconds */\n leaseTtlMs?: number;\n /** How often to renew the worker liveness lease, in milliseconds */\n leaseTickMs?: number;\n}\n\n/**\n * TaskRunner events\n */\nexport interface TaskRunnerEvents {\n 'job:started': (job: SmrtJob) => void;\n 'job:event': (job: SmrtJob, event: SmrtJobEvent) => void;\n 'job:progress': (job: SmrtJob, event: SmrtJobEvent) => void;\n 'job:completed': (job: SmrtJob, result: unknown) => void;\n 'job:failed': (job: SmrtJob, error: Error) => void;\n 'job:retrying': (job: SmrtJob, error: Error, delay: number) => void;\n 'runner:started': () => void;\n 'runner:stopped': () => void;\n 'runner:error': (error: Error) => void;\n}\n\n/**\n * Max time to wait for the liveness thread to report ready before giving up and\n * falling back to main-loop renewal (guards against a hung connect in-thread).\n */\nconst LIVENESS_THREAD_START_TIMEOUT_MS = 10000;\n\n/**\n * Default configuration\n */\nconst DEFAULT_CONFIG: Required<TaskRunnerConfig> = {\n id: '',\n concurrency: 5,\n queues: ['default'],\n pollInterval: 1000,\n heartbeatInterval: DEFAULT_TASK_HEARTBEAT_INTERVAL_MS,\n shutdownTimeout: 30000,\n staleJobThresholdMs: 90000,\n leaseTtlMs: DEFAULT_LEASE_TTL_MS,\n leaseTickMs: DEFAULT_LEASE_TICK_MS,\n};\n\n/**\n * TaskRunner processes SMRT jobs by invoking methods on SmrtObjects\n *\n * Features:\n * - Executes jobs via SmrtObject method invocation\n * - Configurable concurrency and timeout behavior\n * - Automatic retry with configurable strategies\n * - Job context logging for visibility\n * - Embedded mode (in-process) or standalone (CLI)\n */\nexport class TaskRunner extends EventEmitter {\n readonly id: string;\n /**\n * Per-incarnation-unique worker key. Stored as the `worker_id` on claimed\n * jobs and in `_smrt_workers`, so a restart of a runner sharing the same\n * configured `id` does not look like it still owns the previous\n * incarnation's orphaned jobs. The human-facing {@link id} stays stable for\n * events/logs.\n */\n private readonly workerKey: string;\n private readonly config: Required<TaskRunnerConfig>;\n private readonly effectiveLeaseTtlMs: number;\n private collection: SmrtJobCollection | null = null;\n private eventCollection: SmrtJobEventCollection | null = null;\n private workerCollection: SmrtWorkerCollection | null = null;\n private workersTableVerified = false;\n private lastRecoverySweepAt = 0;\n private running = false;\n private activeJobs = new Map<string, SmrtJob>();\n private pollTimer: NodeJS.Timeout | null = null;\n private heartbeatTimer: NodeJS.Timeout | null = null;\n private leaseTimer: NodeJS.Timeout | null = null;\n private livenessWorker: Worker | null = null;\n private shutdownPromise: Promise<void> | null = null;\n private db: DatabaseInterface | null = null;\n private logger = createLogger(true);\n\n constructor(config: TaskRunnerConfig = {}) {\n super();\n this.config = {\n ...DEFAULT_CONFIG,\n ...config,\n id: config.id || `runner_${createId().slice(0, 8)}`,\n };\n this.id = this.config.id;\n this.workerKey = createWorkerKey(this.id);\n this.effectiveLeaseTtlMs = getEffectiveLeaseTtlMs(\n this.config.leaseTtlMs,\n this.config.leaseTickMs,\n );\n }\n\n /**\n * Initialize the runner with database connection\n */\n async initialize(db: DatabaseInterface): Promise<void> {\n this.db = db;\n this.collection = await SmrtJobCollection.create({ db });\n this.eventCollection = await SmrtJobEventCollection.create({ db });\n this.workerCollection = await SmrtWorkerCollection.create({ db });\n }\n\n /**\n * Start processing jobs\n */\n async start(): Promise<void> {\n if (this.running) return;\n if (!this.collection || !this.workerCollection) {\n throw new Error('TaskRunner not initialized. Call initialize() first.');\n }\n\n // Fail fast if the job-system tables were never migrated, with an\n // actionable error rather than a confusing recovery failure later.\n await this.workerCollection.assertReady();\n\n // On SQLite, the runner's writes will race the off-loop ticker's writes to\n // the same file; enable WAL + a busy timeout up front so neither loses a\n // lock race under load (#1474 flake). Best-effort, file-level.\n if (this.db && resolveEngine(this.db) === 'sqlite') {\n await tuneSqliteForConcurrency(this.db);\n }\n\n // Register this worker incarnation with a seeded lease BEFORE polling, so\n // the first claimReady() cannot leave just-claimed jobs looking orphaned to\n // a concurrent recoverer.\n await this.workerCollection.registerWorker({\n workerKey: this.workerKey,\n pid: typeof process !== 'undefined' ? process.pid : null,\n hostname:\n typeof process !== 'undefined' ? (process.env.HOSTNAME ?? null) : null,\n leaseTtlMs: this.effectiveLeaseTtlMs,\n });\n registerLiveWorker(this.workerKey);\n\n this.running = true;\n\n // Renew the worker lease. Prefer an off-loop worker thread so a CPU-bound\n // synchronous handler can never starve renewal (#1474); fall back to\n // main-loop renewal for engines a second connection can't reach\n // (in-memory SQLite, DuckDB) or if the thread fails to start.\n if (offLoopEligible(this.db as DatabaseInterface)) {\n const threadStarted = await this.startLivenessThread();\n if (!threadStarted) this.startLeaseRenewal();\n } else {\n this.startLeaseRenewal();\n }\n\n // Start polling loop\n this.startPolling();\n\n // Start heartbeat loop (per-job telemetry only; no longer gates recovery)\n this.startHeartbeat();\n\n this.emit('runner:started');\n }\n\n /**\n * Stop processing jobs (graceful shutdown)\n */\n async stop(): Promise<void> {\n if (!this.running) return;\n if (this.shutdownPromise) return this.shutdownPromise;\n\n this.running = false;\n\n // Stop polling and the telemetry heartbeat immediately; no new jobs claim.\n if (this.pollTimer) {\n clearTimeout(this.pollTimer);\n this.pollTimer = null;\n }\n if (this.heartbeatTimer) {\n clearInterval(this.heartbeatTimer);\n this.heartbeatTimer = null;\n }\n // NOTE: leave lease renewal (the leaseTimer or the off-loop thread) running\n // through the drain so a still-executing handler keeps its lease fresh and\n // isn't recovered by a peer; both are torn down after the drain below.\n\n // Wait for active jobs to complete (with timeout)\n this.shutdownPromise = this.waitForActiveJobs();\n\n try {\n await this.shutdownPromise;\n } finally {\n this.shutdownPromise = null;\n // Stop off-loop lease renewal (thread) and main-loop renewal (timer) only\n // AFTER the drain, so a still-running handler kept its lease fresh.\n await this.stopLivenessThread();\n if (this.leaseTimer) {\n clearInterval(this.leaseTimer);\n this.leaseTimer = null;\n }\n // Release liveness only AFTER the drain: doing it earlier would make\n // still-draining jobs look orphaned to other recoverers.\n unregisterLiveWorker(this.workerKey);\n // Only delete the lease row if the drain finished cleanly. If jobs are\n // still executing past the shutdown timeout, leave the row to lapse\n // naturally (≤ TTL) so a nearly-finished handler keeps its chance to land\n // its own completion before peers can recover it.\n if (this.activeJobs.size === 0) {\n try {\n await this.workerCollection?.expireWorker(this.workerKey);\n } catch {\n // Best-effort: a left-over row simply expires via its lease.\n }\n }\n this.emit('runner:stopped');\n }\n }\n\n /**\n * Check if runner is running\n */\n isRunning(): boolean {\n return this.running;\n }\n\n /**\n * Get count of active jobs\n */\n activeJobCount(): number {\n return this.activeJobs.size;\n }\n\n /**\n * Start the polling loop\n */\n private startPolling(): void {\n const poll = async () => {\n if (!this.running) return;\n\n try {\n await this.poll();\n } catch (error) {\n this.emit('runner:error', error as Error);\n }\n\n // Schedule next poll\n if (this.running) {\n this.pollTimer = setTimeout(poll, this.config.pollInterval);\n }\n };\n\n // Start immediately\n poll();\n }\n\n /**\n * Poll for and process jobs\n */\n private async poll(): Promise<void> {\n if (!this.collection || !this.db) return;\n\n await this.recoverStaleJobs();\n\n // Calculate how many jobs we can take\n const available = this.config.concurrency - this.activeJobs.size;\n if (available <= 0) return;\n\n // Atomically claim ready jobs before processing so multiple workers cannot\n // receive the same pending row.\n const jobs = await this.collection.claimReady({\n workerId: this.workerKey,\n queues: this.config.queues,\n limit: available,\n });\n\n for (const job of jobs) {\n const jobId = job.id;\n if (!jobId) continue;\n\n // Process asynchronously\n this.processJob(job);\n }\n }\n\n /**\n * Process a single job\n */\n private async processJob(job: SmrtJob): Promise<void> {\n const jobId = job.id;\n if (!jobId) {\n this.emit('runner:error', new Error('Job has no ID'));\n return;\n }\n\n this.activeJobs.set(jobId, job);\n this.emit('job:started', job);\n await this.appendJobEvent(job, {\n type: 'status',\n level: 'info',\n stage: 'started',\n progress: 0,\n message: `Started job: ${job.getDescription()}`,\n });\n\n try {\n // Set up timeout\n const timeoutPromise = new Promise<never>((_, reject) => {\n setTimeout(() => {\n reject(new Error(`Job timeout after ${job.timeout}ms`));\n }, job.timeout);\n });\n\n // Execute the job with timeout\n const result = await Promise.race([this.executeJob(job), timeoutPromise]);\n\n // Job completed successfully. Write the terminal state conditionally so a\n // recovered/reclaimed row (worker died, recovery ran, the work finished\n // anyway) is never stomped back to 'completed'.\n const completedAt = new Date();\n const applied = await this.writeOwnedJob(jobId, {\n status: 'completed',\n completed_at: completedAt.toISOString(),\n result_pointer: result?.resultPointer ?? null,\n updated_at: completedAt.toISOString(),\n });\n\n if (applied) {\n job.status = 'completed';\n job.completedAt = completedAt;\n job.resultPointer = result?.resultPointer ?? null;\n await this.appendJobEvent(job, {\n type: 'progress',\n level: 'info',\n stage: 'completed',\n progress: 100,\n message: `Completed job: ${job.getDescription()}`,\n });\n this.emit('job:completed', job, result);\n }\n } catch (error) {\n await this.handleJobError(job, error as Error);\n } finally {\n this.activeJobs.delete(jobId);\n }\n }\n\n /**\n * Apply a terminal/retry state transition to a job only if this worker still\n * owns it and it is still `running`. Returns whether the write applied.\n *\n * This closes the completion-vs-recovery race: if recovery already failed a\n * job out from under a finishing handler (a genuine zombie), the handler's\n * outcome is dropped rather than resurrecting the row.\n */\n private async writeOwnedJob(\n jobId: string,\n assignments: Record<string, unknown>,\n ): Promise<boolean> {\n if (!this.db) return false;\n const columns = Object.keys(assignments);\n const setSql = columns.map((column) => `${column} = ?`).join(', ');\n const values = columns.map((column) => assignments[column]);\n // RETURNING id (not rowCount): the DuckDB/JSON adapters report rowCount as\n // the number of result rows, so an UPDATE that matched nothing still\n // reports 1 — only the returned-row set is a reliable \"did it apply\".\n const result = await this.db.query(\n `UPDATE _smrt_jobs\n SET ${setSql}\n WHERE id = ? AND worker_id = ? AND status = 'running'\n RETURNING id`,\n ...values,\n jobId,\n this.workerKey,\n );\n return (result.rows?.length ?? 0) > 0;\n }\n\n /**\n * Execute a job by invoking the method on the SmrtObject\n */\n private async executeJob(\n job: SmrtJob,\n ): Promise<{ result?: unknown; resultPointer?: string }> {\n const runJob = async (): Promise<{\n result?: unknown;\n resultPointer?: string;\n }> => {\n // Get the object class from registry\n const registeredClass = ObjectRegistry.getClass(job.objectType);\n if (!registeredClass) {\n throw new Error(`Unknown object type: ${job.objectType}`);\n }\n\n // Get the constructor from the registry entry\n const ObjectClass = registeredClass.constructor as unknown as new (\n options: Record<string, unknown>,\n ) => SmrtObject;\n\n // Extract internal keys from args before passing to constructor/method\n const rawArgs = (job.args ?? {}) as Record<string, unknown>;\n const persistedAgentConfig = (rawArgs._agentConfig ?? {}) as Record<\n string,\n unknown\n >;\n const { _agentConfig: _, _scheduleId: __, ...methodArgs } = rawArgs;\n\n // Resolve any lazy / env-derived config sentinels at execute time so\n // operators can rotate env vars without rewriting persisted schedule\n // rows (issue #1161). Class-level `static configResolvers` are layered\n // on top so live values always win over snapshotted ones.\n //\n // `onError: 'throw'` so a misconfigured deployment fails fast at the\n // job boundary with a clear \"unknown resolver X\" error, rather than\n // silently spreading a `{ $env: '...' }` sentinel object into the\n // agent constructor (where it would surface much later as a confusing\n // downstream failure — e.g. `[object Object]` masquerading as a\n // bucket name).\n const classResolvers = getClassConfigResolvers(ObjectClass);\n const agentConfig = await resolveLazyConfig(persistedAgentConfig, {\n classResolvers,\n onError: 'throw',\n });\n\n // Create or load the object instance\n let instance: SmrtObject;\n\n if (job.objectId) {\n // Load existing object\n instance = new ObjectClass({ db: this.db, ...agentConfig });\n await instance.initialize();\n await (\n instance as SmrtObject & { loadFromId(id: string): Promise<void> }\n ).loadFromId(job.objectId);\n } else {\n // Create new instance for static-like methods\n instance = new ObjectClass({ db: this.db, ...agentConfig });\n await instance.initialize();\n }\n\n const jobId = job.id;\n if (!jobId) {\n throw new Error('Job has no ID');\n }\n\n // Create a base logger for job context\n const baseLogger = createLogger(true);\n\n // Inject job context logger\n const contextLogger = new JobContextLogger(baseLogger, {\n jobId,\n attempt: job.attempts,\n queue: job.queue,\n objectType: job.objectType,\n method: job.method,\n });\n\n // Log job start\n contextLogger.info(`Starting job: ${job.getDescription()}`);\n const executionContext = this.createExecutionContext(job, contextLogger);\n\n // Invoke the method with cleaned args (no internal keys)\n const method = (\n instance as unknown as Record<\n string,\n (\n args: unknown,\n context?: JobExecutionContext,\n ) => Promise<unknown> | unknown\n >\n )[job.method];\n if (typeof method !== 'function') {\n throw new Error(`Method not found: ${job.objectType}.${job.method}`);\n }\n\n // Opt-in allowlist: if the target class declares background-eligible\n // methods, refuse to invoke anything outside that set. Dispatch is already\n // bounded to existing prototype methods (no eval/dynamic import), but a\n // class can narrow the reachable surface to an explicit contract\n // (S5 audit #1402). Classes that don't opt in keep current behaviour.\n if (!isBackgroundEligibleMethod(ObjectClass, job.method)) {\n throw new Error(\n `Method not background-eligible: ${job.objectType}.${job.method}`,\n );\n }\n\n const result = await method.call(instance, methodArgs, executionContext);\n\n return { result };\n };\n\n if (job.tenantId) {\n return TenantContext.runWithJobContext(\n { tenantId: job.tenantId },\n runJob,\n );\n }\n\n return runJob();\n }\n\n /**\n * Handle job execution error\n */\n private async handleJobError(job: SmrtJob, error: Error): Promise<void> {\n const strategy = fromConfig(job.retryStrategy);\n const decision: RetryDecision = strategy.shouldRetry(job.attempts, error);\n\n const jobId = job.id;\n if (!jobId) return;\n\n // `last_error` is persisted to the durable `_smrt_jobs` row and is readable\n // through generated (tenant-scoped) list/get routes. Strip secret-shaped\n // substrings before persistence so a failing job that echoes a credential\n // in its message does not turn into a durable leak (S5 audit #1402).\n // Use the throwable-tolerant wrapper: `error` is typed `Error` but reaches\n // here via an `as Error` cast at the call site, so a non-Error throwable\n // (no `.message`) would otherwise persist an empty `last_error`.\n const safeMessage = redactErrorForPersistence(error);\n\n if (decision.shouldRetry && job.attempts < job.maxAttempts) {\n // Schedule retry\n const nextRunAt = new Date(Date.now() + decision.delay);\n\n // Conditional: don't resurrect a row that recovery already failed.\n const applied = await this.writeOwnedJob(jobId, {\n status: 'pending',\n last_error: safeMessage,\n run_at: nextRunAt.toISOString(),\n worker_id: null,\n worker_heartbeat: null,\n updated_at: new Date().toISOString(),\n });\n if (!applied) return;\n\n job.status = 'pending';\n job.lastError = safeMessage;\n job.runAt = nextRunAt;\n job.workerId = null;\n job.workerHeartbeat = null;\n\n await this.appendJobEvent(job, {\n type: 'status',\n level: 'warn',\n stage: 'retrying',\n message: `Retrying job after failure: ${safeMessage}`,\n data: { delay: decision.delay, attempts: job.attempts },\n });\n this.emit('job:retrying', job, error, decision.delay);\n } else {\n // Job failed permanently\n const completedAt = new Date();\n const applied = await this.writeOwnedJob(jobId, {\n status: 'failed',\n completed_at: completedAt.toISOString(),\n last_error: safeMessage,\n updated_at: completedAt.toISOString(),\n });\n if (!applied) return;\n\n job.status = 'failed';\n job.completedAt = completedAt;\n job.lastError = safeMessage;\n\n await this.appendJobEvent(job, {\n type: 'error',\n level: 'error',\n stage: 'failed',\n message: safeMessage,\n data: { attempts: job.attempts },\n });\n this.emit('job:failed', job, error);\n }\n }\n\n private createExecutionContext(\n job: SmrtJob,\n contextLogger: JobContextLogger,\n ): JobExecutionContext {\n const jobContext = {\n jobId: job.id ?? '',\n tenantId: job.tenantId ?? null,\n attempt: job.attempts,\n queue: job.queue,\n objectType: job.objectType,\n method: job.method,\n };\n\n return {\n job: jobContext,\n logger: contextLogger,\n event: async (input: JobEventInput) => {\n await this.appendJobEvent(job, input);\n },\n progress: async (input: JobProgressInput) => {\n const data = {\n ...(input.data ?? {}),\n ...(input.detail ? { detail: input.detail } : {}),\n ...(input.source ? { source: input.source } : {}),\n };\n await this.appendJobEvent(job, {\n type: 'progress',\n level: 'info',\n stage: input.stage,\n progress: input.progress,\n message:\n input.message ??\n input.detail ??\n `${input.stage} ${Math.round(input.progress)}%`,\n data,\n });\n },\n log: async (\n level: 'debug' | 'info' | 'warn' | 'error',\n message: string,\n data?: Record<string, unknown>,\n ) => {\n contextLogger[level](message, data);\n await this.appendJobEvent(job, {\n type: level === 'error' ? 'error' : 'log',\n level,\n message,\n data,\n });\n },\n };\n }\n\n private async appendJobEvent(\n job: SmrtJob,\n input: JobEventInput,\n ): Promise<SmrtJobEvent | null> {\n if (!this.eventCollection || !job.id) {\n return null;\n }\n\n try {\n const event = await this.eventCollection.append({\n tenantId: job.tenantId ?? null,\n jobId: job.id,\n type: input.type ?? 'log',\n level: input.level ?? 'info',\n stage: input.stage ?? null,\n progress: input.progress ?? null,\n message: input.message ?? '',\n data: input.data ?? {},\n });\n\n this.emit('job:event', job, event);\n if (event.type === 'progress') {\n this.emit('job:progress', job, event);\n }\n\n return event;\n } catch (error) {\n const telemetryError =\n error instanceof Error\n ? error\n : new Error(`Failed to append job telemetry: ${String(error)}`);\n\n try {\n this.emit('runner:error', telemetryError);\n } catch {\n // Telemetry is best-effort and must not change job outcomes.\n }\n\n return null;\n }\n }\n\n /**\n * Whether the `_smrt_workers` table exists. Cached once positive — the table\n * never disappears mid-run, so this avoids a probe query on every poll.\n */\n private async workersTableReady(): Promise<boolean> {\n if (this.workersTableVerified) return true;\n const ready = (await this.workerCollection?.tableReady()) ?? false;\n if (ready) this.workersTableVerified = true;\n return ready;\n }\n\n /**\n * Recover jobs orphaned by dead/restarted workers.\n *\n * A `running` job is recovered only when its owning worker is *not alive*\n * (issue #1474): not live in this process and holding no fresh lease in\n * `_smrt_workers`. This is independent of the handler event loop, so a worker\n * whose handler holds the loop synchronously keeps a fresh lease (renewed off\n * the loop by the liveness thread) or stays in this process's live set, and is\n * never false-recovered. The live set takes precedence over a stale lease, and\n * a runner never recovers its own active jobs.\n *\n * Recovery is swept at most once per lease tick (not every poll), since\n * detection is TTL-bound anyway — this bounds the per-poll database load.\n */\n private async recoverStaleJobs(): Promise<void> {\n if (!this.db || !this.collection || !this.workerCollection) return;\n\n // Without the workers table we cannot reason about liveness; skip rather\n // than treat every worker as unknown and mass-recover live jobs.\n if (!(await this.workersTableReady())) return;\n\n // Throttle: at most one sweep per lease tick. Detection latency is bounded\n // by the lease TTL (>= 3x tick), so a faster cadence only adds DB load.\n const now = Date.now();\n if (now - this.lastRecoverySweepAt < this.config.leaseTickMs) return;\n this.lastRecoverySweepAt = now;\n\n // Drop long-dead worker rows so the table stays small, regardless of\n // whether any orphan is found this sweep.\n try {\n await this.workerCollection.pruneExpired(this.effectiveLeaseTtlMs * 10);\n } catch {\n // Pruning is best-effort and must not affect recovery outcomes.\n }\n\n const freshLeaseKeys = await this.workerCollection.freshLeaseWorkerKeys();\n // Worker-internal cross-tenant recovery scan; SmrtJob is @TenantScoped\n // (S5 #1402) so this raw read needs an explicit opt-in.\n const running = await this.collection.query(\n `SELECT * FROM _smrt_jobs WHERE status = 'running'`,\n [],\n { allowRawOnTenantScoped: true },\n );\n if (running.length === 0) return;\n\n // A job is orphaned iff its owning worker is not alive: not live in this\n // process AND no fresh database lease. The live set takes precedence over a\n // stale lease (a worker whose lease lapsed while its loop was blocked is\n // still alive here); a runner also never recovers its own active jobs.\n const orphans = running.filter((job) => {\n const jobId = job.id;\n if (jobId && this.activeJobs.has(jobId)) return false;\n return !isWorkerAlive(job.workerId, freshLeaseKeys);\n });\n if (orphans.length === 0) return;\n\n const orphanIds = orphans\n .map((job) => job.id)\n .filter((jobId): jobId is string => typeof jobId === 'string');\n if (orphanIds.length === 0) return;\n\n const placeholders = orphanIds.map(() => '?').join(', ');\n const recoveredAt = new Date();\n const errorMessage =\n 'Recovered orphaned running job: its owning worker is no longer alive ' +\n '(no fresh liveness lease in _smrt_workers and not running in this process).';\n\n // RETURNING id so we only emit failures for jobs this pass actually\n // transitioned — a concurrent recoverer or a late completion may have\n // already moved some of the candidates out of 'running'.\n const updated = await this.db.query(\n `UPDATE _smrt_jobs\n SET status = 'failed',\n completed_at = ?,\n last_error = ?,\n worker_id = NULL,\n worker_heartbeat = NULL\n WHERE status = 'running'\n AND id IN (${placeholders})\n RETURNING id`,\n recoveredAt.toISOString(),\n errorMessage,\n ...orphanIds,\n );\n const recoveredIds = new Set(\n (updated.rows as Array<{ id?: unknown }>)\n .map((row) => row.id)\n .filter((id): id is string => typeof id === 'string'),\n );\n if (recoveredIds.size === 0) return;\n\n for (const job of orphans) {\n if (!job.id || !recoveredIds.has(job.id)) continue;\n job.status = 'failed';\n job.completedAt = recoveredAt;\n job.lastError = errorMessage;\n job.workerId = null;\n job.workerHeartbeat = null;\n const error = new Error(errorMessage);\n await this.appendJobEvent(job, {\n type: 'error',\n level: 'error',\n stage: 'stale-recovery',\n message: errorMessage,\n });\n this.emit('job:failed', job, error);\n }\n }\n\n /**\n * Renew this worker's liveness lease.\n *\n * In Stage 1 this runs on the main event loop, so it provides cross-process\n * detection no weaker than the old per-job heartbeat. Stage 2 moves the\n * renewal to an off-loop worker thread so a synchronous handler can no longer\n * starve it. Same-process correctness never depends on this timer — the\n * in-memory live set covers it.\n */\n private startLeaseRenewal(): void {\n this.leaseTimer = setInterval(async () => {\n try {\n await this.workerCollection?.renewLease(\n this.workerKey,\n this.effectiveLeaseTtlMs,\n );\n } catch {\n // Ignore transient lease-renewal errors; the next tick retries.\n }\n }, this.config.leaseTickMs);\n }\n\n /**\n * Spawn the off-loop liveness thread. It opens its own connection and renews\n * this worker's lease on its own thread (unstarvable by handler CPU). Returns\n * false if the thread can't be resolved or fails to start, so the caller can\n * fall back to main-loop renewal.\n */\n private async startLivenessThread(): Promise<boolean> {\n if (!this.db) return false;\n let entry: string;\n try {\n // Resolve by package subpath, not as a sibling of this module: the bundle\n // hoists `runner` into dist/chunks/, so a relative URL would miss.\n entry = (\n import.meta as unknown as { resolve(s: string): string }\n ).resolve('@happyvertical/smrt-jobs/worker-liveness-thread');\n } catch {\n return false;\n }\n\n let worker: Worker;\n try {\n worker = new Worker(new URL(entry), {\n workerData: {\n url: resolveUrl(this.db),\n type: resolveEngine(this.db),\n workerKey: this.workerKey,\n leaseTtlMs: this.effectiveLeaseTtlMs,\n leaseTickMs: this.config.leaseTickMs,\n },\n });\n } catch {\n return false;\n }\n\n // Bound the handshake so a hung connect inside the thread (slow/unreachable\n // database) can't stall start() forever — fall back to main-loop renewal.\n const ready = await new Promise<boolean>((resolve) => {\n const onMessage = (message: unknown) => {\n if (message === 'ready') {\n cleanup();\n resolve(true);\n } else if (\n message &&\n typeof message === 'object' &&\n (message as { type?: string }).type === 'error'\n ) {\n cleanup();\n resolve(false);\n }\n };\n const onFail = () => {\n cleanup();\n resolve(false);\n };\n const cleanup = () => {\n clearTimeout(timer);\n worker.off('message', onMessage);\n worker.off('error', onFail);\n worker.off('exit', onFail);\n };\n const timer = setTimeout(() => {\n cleanup();\n resolve(false);\n }, LIVENESS_THREAD_START_TIMEOUT_MS);\n if (typeof timer.unref === 'function') timer.unref();\n worker.on('message', onMessage);\n worker.once('error', onFail);\n worker.once('exit', onFail);\n });\n\n if (!ready) {\n await worker.terminate().catch(() => {});\n return false;\n }\n\n this.livenessWorker = worker;\n // Don't keep the process alive on the liveness thread alone.\n worker.unref();\n // If the thread dies while we're still running, fall back to main-loop\n // renewal so the lease keeps being renewed.\n worker.once('error', () => this.handleLivenessThreadLoss(worker));\n worker.once('exit', () => this.handleLivenessThreadLoss(worker));\n return true;\n }\n\n private handleLivenessThreadLoss(worker: Worker): void {\n if (this.livenessWorker !== worker) return;\n this.livenessWorker = null;\n if (this.running && !this.leaseTimer) {\n this.startLeaseRenewal();\n }\n }\n\n /** Stop the liveness thread (graceful, with a short bound), if running. */\n private async stopLivenessThread(): Promise<void> {\n const worker = this.livenessWorker;\n if (!worker) return;\n this.livenessWorker = null;\n\n const stopped = new Promise<void>((resolve) => {\n const done = () => {\n clearTimeout(timer);\n worker.off('message', onMessage);\n resolve();\n };\n const onMessage = (message: unknown) => {\n if (message === 'stopped') done();\n };\n worker.on('message', onMessage);\n worker.once('exit', done);\n const timer = setTimeout(done, 2000);\n if (typeof timer.unref === 'function') timer.unref();\n });\n try {\n // The worker may have already exited (it's unref'd and best-effort);\n // postMessage to a dead worker throws ERR_WORKER_NOT_RUNNING.\n worker.postMessage('stop');\n } catch {\n // Nothing to ask it to do; fall through to terminate.\n }\n await stopped;\n await worker.terminate().catch(() => {});\n }\n\n /**\n * Per-job heartbeat loop — telemetry only (\"last activity\" for the UI). It no\n * longer gates recovery (that is the worker lease), so a blocked loop missing\n * a heartbeat is harmless.\n */\n private startHeartbeat(): void {\n this.heartbeatTimer = setInterval(async () => {\n if (!this.db) return;\n const jobIds = [...this.activeJobs.keys()];\n if (jobIds.length === 0) return;\n const placeholders = jobIds.map(() => '?').join(', ');\n try {\n await this.db.query(\n `UPDATE _smrt_jobs\n SET worker_heartbeat = ?\n WHERE status = 'running'\n AND id IN (${placeholders})`,\n new Date().toISOString(),\n ...jobIds,\n );\n } catch {\n // Telemetry is best-effort.\n }\n }, this.config.heartbeatInterval);\n }\n\n /**\n * Wait for active jobs to complete with timeout\n */\n private async waitForActiveJobs(): Promise<void> {\n if (this.activeJobs.size === 0) return;\n\n return new Promise((resolve) => {\n const checkInterval = setInterval(() => {\n if (this.activeJobs.size === 0) {\n clearInterval(checkInterval);\n clearTimeout(timeout);\n resolve();\n }\n }, 100);\n\n const timeout = setTimeout(() => {\n clearInterval(checkInterval);\n this.logger.warn(\n `Shutdown timeout: ${this.activeJobs.size} jobs still active`,\n );\n resolve();\n }, this.config.shutdownTimeout);\n });\n }\n}\n\n/**\n * Create a TaskRunner instance\n */\nexport function createTaskRunner(config?: TaskRunnerConfig): TaskRunner {\n return new TaskRunner(config);\n}\n\nexport default TaskRunner;\n"],"names":["tenantId","__decorateClass"],"mappings":";;;;;;;;AAsBA,eAAe;AAAA,EACb,IAAA,IAAA,mBAAA,YAAA,GAAA;AACF;ACCO,MAAM,kBAAkB;AAOxB,MAAM,yBAAyB;AAQ/B,SAAS,aAAa,WAA2B;AACtD,MAAI,OAAO,MAAM,SAAS,KAAK,YAAY,GAAG;AAC5C,WAAO;AAAA,EACT;AACA,MAAI,cAAc,OAAO,mBAAmB;AAC1C,WAAO;AAAA,EACT;AACA,SAAO,KAAK,IAAI,KAAK,MAAM,SAAS,GAAG,eAAe;AACxD;AAKO,MAAM,kCAAkC,MAAM;AAAA,EACnD,YACkBA,WACA,KACA,SAChB;AACA;AAAA,MACE,WAAWA,SAAQ,yCACb,OAAO,IAAI,GAAG;AAAA,IAAA;AANN,SAAA,WAAAA;AACA,SAAA,MAAA;AACA,SAAA,UAAA;AAMhB,SAAK,OAAO;AAAA,EACd;AAAA,EATkB;AAAA,EACA;AAAA,EACA;AAQpB;AAUO,SAAS,8BACdA,WACA,SACA,KACM;AACN,MAAI,CAACA,aAAY,OAAO,EAAG;AAC3B,MAAI,WAAW,KAAK;AAClB,UAAM,IAAI,0BAA0BA,WAAU,KAAK,OAAO;AAAA,EAC5D;AACF;AA4BO,SAAS,uBACd,SACG,SACG;AACN,QAAM,SAAS;AACf,QAAM,WAAW,OAAO;AACxB,QAAM,MACJ,oBAAoB,MAChB,IAAI,IAAY,QAAQ,IACxB,IAAI,IAAY,YAAY,EAAE;AACpC,aAAW,UAAU,QAAS,KAAI,IAAI,MAAM;AAC5C,SAAO,4BAA4B;AACrC;AAqBO,SAAS,qBAAqB;AACnC,SAAO,CACL,QACA,aACA,eACmC;AAGnC,UAAM,OAAQ,OAAmC;AACjD,2BAAuB,MAAM,OAAO,WAAW,CAAC;AAChD,WAAO;AAAA,EACT;AACF;AASO,SAAS,6BACd,MAC4B;AAC5B,QAAM,WAAY,MACd;AACJ,MAAI,YAAY,KAAM,QAAO;AAC7B,SAAO,oBAAoB,MAAM,WAAW,IAAI,IAAI,QAAQ;AAC9D;AAWO,SAAS,2BACd,MACA,QACS;AACT,QAAM,QAAQ,6BAA6B,IAAI;AAC/C,MAAI,SAAS,KAAM,QAAO;AAC1B,SAAO,MAAM,IAAI,MAAM;AACzB;AC5KA,MAAM,WAAW;AAOjB,MAAM,oBAAoB;AAK1B,MAAM,kBAAkB;AAOxB,MAAM,0BACJ;AAOF,MAAM,sBACJ;AASF,MAAM,2BACJ;AAeF,MAAM,uBACJ;AAuBK,SAAS,mBAAmB,SAAyB;AAC1D,MAAI,OAAO,YAAY,YAAY,QAAQ,WAAW,GAAG;AACvD,WAAO;AAAA,EACT;AAEA,SACE,QACG,QAAQ,mBAAmB,KAAK,QAAQ,GAAG,EAG3C,QAAQ,yBAAyB,iBAAiB,QAAQ,EAAE,EAC5D,QAAQ,iBAAiB,MAAM,QAAQ,EAAE,EAGzC,QAAQ,0BAA0B,OAAO,QAAQ,GAAG,EACpD,QAAQ,qBAAqB,OAAO,QAAQ,EAAE,EAC9C,QAAQ,sBAAsB,QAAQ;AAE7C;AAQO,SAAS,0BAA0B,OAAwB;AAChE,MAAI,iBAAiB,OAAO;AAC1B,WAAO,mBAAmB,MAAM,OAAO;AAAA,EACzC;AACA,SAAO,mBAAmB,OAAO,KAAK,CAAC;AACzC;AC1EO,MAAM,iBAAmC;AAAA,EAC9C,YACmB,YACA,YACjB;AAFiB,SAAA,aAAA;AACA,SAAA,aAAA;AAAA,EAChB;AAAA,EAFgB;AAAA,EACA;AAAA,EAGX,WAAW,MAAyD;AAC1E,WAAO;AAAA,MACL,GAAG;AAAA,MACH,MAAM;AAAA,QACJ,IAAI,KAAK,WAAW;AAAA,QACpB,SAAS,KAAK,WAAW;AAAA,QACzB,OAAO,KAAK,WAAW;AAAA,QACvB,YAAY,KAAK,WAAW;AAAA,QAC5B,QAAQ,KAAK,WAAW;AAAA,MAAA;AAAA,IAC1B;AAAA,EAEJ;AAAA,EAEA,MAAM,SAAiB,MAAsC;AAC3D,SAAK,WAAW,MAAM,SAAS,KAAK,WAAW,IAAI,CAAC;AAAA,EACtD;AAAA,EAEA,KAAK,SAAiB,MAAsC;AAC1D,SAAK,WAAW,KAAK,SAAS,KAAK,WAAW,IAAI,CAAC;AAAA,EACrD;AAAA,EAEA,KAAK,SAAiB,MAAsC;AAC1D,SAAK,WAAW,KAAK,SAAS,KAAK,WAAW,IAAI,CAAC;AAAA,EACrD;AAAA,EAEA,MAAM,SAAiB,MAAsC;AAC3D,SAAK,WAAW,MAAM,SAAS,KAAK,WAAW,IAAI,CAAC;AAAA,EACtD;AACF;;;;;;;;;;;ACjBO,IAAM,UAAN,cAAsB,WAAW;AAAA,EAGtC,WAAsC;AAAA,EAItC,QAAgB;AAAA,EAIhB,aAAqB;AAAA,EAIrB,WAA0B;AAAA,EAI1B,SAAiB;AAAA,EAIjB,OAAgC,CAAA;AAAA,EAIhC,4BAAkB,KAAA;AAAA,EAIlB,WAAmB;AAAA,EAInB,SAAoB;AAAA,EAIpB,WAAmB;AAAA,EAInB,cAAsB;AAAA,EAItB,UAAkB;AAAA,EAIlB,kBAAmC;AAAA,EAInC,YAAyB;AAAA,EAIzB,cAA2B;AAAA,EAI3B,YAA2B;AAAA,EAI3B,gBAA+B;AAAA,EAI/B,gBAAqC;AAAA,IACnC,MAAM;AAAA,IACN,QAAQ,EAAE,cAAc,KAAM,YAAY,GAAG,UAAU,IAAA;AAAA,EAAO;AAAA,EAKhE,WAA0B;AAAA,EAI1B,kBAA+B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAO/B,MAAe,OAAsB;AACnC,QAAI,KAAK,aAAa,QAAW;AAC/B,YAAM,kBAAkB,YAAA;AACxB,UAAI,iBAAiB;AACnB,aAAK,WAAW;AAAA,MAClB;AAAA,IACF;AAEA,WAAO,MAAM,KAAA;AAAA,EACf;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAC3B,QAAI,KAAK,WAAW,aAAa;AAC/B,YAAM,IAAI,MAAM,8BAA8B;AAAA,IAChD;AAEA,SAAK,SAAS;AACd,SAAK,WAAW;AAChB,SAAK,YAAY;AACjB,SAAK,YAAY;AACjB,SAAK,cAAc;AACnB,SAAK,WAAW;AAChB,SAAK,kBAAkB;AAEvB,UAAM,KAAK,KAAA;AAAA,EACb;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,SAAwB;AAC5B,QAAI,KAAK,WAAW,eAAe,KAAK,WAAW,aAAa;AAC9D,YAAM,IAAI,MAAM,kCAAkC,KAAK,MAAM,EAAE;AAAA,IACjE;AAEA,SAAK,SAAS;AACd,SAAK,kCAAkB,KAAA;AAEvB,UAAM,KAAK,KAAA;AAAA,EACb;AAAA;AAAA;AAAA;AAAA,EAKA,iBAAyB;AACvB,UAAM,SAAS,KAAK,WAChB,GAAG,KAAK,UAAU,IAAI,KAAK,QAAQ,KACnC,KAAK;AACT,WAAO,GAAG,MAAM,IAAI,KAAK,MAAM;AAAA,EACjC;AACF;AA3IEC,kBAAA;AAAA,EADC,SAAS,EAAE,UAAU,KAAA,CAAM;AAAA,GAFjB,QAGX,WAAA,YAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM,SAAS,WAAW;AAAA,GANhD,QAOX,WAAA,SAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM;AAAA,GAV5B,QAWX,WAAA,cAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM;AAAA,GAd5B,QAeX,WAAA,YAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM;AAAA,GAlB5B,QAmBX,WAAA,UAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,OAAA,CAAQ;AAAA,GAtBZ,QAuBX,WAAA,QAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,YAAY,UAAU,MAAM;AAAA,GA1BhC,QA2BX,WAAA,SAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,WAAW,UAAU,MAAM,SAAS,IAAI;AAAA,GA9B5C,QA+BX,WAAA,YAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM,SAAS,WAAW;AAAA,GAlChD,QAmCX,WAAA,UAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,WAAW,UAAU,MAAM,SAAS,GAAG;AAAA,GAtC3C,QAuCX,WAAA,YAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,WAAW,UAAU,MAAM,SAAS,GAAG;AAAA,GA1C3C,QA2CX,WAAA,eAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,WAAW,UAAU,MAAM,SAAS,KAAQ;AAAA,GA9ChD,QA+CX,WAAA,WAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM,SAAS,QAAQ;AAAA,GAlD7C,QAmDX,WAAA,mBAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,YAAY,UAAU,MAAM;AAAA,GAtDhC,QAuDX,WAAA,aAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,YAAY,UAAU,MAAM;AAAA,GA1DhC,QA2DX,WAAA,eAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM;AAAA,GA9D5B,QA+DX,WAAA,aAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM;AAAA,GAlE5B,QAmEX,WAAA,iBAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,OAAA,CAAQ;AAAA,GAtEZ,QAuEX,WAAA,iBAAA,CAAA;AAOAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM;AAAA,GA7E5B,QA8EX,WAAA,YAAA,CAAA;AAIAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,YAAY,UAAU,MAAM;AAAA,GAjFhC,QAkFX,WAAA,mBAAA,CAAA;AAlFW,UAANA,kBAAA;AAAA,EAxBN,KAAK;AAAA,IACJ,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQX,KAAK;AAAA;AAAA;AAAA,IAGL,KAAK;AAAA,MACH,SAAS,CAAC,QAAQ,OAAO,SAAS,QAAQ;AAAA,MAC1C,cAAc;AAAA,MACd,MAAM;AAAA,IAAA;AAAA,IAER,KAAK;AAAA,EAAA,CACN;AAAA,EAKA,aAAa,EAAE,MAAM,WAAA,CAAY;AAAA,GACrB,OAAA;AAyMN,MAAM,0BAA0B,eAAwB;AAAA,EAC7D,OAAgB,aAAa;AAAA,EAE7B,MAAe,aAA4B;AACzC,UAAM,MAAM,WAAA;AACZ,UAAM,mCAAmC,KAAK,EAAE;AAChD,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,aACJ,QACA,UAA8C,IAC1B;AACpB,UAAM,QAAiC;AAAA,MACrC,QAAQ,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,MAAM;AAAA,IAAA;AAGlD,QAAI,QAAQ,OAAO;AACjB,YAAM,QAAQ,QAAQ;AAAA,IACxB;AAEA,WAAO,KAAK,KAAK;AAAA,MACf;AAAA,MACA,SAAS,CAAC,iBAAiB,YAAY;AAAA,MACvC,OAAO,QAAQ;AAAA,IAAA,CAChB;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,UACJ,UAAiD,IAC7B;AACpB,UAAM,OAAM,oBAAI,KAAA,GAAO,YAAA;AACvB,UAAM,kBAA4B,CAAC,sBAAsB,aAAa;AACtE,UAAM,SAAoB,CAAC,GAAG;AAE9B,QAAI,QAAQ,QAAQ,QAAQ;AAC1B,YAAM,eAAe,QAAQ,OAAO,IAAI,MAAM,GAAG,EAAE,KAAK,IAAI;AAC5D,sBAAgB,KAAK,aAAa,YAAY,GAAG;AACjD,aAAO,KAAK,GAAG,QAAQ,MAAM;AAAA,IAC/B;AAEA,WAAO,KAAK,QAAQ,SAAS,GAAG;AAKhC,WAAO,KAAK;AAAA,MACV,kCAAkC,gBAAgB,KAAK,OAAO,CAAC;AAAA,MAC/D;AAAA,MACA,EAAE,wBAAwB,KAAA;AAAA,IAAK;AAAA,EAEnC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAM,WAAW,SAAgD;AAC/D,UAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAI,SAAS,EAAG,QAAO,CAAA;AAEvB,UAAM,MAAM,QAAQ,OAAO,oBAAI,KAAA;AAC/B,UAAM,SAAS,IAAI,YAAA;AACnB,UAAM,kBAA4B,CAAC,sBAAsB,aAAa;AACtE,UAAM,cAAyB,CAAC,MAAM;AAEtC,QAAI,QAAQ,QAAQ,QAAQ;AAC1B,YAAM,eAAe,QAAQ,OAAO,IAAI,MAAM,GAAG,EAAE,KAAK,IAAI;AAC5D,sBAAgB,KAAK,aAAa,YAAY,GAAG;AACjD,kBAAY,KAAK,GAAG,QAAQ,MAAM;AAAA,IACpC;AAEA,UAAM,aACJ,kBAAkB,KAAK,EAAE,MAAM,aAC3B,4BACA;AACN,UAAM,kBAAkB;AAAA;AAAA;AAAA,eAGb,gBAAgB,KAAK,OAAO,CAAC;AAAA;AAAA,gBAE5B,UAAU;AAAA;AAGtB,UAAM,UAAU,MAAM,KAAK;AAAA,MACzB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBAOiB,eAAe;AAAA;AAAA;AAAA,MAGhC,CAAC,QAAQ,UAAU,QAAQ,QAAQ,QAAQ,GAAG,aAAa,KAAK;AAAA;AAAA;AAAA,MAGhE,EAAE,wBAAwB,KAAA;AAAA,IAAK;AAGjC,WAAO,QAAQ,SAAS,iBAAiB;AAAA,EAC3C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,uBAAuBD,WAA0C;AACrE,UAAM,YAAYA,cAAa,OAAO,sBAAsB;AAC5D,UAAM,SAASA,cAAa,OAAO,CAAA,IAAK,CAACA,SAAQ;AAOjD,UAAM,SAAS,MAAM,KAAK,GAAG;AAAA,MAC3B;AAAA;AAAA;AAAA,gBAGU,SAAS;AAAA,MACnB,GAAG;AAAA,IAAA;AAGL,UAAM,MAAM,OAAO,KAAK,CAAC;AACzB,WAAO,OAAO,KAAK,SAAS,CAAC;AAAA,EAC/B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAiCA,MAAM,WACJ,MACA,UAA6B,IACX;AAClB,UAAM,MAAM,QAAQ,gBAAgB;AAKpC,UAAM,iBACJ,OAAO,KAAK,aAAa,YAAY,KAAK,SAAS,SAAS,IACxD,KAAK,WACL,KAAK,aAAa,OAChB,OACA;AACR,UAAM,kBACJ,mBAAmB,SAAY,iBAAkB,iBAAiB;AAEpE,QAAI,mBAAmB,MAAM,GAAG;AAC9B,YAAM,UAAU,MAAM,KAAK,uBAAuB,eAAe;AACjE,oCAA8B,iBAAiB,SAAS,GAAG;AAAA,IAC7D;AAEA,UAAM,MAAM,MAAM,KAAK,OAAO;AAAA,MAC5B,GAAG;AAAA;AAAA;AAAA,MAGH,aAAa,aAAa,KAAK,eAAe,CAAC;AAAA,IAAA,CAChD;AACD,UAAM,IAAI,KAAA;AACV,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,MAAM,OAMT;AACD,UAAM,QAAQ,QACV,qFACA;AACJ,UAAM,SAAS,QAAQ,CAAC,KAAK,IAAI,CAAA;AAEjC,UAAM,SAAS,MAAM,KAAK,IAAI,MAAM,OAAO,GAAG,MAAM;AAEpD,UAAM,SAAiC,CAAA;AACvC,eAAW,OAAO,OAAO,MAAM;AAC7B,aAAO,IAAI,MAAgB,IAAI,IAAI;AAAA,IACrC;AAEA,WAAO;AAAA,MACL,SAAS,OAAO,WAAW;AAAA,MAC3B,SAAS,OAAO,WAAW;AAAA,MAC3B,WAAW,OAAO,aAAa;AAAA,MAC/B,QAAQ,OAAO,UAAU;AAAA,MACzB,WAAW,OAAO,aAAa;AAAA,IAAA;AAAA,EAEnC;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAQ,SAKM;AAClB,UAAM,aAAuB,CAAA;AAC7B,UAAM,SAAoB,CAAA;AAE1B,QAAI,QAAQ,iBAAiB;AAC3B,iBAAW,KAAK,6CAA6C;AAC7D,aAAO,KAAK,QAAQ,gBAAgB,YAAA,CAAa;AAAA,IACnD;AAEA,QAAI,QAAQ,cAAc;AACxB,iBAAW,KAAK,0CAA0C;AAC1D,aAAO,KAAK,QAAQ,aAAa,YAAA,CAAa;AAAA,IAChD;AAEA,QAAI,QAAQ,iBAAiB;AAC3B,iBAAW,KAAK,6CAA6C;AAC7D,aAAO,KAAK,QAAQ,gBAAgB,YAAA,CAAa;AAAA,IACnD;AAEA,QAAI,WAAW,WAAW,EAAG,QAAO;AAEpC,QAAI,QAAQ,iCAAiC,WAAW,KAAK,MAAM,CAAC;AAEpE,QAAI,QAAQ,OAAO;AACjB,cAAQ;AAAA;AAAA;AAAA;AAAA,mBAIK,WAAW,KAAK,MAAM,CAAC;AAAA;AAAA;AAAA;AAIpC,aAAO,KAAK,QAAQ,KAAK;AAAA,IAC3B;AAEA,UAAM,SAAS,MAAM,KAAK,IAAI,MAAM,OAAO,GAAG,MAAM;AACpD,WAAO,OAAO,YAAY;AAAA,EAC5B;AACF;AAEA,SAAS,kBACP,IACiC;AACjC,QAAM,eAAe;AACrB,SAAO;AAAA,IACL,GAAG,OAAO,aAAa,QAAQ,OAAO;AAAA,IACtC,aAAa,QAAQ,aAAa,QAAQ;AAAA,EAAA;AAE9C;AAEA,SAAS,kBAAkB,MAAe,OAAwB;AAChE,QAAM,WAAW,MAAM,WAAW,KAAK;AACvC,MAAI,aAAa,EAAG,QAAO;AAE3B,QAAM,QAAQ,KAAK,MAAM,YAAY,MAAM,MAAM,QAAA;AACjD,MAAI,UAAU,EAAG,QAAO;AAExB,QAAM,YAAY,UAAU,KAAK,UAAU,IAAI,UAAU,MAAM,UAAU;AACzE,MAAI,cAAc,EAAG,QAAO;AAE5B,UAAQ,KAAK,MAAM,IAAI,cAAc,MAAM,MAAM,EAAE;AACrD;AAEA,SAAS,UAAU,OAAwC;AACzD,SAAO,OAAO,aAAa;AAC7B;;;;;;;;;;;ACxhBA,MAAM,4BAA4B;AAAA,EAChC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,IAAI;AAsBJ,IAAM,eAAN,cAA2B,WAAW;AAAA,EAE3C,WAAsC;AAAA,EAGtC,QAAgB;AAAA,EAGhB,OAAyB;AAAA,EAGzB,QAA2B;AAAA,EAG3B,QAAuB;AAAA,EAGvB,WAA0B;AAAA,EAG1B,UAAkB;AAAA,EAGlB,OAAgC,CAAA;AAAA,EAGhC,gCAAsB,KAAA;AAAA,EAEtB,WAAmB;AACjB,UAAM,YACJ,KAAK,qBAAqB,OACtB,KAAK,UAAU,YAAA,IACf,OAAO,KAAK,SAAS;AAC3B,WAAO,GAAG,SAAS,IAAI,KAAK,MAAM,EAAE;AAAA,EACtC;AACF;AAjCEC,kBAAA;AAAA,EADC,SAAS,EAAE,UAAU,KAAA,CAAM;AAAA,GADjB,aAEX,WAAA,YAAA,CAAA;AAGAA,kBAAA;AAAA,EADC,WAAW,WAAW,EAAE,UAAU,MAAM;AAAA,GAJ9B,aAKX,WAAA,SAAA,CAAA;AAGAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM,SAAS,OAAO;AAAA,GAP5C,aAQX,WAAA,QAAA,CAAA;AAGAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM,SAAS,QAAQ;AAAA,GAV7C,aAWX,WAAA,SAAA,CAAA;AAGAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM;AAAA,GAb5B,aAcX,WAAA,SAAA,CAAA;AAGAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,WAAW,UAAU,MAAM;AAAA,GAhB/B,aAiBX,WAAA,YAAA,CAAA;AAGAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM,SAAS,IAAI;AAAA,GAnBzC,aAoBX,WAAA,WAAA,CAAA;AAGAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,OAAA,CAAQ;AAAA,GAtBZ,aAuBX,WAAA,QAAA,CAAA;AAGAA,kBAAA;AAAA,EADC,MAAM,EAAE,MAAM,YAAY,UAAU,MAAM;AAAA,GAzBhC,aA0BX,WAAA,aAAA,CAAA;AA1BW,eAANA,kBAAA;AAAA,EApBN,KAAK;AAAA,IACJ,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQX,KAAK;AAAA;AAAA;AAAA;AAAA,IAIL,KAAK,EAAE,SAAS,CAAC,QAAQ,KAAK,GAAG,MAAM,OAAO,cAAc,KAAA;AAAA,IAC5D,KAAK;AAAA,EAAA,CACN;AAAA,EAIA,aAAa,EAAE,MAAM,WAAA,CAAY;AAAA,GACrB,YAAA;AAqCb,SAAS,eAAe,OAAmC;AACzD,QAAM,UACJ,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ;AAChE,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,KAAM,KAAK,MAAM,OAAO,CAAC,CAAC;AACxD;AAEA,SAAS,kBAAkB,UAAkC;AAC3D,MAAI,OAAO,aAAa,YAAY,CAAC,OAAO,SAAS,QAAQ,GAAG;AAC9D,WAAO;AAAA,EACT;AAEA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,KAAK,MAAM,QAAQ,CAAC,CAAC;AACxD;AAEA,SAAS,YAAY,QAAiD;AACpE,MAAI,OAAO,WAAW,SAAU,QAAO;AACvC,QAAM,YAAY,OAAO,YAAY,GAAG;AACxC,MAAI,cAAc,IAAI;AACpB,WAAO,EAAE,WAAW,QAAQ,IAAI,GAAA;AAAA,EAClC;AACA,SAAO;AAAA,IACL,WAAW,OAAO,MAAM,GAAG,SAAS;AAAA,IACpC,IAAI,OAAO,MAAM,YAAY,CAAC;AAAA,EAAA;AAElC;AAEA,SAAS,oBAAoB,OAA8B;AACzD,MAAI,iBAAiB,MAAM;AACzB,WAAO,MAAM,YAAA;AAAA,EACf;AAEA,QAAM,SAAS,IAAI,KAAK,KAAK;AAC7B,MAAI,CAAC,OAAO,MAAM,OAAO,QAAA,CAAS,GAAG;AACnC,WAAO,OAAO,YAAA;AAAA,EAChB;AAEA,SAAO;AACT;AAEA,SAAS,wBAAwB,OAAwB;AACvD,QAAM,aAAa,MAAM,YAAA;AACzB,SAAO,EACL,WAAW,WAAW,WAAW,KAAK,WAAW,WAAW,aAAa;AAE7E;AAEA,SAAS,aAAa,QAA4C;AAChE,MAAI,MAAM,QAAQ,MAAM,GAAG;AACzB,WAAO;AAAA,EACT;AAEA,SAAQ,OAAgD,QAAQ,CAAA;AAClE;AAEO,MAAM,+BAA+B,eAA6B;AAAA,EACvE,OAAgB,aAAa;AAAA,EAE7B,MAAe,aAA4B;AACzC,UAAM,MAAM,WAAA;AACZ,UAAM,wCAAwC,KAAK,EAAE;AACrD,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAO,OAAgD;AAC3D,WAAO,KAAK,OAAO;AAAA,MACjB,UAAU,MAAM;AAAA,MAChB,OAAO,MAAM;AAAA,MACb,MAAM,MAAM,QAAQ;AAAA,MACpB,OAAO,MAAM,SAAS;AAAA,MACtB,OAAO,MAAM,SAAS;AAAA,MACtB,UAAU,kBAAkB,MAAM,QAAQ;AAAA,MAC1C,SAAS,MAAM,WAAW;AAAA,MAC1B,MAAM,MAAM,QAAQ,CAAA;AAAA,MACpB,WAAW,MAAM,aAAa,oBAAI,KAAA;AAAA,IAAK,CACxC;AAAA,EACH;AAAA,EAEA,MAAM,UACJ,OACA,UAAgC,IACP;AACzB,WAAO,KAAK,gBAAgB;AAAA,MAC1B,GAAG;AAAA,MACH;AAAA,IAAA,CACD;AAAA,EACH;AAAA,EAEA,MAAM,gBACJ,UAAqD,IAC5B;AACzB,UAAM,QAAkB,CAAA;AACxB,UAAM,SAAoB,CAAA;AAE1B,QAAI,QAAQ,OAAO;AACjB,YAAM,KAAK,YAAY;AACvB,aAAO,KAAK,QAAQ,KAAK;AAAA,IAC3B;AAEA,SAAK,mBAAmB,OAAO,QAAQ,OAAO;AAE9C,QAAI,QAAQ,QAAQ;AAClB,YAAM,SAAS,YAAY,QAAQ,MAAM;AACzC,YAAM,YAAY,MAAM,KAAK,uBAAuB,QAAQ,OAAO;AACnE,YAAM,sBAAsB,KAAK,8BAAA;AACjC,YAAM;AAAA,QACJ,IAAI,mBAAmB,YAAY,mBAAmB;AAAA,MAAA;AAExD,aAAO,KAAK,WAAW,WAAW,OAAO,EAAE;AAAA,IAC7C,WAAW,QAAQ,OAAO;AACxB,YAAM,KAAK,GAAG,KAAK,8BAAA,CAA+B,MAAM;AACxD,aAAO,KAAK,oBAAoB,QAAQ,KAAK,CAAC;AAAA,IAChD;AAEA,QAAI,QAAQ,SAAS;AACnB,YAAM,KAAK,QAAQ;AACnB,aAAO,KAAK,QAAQ,OAAO;AAAA,IAC7B;AAEA,WAAO,KAAK,eAAe,QAAQ,KAAK,CAAC;AAEzC,UAAM,WAAW,MAAM,SAAS,SAAS,MAAM,KAAK,OAAO,CAAC,KAAK;AACjE,WAAO,KAAK;AAAA,MACV,UAAU,yBAAyB;AAAA;AAAA,UAE/B,QAAQ;AAAA,mBACC,KAAK,+BAA+B;AAAA;AAAA,MAEjD;AAAA,MACA,EAAE,wBAAwB,KAAA;AAAA,IAAK;AAAA,EAEnC;AAAA,EAEA,MAAM,uBACJ,QACA,UAAwC,IACJ;AACpC,UAAM,eAAe,CAAC,GAAG,IAAI,IAAI,OAAO,OAAO,OAAO,CAAC,CAAC;AACxD,UAAM,oCAAoB,IAAA;AAC1B,QAAI,aAAa,WAAW,EAAG,QAAO;AAEtC,UAAM,eAAe,aAAa,IAAI,MAAM,GAAG,EAAE,KAAK,IAAI;AAC1D,UAAM,QAAkB;AAAA,MACtB,cAAc,YAAY;AAAA,MAC1B;AAAA,IAAA;AAEF,UAAM,SAAoB,CAAC,GAAG,YAAY;AAE1C,SAAK,mBAAmB,OAAO,QAAQ,OAAO;AAC9C,UAAM,sBAAsB,KAAK,8BAAA;AAEjC,UAAM,SAAS,MAAM,KAAK;AAAA,MACxB,UAAU,yBAAyB;AAAA;AAAA,oBAErB,yBAAyB;AAAA,oBACzB,mBAAmB;AAAA;AAAA;AAAA,+BAGR,mBAAmB;AAAA;AAAA;AAAA,oBAG9B,MAAM,KAAK,OAAO,CAAC;AAAA;AAAA;AAAA;AAAA,MAIjC;AAAA,MACA,EAAE,wBAAwB,KAAA;AAAA,IAAK;AAGjC,eAAW,SAAS,QAAQ;AAC1B,oBAAc,IAAI,MAAM,OAAO,KAAK;AAAA,IACtC;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,mBACN,OACA,QACA,SACM;AACN,QAAI,QAAQ,aAAa,MAAM;AAC7B,YAAM,KAAK,mBAAmB;AAC9B;AAAA,IACF;AAEA,UAAMD,YACJ,OAAO,QAAQ,aAAa,WAAW,QAAQ,WAAW,YAAA;AAE5D,QAAIA,WAAU;AACZ,YAAM,KAAK,eAAe;AAC1B,aAAO,KAAKA,SAAQ;AACpB;AAAA,IACF;AAEA,UAAM,IAAI;AAAA,MACR;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEQ,gCAAwC;AAC9C,QAAI,wBAAwB,KAAK,GAAG,GAAG,GAAG;AACxC,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,uBACZ,QACA,SACiB;AACjB,QAAI,CAAC,OAAO,IAAI;AACd,aAAO,oBAAoB,OAAO,SAAS;AAAA,IAC7C;AAEA,UAAM,QAAQ,CAAC,QAAQ;AACvB,UAAM,SAAoB,CAAC,OAAO,EAAE;AACpC,QAAI,QAAQ,OAAO;AACjB,YAAM,KAAK,YAAY;AACvB,aAAO,KAAK,QAAQ,KAAK;AAAA,IAC3B;AACA,SAAK,mBAAmB,OAAO,QAAQ,OAAO;AAE9C,UAAM,SAAS,MAAM,KAAK,GAAG;AAAA,MAC3B,UAAU,KAAK,+BAA+B;AAAA;AAAA,gBAEpC,MAAM,KAAK,OAAO,CAAC;AAAA;AAAA,MAE7B,GAAG;AAAA,IAAA;AAEL,UAAM,kBAAkB,aAAa,MAAM,EAAE,CAAC,GAAG;AAEjD,WAAO,OAAO,oBAAoB,YAAY,gBAAgB,SAC1D,kBACA,oBAAoB,OAAO,SAAS;AAAA,EAC1C;AACF;;;;;;;;;;;AC9TO,IAAM,aAAN,cAAyB,WAAW;AAAA,EAGzC,WAAmB;AAAA,EAInB,MAAqB;AAAA,EAIrB,WAA0B;AAAA,EAI1B,YAAyB;AAAA,EAIzB,cAA2B;AAAA,EAI3B,iBAA8B;AAAA,EAI9B,SAAiB;AACnB;AAzBE,gBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM;AAAA,GAF5B,WAGX,WAAA,YAAA,CAAA;AAIA,gBAAA;AAAA,EADC,MAAM,EAAE,MAAM,WAAW,UAAU,MAAM;AAAA,GAN/B,WAOX,WAAA,OAAA,CAAA;AAIA,gBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM;AAAA,GAV5B,WAWX,WAAA,YAAA,CAAA;AAIA,gBAAA;AAAA,EADC,MAAM,EAAE,MAAM,YAAY,UAAU,MAAM;AAAA,GAdhC,WAeX,WAAA,aAAA,CAAA;AAIA,gBAAA;AAAA,EADC,MAAM,EAAE,MAAM,YAAY,UAAU,MAAM;AAAA,GAlBhC,WAmBX,WAAA,eAAA,CAAA;AAIA,gBAAA;AAAA,EADC,MAAM,EAAE,MAAM,YAAY,UAAU,MAAM;AAAA,GAtBhC,WAuBX,WAAA,kBAAA,CAAA;AAIA,gBAAA;AAAA,EADC,MAAM,EAAE,MAAM,QAAQ,UAAU,MAAM,SAAS,WAAW;AAAA,GA1BhD,WA2BX,WAAA,UAAA,CAAA;AA3BW,aAAN,gBAAA;AAAA,EAPN,KAAK;AAAA,IACJ,WAAW;AAAA,IACX,iBAAiB,CAAC,WAAW;AAAA,IAC7B,KAAK;AAAA,IACL,KAAK;AAAA,IACL,KAAK;AAAA,EAAA,CACN;AAAA,GACY,UAAA;AAwCN,MAAM,6BAA6B,eAA2B;AAAA,EACnE,OAAgB,aAAa;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAU7B,MAAM,cAA6B;AACjC,QAAI;AACF,YAAM,KAAK,GAAG,MAAM,qCAAqC;AAAA,IAC3D,SAAS,OAAO;AACd,YAAM,IAAI;AAAA,QACR,2JAEyB,MAAgB,OAAO;AAAA,MAAA;AAAA,IAEpD;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,aAA+B;AACnC,QAAI;AACF,YAAM,KAAK,GAAG,MAAM,qCAAqC;AACzD,aAAO;AAAA,IACT,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,eAAe,OAA2C;AAC9D,UAAM,0BAAU,KAAA;AAChB,UAAM,KAAK,OAAO;AAAA,MAChB,UAAU,MAAM;AAAA,MAChB,KAAK,MAAM,OAAO;AAAA,MAClB,UAAU,MAAM,YAAY;AAAA,MAC5B,WAAW;AAAA,MACX,aAAa;AAAA;AAAA;AAAA,MAGb,gBAAgB,IAAI,KAAK,IAAI,QAAA,IAAY,MAAM,UAAU;AAAA,MACzD,QAAQ;AAAA,IAAA,CACT;AAAA,EACH;AAAA;AAAA,EAGA,MAAM,WAAW,WAAmB,YAAmC;AACrE,UAAM,0BAAU,KAAA;AAChB,UAAM,KAAK,GAAG;AAAA,MACZ;AAAA;AAAA;AAAA;AAAA,MAIA,IAAI,KAAK,IAAI,YAAY,UAAU,EAAE,YAAA;AAAA,MACrC,IAAI,YAAA;AAAA,MACJ;AAAA,IAAA;AAAA,EAEJ;AAAA;AAAA,EAGA,MAAM,aAAa,WAAkC;AACnD,UAAM,KAAK,GAAG;AAAA,MACZ;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAAA;AAAA,EAGA,MAAM,uBAA6C;AACjD,UAAM,SAAS,MAAM,KAAK,GAAG;AAAA,MAC3B;AAAA;AAAA;AAAA;AAAA,OAIA,oBAAI,KAAA,GAAO,YAAA;AAAA,IAAY;AAEzB,UAAM,2BAAW,IAAA;AACjB,eAAW,OAAO,OAAO,MAAwC;AAC/D,UAAI,OAAO,IAAI,cAAc,SAAU,MAAK,IAAI,IAAI,SAAS;AAAA,IAC/D;AACA,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,MAAM,aAAa,SAAgC;AACjD,UAAM,SAAS,IAAI,KAAK,KAAK,IAAA,IAAQ,KAAK,IAAI,GAAG,OAAO,CAAC,EAAE,YAAA;AAC3D,UAAM,KAAK,GAAG;AAAA,MACZ;AAAA;AAAA;AAAA,MAGA;AAAA,IAAA;AAAA,EAEJ;AACF;AC/KO,MAAM,qCAAqC;AAU3C,MAAM,wBAAwB;AAS9B,MAAM,uBAAuB;AAEpC,MAAM,6BAA6B;AA0B5B,SAAS,uBACd,YACA,aACQ;AACR,SAAO,KAAK,IAAI,YAAY,cAAc,0BAA0B;AACtE;ACuCA,MAAM,mCAAmC;AAKzC,MAAM,iBAA6C;AAAA,EACjD,IAAI;AAAA,EACJ,aAAa;AAAA,EACb,QAAQ,CAAC,SAAS;AAAA,EAClB,cAAc;AAAA,EACd,mBAAmB;AAAA,EACnB,iBAAiB;AAAA,EACjB,qBAAqB;AAAA,EACrB,YAAY;AAAA,EACZ,aAAa;AACf;AAYO,MAAM,mBAAmB,aAAa;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQQ;AAAA,EACA;AAAA,EACA;AAAA,EACT,aAAuC;AAAA,EACvC,kBAAiD;AAAA,EACjD,mBAAgD;AAAA,EAChD,uBAAuB;AAAA,EACvB,sBAAsB;AAAA,EACtB,UAAU;AAAA,EACV,iCAAiB,IAAA;AAAA,EACjB,YAAmC;AAAA,EACnC,iBAAwC;AAAA,EACxC,aAAoC;AAAA,EACpC,iBAAgC;AAAA,EAChC,kBAAwC;AAAA,EACxC,KAA+B;AAAA,EAC/B,SAAS,aAAa,IAAI;AAAA,EAElC,YAAY,SAA2B,IAAI;AACzC,UAAA;AACA,SAAK,SAAS;AAAA,MACZ,GAAG;AAAA,MACH,GAAG;AAAA,MACH,IAAI,OAAO,MAAM,UAAU,WAAW,MAAM,GAAG,CAAC,CAAC;AAAA,IAAA;AAEnD,SAAK,KAAK,KAAK,OAAO;AACtB,SAAK,YAAY,gBAAgB,KAAK,EAAE;AACxC,SAAK,sBAAsB;AAAA,MACzB,KAAK,OAAO;AAAA,MACZ,KAAK,OAAO;AAAA,IAAA;AAAA,EAEhB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,WAAW,IAAsC;AACrD,SAAK,KAAK;AACV,SAAK,aAAa,MAAM,kBAAkB,OAAO,EAAE,IAAI;AACvD,SAAK,kBAAkB,MAAM,uBAAuB,OAAO,EAAE,IAAI;AACjE,SAAK,mBAAmB,MAAM,qBAAqB,OAAO,EAAE,IAAI;AAAA,EAClE;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AAC3B,QAAI,KAAK,QAAS;AAClB,QAAI,CAAC,KAAK,cAAc,CAAC,KAAK,kBAAkB;AAC9C,YAAM,IAAI,MAAM,sDAAsD;AAAA,IACxE;AAIA,UAAM,KAAK,iBAAiB,YAAA;AAK5B,QAAI,KAAK,MAAM,cAAc,KAAK,EAAE,MAAM,UAAU;AAClD,YAAM,yBAAyB,KAAK,EAAE;AAAA,IACxC;AAKA,UAAM,KAAK,iBAAiB,eAAe;AAAA,MACzC,WAAW,KAAK;AAAA,MAChB,KAAK,OAAO,YAAY,cAAc,QAAQ,MAAM;AAAA,MACpD,UACE,OAAO,YAAY,cAAe,QAAQ,IAAI,YAAY,OAAQ;AAAA,MACpE,YAAY,KAAK;AAAA,IAAA,CAClB;AACD,uBAAmB,KAAK,SAAS;AAEjC,SAAK,UAAU;AAMf,QAAI,gBAAgB,KAAK,EAAuB,GAAG;AACjD,YAAM,gBAAgB,MAAM,KAAK,oBAAA;AACjC,UAAI,CAAC,cAAe,MAAK,kBAAA;AAAA,IAC3B,OAAO;AACL,WAAK,kBAAA;AAAA,IACP;AAGA,SAAK,aAAA;AAGL,SAAK,eAAA;AAEL,SAAK,KAAK,gBAAgB;AAAA,EAC5B;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,OAAsB;AAC1B,QAAI,CAAC,KAAK,QAAS;AACnB,QAAI,KAAK,gBAAiB,QAAO,KAAK;AAEtC,SAAK,UAAU;AAGf,QAAI,KAAK,WAAW;AAClB,mBAAa,KAAK,SAAS;AAC3B,WAAK,YAAY;AAAA,IACnB;AACA,QAAI,KAAK,gBAAgB;AACvB,oBAAc,KAAK,cAAc;AACjC,WAAK,iBAAiB;AAAA,IACxB;AAMA,SAAK,kBAAkB,KAAK,kBAAA;AAE5B,QAAI;AACF,YAAM,KAAK;AAAA,IACb,UAAA;AACE,WAAK,kBAAkB;AAGvB,YAAM,KAAK,mBAAA;AACX,UAAI,KAAK,YAAY;AACnB,sBAAc,KAAK,UAAU;AAC7B,aAAK,aAAa;AAAA,MACpB;AAGA,2BAAqB,KAAK,SAAS;AAKnC,UAAI,KAAK,WAAW,SAAS,GAAG;AAC9B,YAAI;AACF,gBAAM,KAAK,kBAAkB,aAAa,KAAK,SAAS;AAAA,QAC1D,QAAQ;AAAA,QAER;AAAA,MACF;AACA,WAAK,KAAK,gBAAgB;AAAA,IAC5B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,YAAqB;AACnB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAKA,iBAAyB;AACvB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA;AAAA;AAAA;AAAA,EAKQ,eAAqB;AAC3B,UAAM,OAAO,YAAY;AACvB,UAAI,CAAC,KAAK,QAAS;AAEnB,UAAI;AACF,cAAM,KAAK,KAAA;AAAA,MACb,SAAS,OAAO;AACd,aAAK,KAAK,gBAAgB,KAAc;AAAA,MAC1C;AAGA,UAAI,KAAK,SAAS;AAChB,aAAK,YAAY,WAAW,MAAM,KAAK,OAAO,YAAY;AAAA,MAC5D;AAAA,IACF;AAGA,SAAA;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,OAAsB;AAClC,QAAI,CAAC,KAAK,cAAc,CAAC,KAAK,GAAI;AAElC,UAAM,KAAK,iBAAA;AAGX,UAAM,YAAY,KAAK,OAAO,cAAc,KAAK,WAAW;AAC5D,QAAI,aAAa,EAAG;AAIpB,UAAM,OAAO,MAAM,KAAK,WAAW,WAAW;AAAA,MAC5C,UAAU,KAAK;AAAA,MACf,QAAQ,KAAK,OAAO;AAAA,MACpB,OAAO;AAAA,IAAA,CACR;AAED,eAAW,OAAO,MAAM;AACtB,YAAM,QAAQ,IAAI;AAClB,UAAI,CAAC,MAAO;AAGZ,WAAK,WAAW,GAAG;AAAA,IACrB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,WAAW,KAA6B;AACpD,UAAM,QAAQ,IAAI;AAClB,QAAI,CAAC,OAAO;AACV,WAAK,KAAK,gBAAgB,IAAI,MAAM,eAAe,CAAC;AACpD;AAAA,IACF;AAEA,SAAK,WAAW,IAAI,OAAO,GAAG;AAC9B,SAAK,KAAK,eAAe,GAAG;AAC5B,UAAM,KAAK,eAAe,KAAK;AAAA,MAC7B,MAAM;AAAA,MACN,OAAO;AAAA,MACP,OAAO;AAAA,MACP,UAAU;AAAA,MACV,SAAS,gBAAgB,IAAI,eAAA,CAAgB;AAAA,IAAA,CAC9C;AAED,QAAI;AAEF,YAAM,iBAAiB,IAAI,QAAe,CAAC,GAAG,WAAW;AACvD,mBAAW,MAAM;AACf,iBAAO,IAAI,MAAM,qBAAqB,IAAI,OAAO,IAAI,CAAC;AAAA,QACxD,GAAG,IAAI,OAAO;AAAA,MAChB,CAAC;AAGD,YAAM,SAAS,MAAM,QAAQ,KAAK,CAAC,KAAK,WAAW,GAAG,GAAG,cAAc,CAAC;AAKxE,YAAM,kCAAkB,KAAA;AACxB,YAAM,UAAU,MAAM,KAAK,cAAc,OAAO;AAAA,QAC9C,QAAQ;AAAA,QACR,cAAc,YAAY,YAAA;AAAA,QAC1B,gBAAgB,QAAQ,iBAAiB;AAAA,QACzC,YAAY,YAAY,YAAA;AAAA,MAAY,CACrC;AAED,UAAI,SAAS;AACX,YAAI,SAAS;AACb,YAAI,cAAc;AAClB,YAAI,gBAAgB,QAAQ,iBAAiB;AAC7C,cAAM,KAAK,eAAe,KAAK;AAAA,UAC7B,MAAM;AAAA,UACN,OAAO;AAAA,UACP,OAAO;AAAA,UACP,UAAU;AAAA,UACV,SAAS,kBAAkB,IAAI,eAAA,CAAgB;AAAA,QAAA,CAChD;AACD,aAAK,KAAK,iBAAiB,KAAK,MAAM;AAAA,MACxC;AAAA,IACF,SAAS,OAAO;AACd,YAAM,KAAK,eAAe,KAAK,KAAc;AAAA,IAC/C,UAAA;AACE,WAAK,WAAW,OAAO,KAAK;AAAA,IAC9B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,MAAc,cACZ,OACA,aACkB;AAClB,QAAI,CAAC,KAAK,GAAI,QAAO;AACrB,UAAM,UAAU,OAAO,KAAK,WAAW;AACvC,UAAM,SAAS,QAAQ,IAAI,CAAC,WAAW,GAAG,MAAM,MAAM,EAAE,KAAK,IAAI;AACjE,UAAM,SAAS,QAAQ,IAAI,CAAC,WAAW,YAAY,MAAM,CAAC;AAI1D,UAAM,SAAS,MAAM,KAAK,GAAG;AAAA,MAC3B;AAAA,gBACU,MAAM;AAAA;AAAA;AAAA,MAGhB,GAAG;AAAA,MACH;AAAA,MACA,KAAK;AAAA,IAAA;AAEP,YAAQ,OAAO,MAAM,UAAU,KAAK;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,WACZ,KACuD;AACvD,UAAM,SAAS,YAGT;AAEJ,YAAM,kBAAkB,eAAe,SAAS,IAAI,UAAU;AAC9D,UAAI,CAAC,iBAAiB;AACpB,cAAM,IAAI,MAAM,wBAAwB,IAAI,UAAU,EAAE;AAAA,MAC1D;AAGA,YAAM,cAAc,gBAAgB;AAKpC,YAAM,UAAW,IAAI,QAAQ,CAAA;AAC7B,YAAM,uBAAwB,QAAQ,gBAAgB,CAAA;AAItD,YAAM,EAAE,cAAc,GAAG,aAAa,IAAI,GAAG,eAAe;AAa5D,YAAM,iBAAiB,wBAAwB,WAAW;AAC1D,YAAM,cAAc,MAAM,kBAAkB,sBAAsB;AAAA,QAChE;AAAA,QACA,SAAS;AAAA,MAAA,CACV;AAGD,UAAI;AAEJ,UAAI,IAAI,UAAU;AAEhB,mBAAW,IAAI,YAAY,EAAE,IAAI,KAAK,IAAI,GAAG,aAAa;AAC1D,cAAM,SAAS,WAAA;AACf,cACE,SACA,WAAW,IAAI,QAAQ;AAAA,MAC3B,OAAO;AAEL,mBAAW,IAAI,YAAY,EAAE,IAAI,KAAK,IAAI,GAAG,aAAa;AAC1D,cAAM,SAAS,WAAA;AAAA,MACjB;AAEA,YAAM,QAAQ,IAAI;AAClB,UAAI,CAAC,OAAO;AACV,cAAM,IAAI,MAAM,eAAe;AAAA,MACjC;AAGA,YAAM,aAAa,aAAa,IAAI;AAGpC,YAAM,gBAAgB,IAAI,iBAAiB,YAAY;AAAA,QACrD;AAAA,QACA,SAAS,IAAI;AAAA,QACb,OAAO,IAAI;AAAA,QACX,YAAY,IAAI;AAAA,QAChB,QAAQ,IAAI;AAAA,MAAA,CACb;AAGD,oBAAc,KAAK,iBAAiB,IAAI,eAAA,CAAgB,EAAE;AAC1D,YAAM,mBAAmB,KAAK,uBAAuB,KAAK,aAAa;AAGvE,YAAM,SACJ,SAOA,IAAI,MAAM;AACZ,UAAI,OAAO,WAAW,YAAY;AAChC,cAAM,IAAI,MAAM,qBAAqB,IAAI,UAAU,IAAI,IAAI,MAAM,EAAE;AAAA,MACrE;AAOA,UAAI,CAAC,2BAA2B,aAAa,IAAI,MAAM,GAAG;AACxD,cAAM,IAAI;AAAA,UACR,mCAAmC,IAAI,UAAU,IAAI,IAAI,MAAM;AAAA,QAAA;AAAA,MAEnE;AAEA,YAAM,SAAS,MAAM,OAAO,KAAK,UAAU,YAAY,gBAAgB;AAEvE,aAAO,EAAE,OAAA;AAAA,IACX;AAEA,QAAI,IAAI,UAAU;AAChB,aAAO,cAAc;AAAA,QACnB,EAAE,UAAU,IAAI,SAAA;AAAA,QAChB;AAAA,MAAA;AAAA,IAEJ;AAEA,WAAO,OAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,eAAe,KAAc,OAA6B;AACtE,UAAM,WAAW,WAAW,IAAI,aAAa;AAC7C,UAAM,WAA0B,SAAS,YAAY,IAAI,UAAU,KAAK;AAExE,UAAM,QAAQ,IAAI;AAClB,QAAI,CAAC,MAAO;AASZ,UAAM,cAAc,0BAA0B,KAAK;AAEnD,QAAI,SAAS,eAAe,IAAI,WAAW,IAAI,aAAa;AAE1D,YAAM,YAAY,IAAI,KAAK,KAAK,IAAA,IAAQ,SAAS,KAAK;AAGtD,YAAM,UAAU,MAAM,KAAK,cAAc,OAAO;AAAA,QAC9C,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,QAAQ,UAAU,YAAA;AAAA,QAClB,WAAW;AAAA,QACX,kBAAkB;AAAA,QAClB,aAAY,oBAAI,KAAA,GAAO,YAAA;AAAA,MAAY,CACpC;AACD,UAAI,CAAC,QAAS;AAEd,UAAI,SAAS;AACb,UAAI,YAAY;AAChB,UAAI,QAAQ;AACZ,UAAI,WAAW;AACf,UAAI,kBAAkB;AAEtB,YAAM,KAAK,eAAe,KAAK;AAAA,QAC7B,MAAM;AAAA,QACN,OAAO;AAAA,QACP,OAAO;AAAA,QACP,SAAS,+BAA+B,WAAW;AAAA,QACnD,MAAM,EAAE,OAAO,SAAS,OAAO,UAAU,IAAI,SAAA;AAAA,MAAS,CACvD;AACD,WAAK,KAAK,gBAAgB,KAAK,OAAO,SAAS,KAAK;AAAA,IACtD,OAAO;AAEL,YAAM,kCAAkB,KAAA;AACxB,YAAM,UAAU,MAAM,KAAK,cAAc,OAAO;AAAA,QAC9C,QAAQ;AAAA,QACR,cAAc,YAAY,YAAA;AAAA,QAC1B,YAAY;AAAA,QACZ,YAAY,YAAY,YAAA;AAAA,MAAY,CACrC;AACD,UAAI,CAAC,QAAS;AAEd,UAAI,SAAS;AACb,UAAI,cAAc;AAClB,UAAI,YAAY;AAEhB,YAAM,KAAK,eAAe,KAAK;AAAA,QAC7B,MAAM;AAAA,QACN,OAAO;AAAA,QACP,OAAO;AAAA,QACP,SAAS;AAAA,QACT,MAAM,EAAE,UAAU,IAAI,SAAA;AAAA,MAAS,CAChC;AACD,WAAK,KAAK,cAAc,KAAK,KAAK;AAAA,IACpC;AAAA,EACF;AAAA,EAEQ,uBACN,KACA,eACqB;AACrB,UAAM,aAAa;AAAA,MACjB,OAAO,IAAI,MAAM;AAAA,MACjB,UAAU,IAAI,YAAY;AAAA,MAC1B,SAAS,IAAI;AAAA,MACb,OAAO,IAAI;AAAA,MACX,YAAY,IAAI;AAAA,MAChB,QAAQ,IAAI;AAAA,IAAA;AAGd,WAAO;AAAA,MACL,KAAK;AAAA,MACL,QAAQ;AAAA,MACR,OAAO,OAAO,UAAyB;AACrC,cAAM,KAAK,eAAe,KAAK,KAAK;AAAA,MACtC;AAAA,MACA,UAAU,OAAO,UAA4B;AAC3C,cAAM,OAAO;AAAA,UACX,GAAI,MAAM,QAAQ,CAAA;AAAA,UAClB,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,OAAA,IAAW,CAAA;AAAA,UAC9C,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,OAAA,IAAW,CAAA;AAAA,QAAC;AAEjD,cAAM,KAAK,eAAe,KAAK;AAAA,UAC7B,MAAM;AAAA,UACN,OAAO;AAAA,UACP,OAAO,MAAM;AAAA,UACb,UAAU,MAAM;AAAA,UAChB,SACE,MAAM,WACN,MAAM,UACN,GAAG,MAAM,KAAK,IAAI,KAAK,MAAM,MAAM,QAAQ,CAAC;AAAA,UAC9C;AAAA,QAAA,CACD;AAAA,MACH;AAAA,MACA,KAAK,OACH,OACA,SACA,SACG;AACH,sBAAc,KAAK,EAAE,SAAS,IAAI;AAClC,cAAM,KAAK,eAAe,KAAK;AAAA,UAC7B,MAAM,UAAU,UAAU,UAAU;AAAA,UACpC;AAAA,UACA;AAAA,UACA;AAAA,QAAA,CACD;AAAA,MACH;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEA,MAAc,eACZ,KACA,OAC8B;AAC9B,QAAI,CAAC,KAAK,mBAAmB,CAAC,IAAI,IAAI;AACpC,aAAO;AAAA,IACT;AAEA,QAAI;AACF,YAAM,QAAQ,MAAM,KAAK,gBAAgB,OAAO;AAAA,QAC9C,UAAU,IAAI,YAAY;AAAA,QAC1B,OAAO,IAAI;AAAA,QACX,MAAM,MAAM,QAAQ;AAAA,QACpB,OAAO,MAAM,SAAS;AAAA,QACtB,OAAO,MAAM,SAAS;AAAA,QACtB,UAAU,MAAM,YAAY;AAAA,QAC5B,SAAS,MAAM,WAAW;AAAA,QAC1B,MAAM,MAAM,QAAQ,CAAA;AAAA,MAAC,CACtB;AAED,WAAK,KAAK,aAAa,KAAK,KAAK;AACjC,UAAI,MAAM,SAAS,YAAY;AAC7B,aAAK,KAAK,gBAAgB,KAAK,KAAK;AAAA,MACtC;AAEA,aAAO;AAAA,IACT,SAAS,OAAO;AACd,YAAM,iBACJ,iBAAiB,QACb,QACA,IAAI,MAAM,mCAAmC,OAAO,KAAK,CAAC,EAAE;AAElE,UAAI;AACF,aAAK,KAAK,gBAAgB,cAAc;AAAA,MAC1C,QAAQ;AAAA,MAER;AAEA,aAAO;AAAA,IACT;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,oBAAsC;AAClD,QAAI,KAAK,qBAAsB,QAAO;AACtC,UAAM,QAAS,MAAM,KAAK,kBAAkB,gBAAiB;AAC7D,QAAI,YAAY,uBAAuB;AACvC,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAgBA,MAAc,mBAAkC;AAC9C,QAAI,CAAC,KAAK,MAAM,CAAC,KAAK,cAAc,CAAC,KAAK,iBAAkB;AAI5D,QAAI,CAAE,MAAM,KAAK,oBAAsB;AAIvC,UAAM,MAAM,KAAK,IAAA;AACjB,QAAI,MAAM,KAAK,sBAAsB,KAAK,OAAO,YAAa;AAC9D,SAAK,sBAAsB;AAI3B,QAAI;AACF,YAAM,KAAK,iBAAiB,aAAa,KAAK,sBAAsB,EAAE;AAAA,IACxE,QAAQ;AAAA,IAER;AAEA,UAAM,iBAAiB,MAAM,KAAK,iBAAiB,qBAAA;AAGnD,UAAM,UAAU,MAAM,KAAK,WAAW;AAAA,MACpC;AAAA,MACA,CAAA;AAAA,MACA,EAAE,wBAAwB,KAAA;AAAA,IAAK;AAEjC,QAAI,QAAQ,WAAW,EAAG;AAM1B,UAAM,UAAU,QAAQ,OAAO,CAAC,QAAQ;AACtC,YAAM,QAAQ,IAAI;AAClB,UAAI,SAAS,KAAK,WAAW,IAAI,KAAK,EAAG,QAAO;AAChD,aAAO,CAAC,cAAc,IAAI,UAAU,cAAc;AAAA,IACpD,CAAC;AACD,QAAI,QAAQ,WAAW,EAAG;AAE1B,UAAM,YAAY,QACf,IAAI,CAAC,QAAQ,IAAI,EAAE,EACnB,OAAO,CAAC,UAA2B,OAAO,UAAU,QAAQ;AAC/D,QAAI,UAAU,WAAW,EAAG;AAE5B,UAAM,eAAe,UAAU,IAAI,MAAM,GAAG,EAAE,KAAK,IAAI;AACvD,UAAM,kCAAkB,KAAA;AACxB,UAAM,eACJ;AAMF,UAAM,UAAU,MAAM,KAAK,GAAG;AAAA,MAC5B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,uBAOiB,YAAY;AAAA;AAAA,MAE7B,YAAY,YAAA;AAAA,MACZ;AAAA,MACA,GAAG;AAAA,IAAA;AAEL,UAAM,eAAe,IAAI;AAAA,MACtB,QAAQ,KACN,IAAI,CAAC,QAAQ,IAAI,EAAE,EACnB,OAAO,CAAC,OAAqB,OAAO,OAAO,QAAQ;AAAA,IAAA;AAExD,QAAI,aAAa,SAAS,EAAG;AAE7B,eAAW,OAAO,SAAS;AACzB,UAAI,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,IAAI,EAAE,EAAG;AAC1C,UAAI,SAAS;AACb,UAAI,cAAc;AAClB,UAAI,YAAY;AAChB,UAAI,WAAW;AACf,UAAI,kBAAkB;AACtB,YAAM,QAAQ,IAAI,MAAM,YAAY;AACpC,YAAM,KAAK,eAAe,KAAK;AAAA,QAC7B,MAAM;AAAA,QACN,OAAO;AAAA,QACP,OAAO;AAAA,QACP,SAAS;AAAA,MAAA,CACV;AACD,WAAK,KAAK,cAAc,KAAK,KAAK;AAAA,IACpC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWQ,oBAA0B;AAChC,SAAK,aAAa,YAAY,YAAY;AACxC,UAAI;AACF,cAAM,KAAK,kBAAkB;AAAA,UAC3B,KAAK;AAAA,UACL,KAAK;AAAA,QAAA;AAAA,MAET,QAAQ;AAAA,MAER;AAAA,IACF,GAAG,KAAK,OAAO,WAAW;AAAA,EAC5B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAc,sBAAwC;AACpD,QAAI,CAAC,KAAK,GAAI,QAAO;AACrB,QAAI;AACJ,QAAI;AAGF,cACE,YACA,QAAQ,iDAAiD;AAAA,IAC7D,QAAQ;AACN,aAAO;AAAA,IACT;AAEA,QAAI;AACJ,QAAI;AACF,eAAS,IAAI,OAAO,IAAI,IAAI,KAAK,GAAG;AAAA,QAClC,YAAY;AAAA,UACV,KAAK,WAAW,KAAK,EAAE;AAAA,UACvB,MAAM,cAAc,KAAK,EAAE;AAAA,UAC3B,WAAW,KAAK;AAAA,UAChB,YAAY,KAAK;AAAA,UACjB,aAAa,KAAK,OAAO;AAAA,QAAA;AAAA,MAC3B,CACD;AAAA,IACH,QAAQ;AACN,aAAO;AAAA,IACT;AAIA,UAAM,QAAQ,MAAM,IAAI,QAAiB,CAAC,YAAY;AACpD,YAAM,YAAY,CAAC,YAAqB;AACtC,YAAI,YAAY,SAAS;AACvB,kBAAA;AACA,kBAAQ,IAAI;AAAA,QACd,WACE,WACA,OAAO,YAAY,YAClB,QAA8B,SAAS,SACxC;AACA,kBAAA;AACA,kBAAQ,KAAK;AAAA,QACf;AAAA,MACF;AACA,YAAM,SAAS,MAAM;AACnB,gBAAA;AACA,gBAAQ,KAAK;AAAA,MACf;AACA,YAAM,UAAU,MAAM;AACpB,qBAAa,KAAK;AAClB,eAAO,IAAI,WAAW,SAAS;AAC/B,eAAO,IAAI,SAAS,MAAM;AAC1B,eAAO,IAAI,QAAQ,MAAM;AAAA,MAC3B;AACA,YAAM,QAAQ,WAAW,MAAM;AAC7B,gBAAA;AACA,gBAAQ,KAAK;AAAA,MACf,GAAG,gCAAgC;AACnC,UAAI,OAAO,MAAM,UAAU,kBAAkB,MAAA;AAC7C,aAAO,GAAG,WAAW,SAAS;AAC9B,aAAO,KAAK,SAAS,MAAM;AAC3B,aAAO,KAAK,QAAQ,MAAM;AAAA,IAC5B,CAAC;AAED,QAAI,CAAC,OAAO;AACV,YAAM,OAAO,YAAY,MAAM,MAAM;AAAA,MAAC,CAAC;AACvC,aAAO;AAAA,IACT;AAEA,SAAK,iBAAiB;AAEtB,WAAO,MAAA;AAGP,WAAO,KAAK,SAAS,MAAM,KAAK,yBAAyB,MAAM,CAAC;AAChE,WAAO,KAAK,QAAQ,MAAM,KAAK,yBAAyB,MAAM,CAAC;AAC/D,WAAO;AAAA,EACT;AAAA,EAEQ,yBAAyB,QAAsB;AACrD,QAAI,KAAK,mBAAmB,OAAQ;AACpC,SAAK,iBAAiB;AACtB,QAAI,KAAK,WAAW,CAAC,KAAK,YAAY;AACpC,WAAK,kBAAA;AAAA,IACP;AAAA,EACF;AAAA;AAAA,EAGA,MAAc,qBAAoC;AAChD,UAAM,SAAS,KAAK;AACpB,QAAI,CAAC,OAAQ;AACb,SAAK,iBAAiB;AAEtB,UAAM,UAAU,IAAI,QAAc,CAAC,YAAY;AAC7C,YAAM,OAAO,MAAM;AACjB,qBAAa,KAAK;AAClB,eAAO,IAAI,WAAW,SAAS;AAC/B,gBAAA;AAAA,MACF;AACA,YAAM,YAAY,CAAC,YAAqB;AACtC,YAAI,YAAY,UAAW,MAAA;AAAA,MAC7B;AACA,aAAO,GAAG,WAAW,SAAS;AAC9B,aAAO,KAAK,QAAQ,IAAI;AACxB,YAAM,QAAQ,WAAW,MAAM,GAAI;AACnC,UAAI,OAAO,MAAM,UAAU,kBAAkB,MAAA;AAAA,IAC/C,CAAC;AACD,QAAI;AAGF,aAAO,YAAY,MAAM;AAAA,IAC3B,QAAQ;AAAA,IAER;AACA,UAAM;AACN,UAAM,OAAO,YAAY,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOQ,iBAAuB;AAC7B,SAAK,iBAAiB,YAAY,YAAY;AAC5C,UAAI,CAAC,KAAK,GAAI;AACd,YAAM,SAAS,CAAC,GAAG,KAAK,WAAW,MAAM;AACzC,UAAI,OAAO,WAAW,EAAG;AACzB,YAAM,eAAe,OAAO,IAAI,MAAM,GAAG,EAAE,KAAK,IAAI;AACpD,UAAI;AACF,cAAM,KAAK,GAAG;AAAA,UACZ;AAAA;AAAA;AAAA,2BAGiB,YAAY;AAAA,WAC7B,oBAAI,KAAA,GAAO,YAAA;AAAA,UACX,GAAG;AAAA,QAAA;AAAA,MAEP,QAAQ;AAAA,MAER;AAAA,IACF,GAAG,KAAK,OAAO,iBAAiB;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,oBAAmC;AAC/C,QAAI,KAAK,WAAW,SAAS,EAAG;AAEhC,WAAO,IAAI,QAAQ,CAAC,YAAY;AAC9B,YAAM,gBAAgB,YAAY,MAAM;AACtC,YAAI,KAAK,WAAW,SAAS,GAAG;AAC9B,wBAAc,aAAa;AAC3B,uBAAa,OAAO;AACpB,kBAAA;AAAA,QACF;AAAA,MACF,GAAG,GAAG;AAEN,YAAM,UAAU,WAAW,MAAM;AAC/B,sBAAc,aAAa;AAC3B,aAAK,OAAO;AAAA,UACV,qBAAqB,KAAK,WAAW,IAAI;AAAA,QAAA;AAE3C,gBAAA;AAAA,MACF,GAAG,KAAK,OAAO,eAAe;AAAA,IAChC,CAAC;AAAA,EACH;AACF;AAKO,SAAS,iBAAiB,QAAuC;AACtE,SAAO,IAAI,WAAW,MAAM;AAC9B;"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { detectEngine } from "@happyvertical/smrt-core";
|
|
2
|
+
import { createId } from "@happyvertical/utils";
|
|
3
|
+
const LIVE_WORKERS_KEY = "__smrtLiveWorkers";
|
|
4
|
+
function liveWorkers() {
|
|
5
|
+
const g = globalThis;
|
|
6
|
+
let set = g[LIVE_WORKERS_KEY];
|
|
7
|
+
if (!set) {
|
|
8
|
+
set = /* @__PURE__ */ new Set();
|
|
9
|
+
g[LIVE_WORKERS_KEY] = set;
|
|
10
|
+
}
|
|
11
|
+
return set;
|
|
12
|
+
}
|
|
13
|
+
function registerLiveWorker(workerKey) {
|
|
14
|
+
liveWorkers().add(workerKey);
|
|
15
|
+
}
|
|
16
|
+
function unregisterLiveWorker(workerKey) {
|
|
17
|
+
liveWorkers().delete(workerKey);
|
|
18
|
+
}
|
|
19
|
+
function isLiveWorker(workerKey) {
|
|
20
|
+
return liveWorkers().has(workerKey);
|
|
21
|
+
}
|
|
22
|
+
function isWorkerAlive(workerKey, freshLeaseKeys) {
|
|
23
|
+
if (!workerKey) return false;
|
|
24
|
+
return isLiveWorker(workerKey) || freshLeaseKeys.has(workerKey);
|
|
25
|
+
}
|
|
26
|
+
function createWorkerKey(baseId) {
|
|
27
|
+
return `${baseId}~${createId().slice(0, 8)}`;
|
|
28
|
+
}
|
|
29
|
+
function resolveUrl(db) {
|
|
30
|
+
const withConfig = db;
|
|
31
|
+
return db.url || withConfig.config?.url || "";
|
|
32
|
+
}
|
|
33
|
+
async function tuneSqliteForConcurrency(db) {
|
|
34
|
+
try {
|
|
35
|
+
await db.query("PRAGMA busy_timeout = 5000");
|
|
36
|
+
await db.query("PRAGMA journal_mode = WAL");
|
|
37
|
+
} catch {
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
function resolveEngine(db) {
|
|
41
|
+
const withConfig = db;
|
|
42
|
+
const type = withConfig.type || withConfig.config?.type;
|
|
43
|
+
return detectEngine(resolveUrl(db), type);
|
|
44
|
+
}
|
|
45
|
+
function isInMemory(db) {
|
|
46
|
+
const url = resolveUrl(db).toLowerCase();
|
|
47
|
+
return url === ":memory:" || url.includes("mode=memory") || url.includes("file::memory:");
|
|
48
|
+
}
|
|
49
|
+
function offLoopEligible(db) {
|
|
50
|
+
const engine = resolveEngine(db);
|
|
51
|
+
if (engine === "postgres") return true;
|
|
52
|
+
if (engine === "sqlite") return !isInMemory(db);
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
export {
|
|
56
|
+
resolveEngine as a,
|
|
57
|
+
resolveUrl as b,
|
|
58
|
+
createWorkerKey as c,
|
|
59
|
+
isWorkerAlive as i,
|
|
60
|
+
offLoopEligible as o,
|
|
61
|
+
registerLiveWorker as r,
|
|
62
|
+
tuneSqliteForConcurrency as t,
|
|
63
|
+
unregisterLiveWorker as u
|
|
64
|
+
};
|
|
65
|
+
//# sourceMappingURL=worker-liveness-DOTjoIjr.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-liveness-DOTjoIjr.js","sources":["../../src/worker-liveness.ts"],"sourcesContent":["import { detectEngine } from '@happyvertical/smrt-core';\nimport type { DatabaseInterface } from '@happyvertical/sql';\nimport { createId } from '@happyvertical/utils';\n\ntype DatabaseEngine = ReturnType<typeof detectEngine>;\n\n/**\n * Worker liveness primitives shared by {@link ../runner.js} and\n * {@link ../schedule-runner.js}.\n *\n * Job recovery keys on whether a job's *owning worker* is alive, never on\n * per-job heartbeat freshness (issue #1474). \"Alive\" has two independent\n * sources, combined by {@link isWorkerAlive}:\n *\n * 1. **Process-global live set** — an in-memory {@link Set} of worker keys\n * whose runner is live *in this process*. Checked synchronously, so it can\n * never be starved by a CPU-bound handler holding the event loop. Covers\n * every same-process topology (a runner's own jobs, a co-located\n * ScheduleRunner, a second in-process runner).\n * 2. **Database lease** — a `_smrt_workers` row whose `lease_expires_at` is\n * still in the future. Covers cross-process recovery. The lease is renewed\n * by the owning runner; a dead worker stops renewing and its lease expires.\n *\n * The live set takes precedence: a worker live in this process is alive even\n * if its database lease looks stale.\n *\n * The lease is compared against the recovering host's clock (the same approach\n * the previous heartbeat recovery used), so it is no more sensitive to clock\n * skew than the code it replaces. For eligible engines the lease is renewed off\n * the main loop by a worker thread (see worker-liveness-ticker.ts) so a blocked\n * handler can't starve it; otherwise it is renewed on the main loop and the\n * live set covers same-process correctness.\n */\n\nconst LIVE_WORKERS_KEY = '__smrtLiveWorkers';\n\ntype GlobalWithLiveWorkers = typeof globalThis & {\n [LIVE_WORKERS_KEY]?: Set<string>;\n};\n\nfunction liveWorkers(): Set<string> {\n const g = globalThis as GlobalWithLiveWorkers;\n let set = g[LIVE_WORKERS_KEY];\n if (!set) {\n // Singleton on globalThis so it survives HMR / duplicate module instances,\n // mirroring the ObjectRegistry pattern.\n set = new Set<string>();\n g[LIVE_WORKERS_KEY] = set;\n }\n return set;\n}\n\n/** Mark a worker key as live in this process. */\nexport function registerLiveWorker(workerKey: string): void {\n liveWorkers().add(workerKey);\n}\n\n/** Remove a worker key from this process's live set. */\nexport function unregisterLiveWorker(workerKey: string): void {\n liveWorkers().delete(workerKey);\n}\n\n/** Whether a worker key is live in this process. */\nexport function isLiveWorker(workerKey: string): boolean {\n return liveWorkers().has(workerKey);\n}\n\n/** Snapshot of every worker key live in this process. */\nexport function liveWorkerKeys(): Set<string> {\n return new Set(liveWorkers());\n}\n\n/**\n * Whether a worker is alive: live in *this* process (synchronous truth, never\n * starved), or holding a fresh database lease in some process. `null`/unknown\n * worker keys are never alive.\n */\nexport function isWorkerAlive(\n workerKey: string | null | undefined,\n freshLeaseKeys: Set<string>,\n): boolean {\n if (!workerKey) return false;\n return isLiveWorker(workerKey) || freshLeaseKeys.has(workerKey);\n}\n\n/**\n * Build a per-incarnation-unique worker key.\n *\n * Recovery treats a worker key as the unit of liveness, so the key must be\n * unique per process incarnation: a runner that crashes and restarts under the\n * same configured `id` must get a *new* key, otherwise its orphaned `running`\n * jobs would look owned by the live restart and never be recovered. We append\n * a random token to the (optional) configured id, which also keeps the\n * human-facing runner id stable for logs/events.\n */\nexport function createWorkerKey(baseId: string): string {\n return `${baseId}~${createId().slice(0, 8)}`;\n}\n\n/**\n * The connection URL, honoring adapters that leave `db.url` empty and carry the\n * real URL on `db.config.url`. Used consistently for engine detection, the\n * in-memory check, and the URL handed to the off-loop thread so they never\n * disagree.\n */\nexport function resolveUrl(db: DatabaseInterface): string {\n const withConfig = db as DatabaseInterface & { config?: { url?: string } };\n return db.url || withConfig.config?.url || '';\n}\n\n/**\n * Tune a SQLite connection for the off-loop liveness topology.\n *\n * The runner's main connection and the off-loop ticker's connection both touch\n * the same file. Stock libsql/SQLite opens in rollback-journal mode with no busy\n * timeout, so the instant two connections contend it returns `SQLITE_BUSY` —\n * surfacing as a flaky \"Failed to execute raw query\" under CI load. WAL lets\n * readers run concurrently with the single writer, and `busy_timeout` makes a\n * contended writer wait for the lock instead of failing immediately. Both are\n * idempotent and file-level (WAL persists for every connection to the file).\n *\n * Best-effort: a PRAGMA failure must never break startup — the in-process live\n * set keeps same-process recovery correct regardless. Callers gate this on a\n * SQLite engine so it is never issued to Postgres.\n */\nexport async function tuneSqliteForConcurrency(\n db: DatabaseInterface,\n): Promise<void> {\n try {\n await db.query('PRAGMA busy_timeout = 5000');\n await db.query('PRAGMA journal_mode = WAL');\n } catch {\n // Best-effort connection tuning.\n }\n}\n\n/** Resolve the database engine for a connection. */\nexport function resolveEngine(db: DatabaseInterface): DatabaseEngine {\n const withConfig = db as DatabaseInterface & {\n config?: { type?: string };\n type?: string;\n };\n const type = withConfig.type || withConfig.config?.type;\n return detectEngine(resolveUrl(db), type);\n}\n\n/**\n * Whether a connection points at an in-memory SQLite database. In-memory\n * databases are single-process (nothing to recover cross-process) and a second\n * connection cannot see the same data, so the off-loop liveness thread is\n * skipped for them.\n */\nexport function isInMemory(db: DatabaseInterface): boolean {\n const url = resolveUrl(db).toLowerCase();\n return (\n url === ':memory:' ||\n url.includes('mode=memory') ||\n url.includes('file::memory:')\n );\n}\n\n/**\n * Whether the off-loop liveness thread can run against this connection.\n *\n * Requires a second independent connection to the same data: true for Postgres\n * and file-backed SQLite. In-memory SQLite cannot be reached from another\n * connection, and DuckDB is single-writer per file — both fall back to\n * main-loop lease renewal + the in-process live set.\n */\nexport function offLoopEligible(db: DatabaseInterface): boolean {\n const engine = resolveEngine(db);\n if (engine === 'postgres') return true;\n if (engine === 'sqlite') return !isInMemory(db);\n return false;\n}\n"],"names":[],"mappings":";;AAkCA,MAAM,mBAAmB;AAMzB,SAAS,cAA2B;AAClC,QAAM,IAAI;AACV,MAAI,MAAM,EAAE,gBAAgB;AAC5B,MAAI,CAAC,KAAK;AAGR,8BAAU,IAAA;AACV,MAAE,gBAAgB,IAAI;AAAA,EACxB;AACA,SAAO;AACT;AAGO,SAAS,mBAAmB,WAAyB;AAC1D,cAAA,EAAc,IAAI,SAAS;AAC7B;AAGO,SAAS,qBAAqB,WAAyB;AAC5D,cAAA,EAAc,OAAO,SAAS;AAChC;AAGO,SAAS,aAAa,WAA4B;AACvD,SAAO,YAAA,EAAc,IAAI,SAAS;AACpC;AAYO,SAAS,cACd,WACA,gBACS;AACT,MAAI,CAAC,UAAW,QAAO;AACvB,SAAO,aAAa,SAAS,KAAK,eAAe,IAAI,SAAS;AAChE;AAYO,SAAS,gBAAgB,QAAwB;AACtD,SAAO,GAAG,MAAM,IAAI,SAAA,EAAW,MAAM,GAAG,CAAC,CAAC;AAC5C;AAQO,SAAS,WAAW,IAA+B;AACxD,QAAM,aAAa;AACnB,SAAO,GAAG,OAAO,WAAW,QAAQ,OAAO;AAC7C;AAiBA,eAAsB,yBACpB,IACe;AACf,MAAI;AACF,UAAM,GAAG,MAAM,4BAA4B;AAC3C,UAAM,GAAG,MAAM,2BAA2B;AAAA,EAC5C,QAAQ;AAAA,EAER;AACF;AAGO,SAAS,cAAc,IAAuC;AACnE,QAAM,aAAa;AAInB,QAAM,OAAO,WAAW,QAAQ,WAAW,QAAQ;AACnD,SAAO,aAAa,WAAW,EAAE,GAAG,IAAI;AAC1C;AAQO,SAAS,WAAW,IAAgC;AACzD,QAAM,MAAM,WAAW,EAAE,EAAE,YAAA;AAC3B,SACE,QAAQ,cACR,IAAI,SAAS,aAAa,KAC1B,IAAI,SAAS,eAAe;AAEhC;AAUO,SAAS,gBAAgB,IAAgC;AAC9D,QAAM,SAAS,cAAc,EAAE;AAC/B,MAAI,WAAW,WAAY,QAAO;AAClC,MAAI,WAAW,SAAU,QAAO,CAAC,WAAW,EAAE;AAC9C,SAAO;AACT;"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Redact secret-shaped substrings from a job error message before it is
|
|
3
|
+
* persisted to the `_smrt_jobs.last_error` column.
|
|
4
|
+
*
|
|
5
|
+
* @remarks
|
|
6
|
+
* `last_error` is durable and readable through generated `list`/`get` routes
|
|
7
|
+
* (now tenant-scoped — see {@link "./smrt-job".SmrtJob}). Error messages thrown
|
|
8
|
+
* from a failing job frequently echo back the arguments or environment that
|
|
9
|
+
* caused the failure: a database URL with embedded credentials, a `Bearer`
|
|
10
|
+
* token, an `Authorization` header, an API key, or a `key=value` secret pair.
|
|
11
|
+
* Persisting those verbatim turns a transient failure into a durable credential
|
|
12
|
+
* leak (S5 audit #1402).
|
|
13
|
+
*
|
|
14
|
+
* The policy biases toward over-redaction: matching a benign string is a
|
|
15
|
+
* cosmetic loss, while leaking a credential is a security incident. Patterns are
|
|
16
|
+
* intentionally conservative and order-independent — each is applied globally so
|
|
17
|
+
* multiple secrets in one message are all masked.
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Mask secret-shaped substrings in an error message.
|
|
21
|
+
*
|
|
22
|
+
* Strictly `string => string`: pass a known message here. For an arbitrary
|
|
23
|
+
* throwable of unknown shape (e.g. a caught `unknown`), call
|
|
24
|
+
* {@link redactErrorForPersistence} instead — it coerces non-`Error` values to
|
|
25
|
+
* a string first. The runtime non-string guard below is defensive depth only;
|
|
26
|
+
* the type contract is that callers supply a string.
|
|
27
|
+
*
|
|
28
|
+
* @param message - Raw error message (e.g. `error.message`).
|
|
29
|
+
* @returns The message with credential-like substrings replaced by
|
|
30
|
+
* `***REDACTED***`. Empty input is returned unchanged.
|
|
31
|
+
*
|
|
32
|
+
* @example
|
|
33
|
+
* ```ts
|
|
34
|
+
* redactErrorMessage('connect failed: postgres://u:p@db/app')
|
|
35
|
+
* // => 'connect failed: postgres://***REDACTED***@db/app'
|
|
36
|
+
* redactErrorMessage('401 from api: apiKey=<the-key>')
|
|
37
|
+
* // => '401 from api: apiKey=***REDACTED***'
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
export declare function redactErrorMessage(message: string): string;
|
|
41
|
+
/**
|
|
42
|
+
* Redact an arbitrary error's message, tolerating non-Error throwables.
|
|
43
|
+
*
|
|
44
|
+
* @param error - Anything thrown (`Error`, string, or unknown).
|
|
45
|
+
* @returns A redacted message string suitable for persistence.
|
|
46
|
+
*/
|
|
47
|
+
export declare function redactErrorForPersistence(error: unknown): string;
|
|
48
|
+
//# sourceMappingURL=error-redaction.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"error-redaction.d.ts","sourceRoot":"","sources":["../src/error-redaction.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AA0DH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAkB1D;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAKhE"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export { assertWithinTenantCreationCap, type BackgroundEligibleClass, backgroundEligible, clampRetries, DEFAULT_TENANT_JOB_CAP, getBackgroundEligibleMethods, isBackgroundEligibleMethod, MAX_JOB_RETRIES, markBackgroundEligible, TenantJobCapExceededError, } from './background-policy.js';
|
|
2
|
+
export { redactErrorForPersistence, redactErrorMessage, } from './error-redaction.js';
|
|
3
|
+
export { JobBuilder, type Priority, parseDelay, priorityToNumber, } from './job-builder.js';
|
|
4
|
+
export { JobHandle, type JobResult, type WaitOptions, } from './job-handle.js';
|
|
5
|
+
export { type JobContext, JobContextLogger, type JobEventInput, type JobExecutionContext, type JobProgressInput, } from './logger-extension.js';
|
|
6
|
+
export { type BackgroundCapable, type BgOptions, withBackgroundJobs, } from './object-extension.js';
|
|
7
|
+
export { createTaskRunner, TaskRunner, type TaskRunnerConfig, type TaskRunnerEvents, } from './runner.js';
|
|
8
|
+
export { createScheduleRunner, type ScheduleInfo, ScheduleRunner, type ScheduleRunnerConfig, type ScheduleRunnerEvents, validateCronExpression, } from './schedule-runner.js';
|
|
9
|
+
export { type ClaimReadyOptions, type JobStatus, type ListReadyOptions, SmrtJob, SmrtJobCollection, type SmrtJobData, type TimeoutBehavior, } from './smrt-job.js';
|
|
10
|
+
export { type JobEventCursor, type ListJobEventsOptions, SmrtJobEvent, SmrtJobEventCollection, type SmrtJobEventData, type SmrtJobEventLevel, type SmrtJobEventType, } from './smrt-job-event.js';
|
|
11
|
+
export { type RegisterWorkerInput, SmrtWorker, SmrtWorkerCollection, } from './smrt-worker.js';
|
|
12
|
+
export { createWorkerKey, isWorkerAlive, registerLiveWorker, unregisterLiveWorker, } from './worker-liveness.js';
|
|
13
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAKH,OAAO,wBAAwB,CAAC;AAIhC,OAAO,EACL,6BAA6B,EAC7B,KAAK,uBAAuB,EAC5B,kBAAkB,EAClB,YAAY,EACZ,sBAAsB,EACtB,4BAA4B,EAC5B,0BAA0B,EAC1B,eAAe,EACf,sBAAsB,EACtB,yBAAyB,GAC1B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EACL,yBAAyB,EACzB,kBAAkB,GACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,UAAU,EACV,KAAK,QAAQ,EACb,UAAU,EACV,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EACL,SAAS,EACT,KAAK,SAAS,EACd,KAAK,WAAW,GACjB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,KAAK,UAAU,EACf,gBAAgB,EAChB,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,gBAAgB,GACtB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,SAAS,EACd,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,GACtB,MAAM,aAAa,CAAC;AAGrB,OAAO,EACL,oBAAoB,EACpB,KAAK,YAAY,EACjB,cAAc,EACd,KAAK,oBAAoB,EACzB,KAAK,oBAAoB,EACzB,sBAAsB,GACvB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,SAAS,EACd,KAAK,gBAAgB,EACrB,OAAO,EACP,iBAAiB,EACjB,KAAK,WAAW,EAChB,KAAK,eAAe,GACrB,MAAM,eAAe,CAAC;AAGvB,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,oBAAoB,EACzB,YAAY,EACZ,sBAAsB,EACtB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,GACtB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EACL,KAAK,mBAAmB,EACxB,UAAU,EACV,oBAAoB,GACrB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,sBAAsB,CAAC"}
|