@plitzi/sdk-server 0.37.1 → 0.37.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1296 -0
- package/README.md +126 -7
- package/dist/actions.d.ts +20 -8
- package/dist/actions.js +11 -4
- package/dist/adapters/cloudAdapters.js +4 -3
- package/dist/adapters/jsonAdapters.js +28 -6
- package/dist/auth.d.ts +7 -4
- package/dist/auth.js +8 -6
- package/dist/core/auth/api.d.ts +29 -1
- package/dist/core/auth/api.js +47 -13
- package/dist/core/auth/createAuth.d.ts +4 -0
- package/dist/core/auth/createAuth.js +3 -3
- package/dist/core/auth/credentials.d.ts +3 -0
- package/dist/core/auth/credentials.js +1 -1
- package/dist/core/auth/csrf.d.ts +25 -1
- package/dist/core/auth/csrf.js +29 -4
- package/dist/core/auth/renewal.d.ts +26 -0
- package/dist/core/auth/renewal.js +24 -0
- package/dist/core/auth/routes.js +8 -6
- package/dist/core/auth/session.d.ts +45 -0
- package/dist/core/auth/session.js +47 -1
- package/dist/core/auth/sessionActivity.d.ts +11 -0
- package/dist/core/auth/sessionActivity.js +14 -0
- package/dist/core/auth/throttle.d.ts +15 -0
- package/dist/core/auth/throttle.js +29 -1
- package/dist/core/auth/totp.d.ts +9 -1
- package/dist/core/auth/totp.js +11 -6
- package/dist/core/createServer.actions.test.d.ts +1 -0
- package/dist/core/createServer.d.ts +14 -1
- package/dist/core/createServer.functions.test.d.ts +1 -0
- package/dist/core/createServer.js +22 -12
- package/dist/core/fontStore.js +1 -1
- package/dist/core/handlers/authRouteHandlers.d.ts +1 -1
- package/dist/core/handlers/authRouteHandlers.js +57 -21
- package/dist/core/handlers/csrfMiddleware.d.ts +23 -0
- package/dist/core/handlers/csrfMiddleware.js +31 -7
- package/dist/core/handlers/csrfMiddleware.test.d.ts +1 -0
- package/dist/core/handlers/oauthRouteHandlers.js +5 -3
- package/dist/core/handlers/socialRouteHandlers.d.ts +1 -1
- package/dist/core/handlers/socialRouteHandlers.js +2 -1
- package/dist/core/http/dispatcher.d.ts +9 -1
- package/dist/core/http/dispatcher.js +38 -11
- package/dist/core/http/navigation.d.ts +5 -2
- package/dist/core/http/navigation.js +7 -4
- package/dist/core/http/socketResponse.d.ts +18 -0
- package/dist/core/http/socketResponse.js +59 -0
- package/dist/core/http/sse.d.ts +26 -0
- package/dist/core/http/sse.js +78 -0
- package/dist/core/http/stages/authRoutes.js +3 -1
- package/dist/core/http/stages/exchangeRoute.test.d.ts +1 -0
- package/dist/core/http/stages/middlewares.d.ts +1 -1
- package/dist/core/http/stages/middlewares.js +16 -10
- package/dist/core/http/stages/middlewares.test.d.ts +1 -0
- package/dist/core/http/stages/oauth.js +2 -2
- package/dist/core/http/stages/sessionRenewal.d.ts +16 -0
- package/dist/core/http/stages/sessionRenewal.js +81 -0
- package/dist/core/http/stages/sessionRenewal.test.d.ts +1 -0
- package/dist/core/http/stages/signInRoute.d.ts +2 -0
- package/dist/core/http/stages/signInRoute.js +139 -0
- package/dist/core/http/stages/signInRoute.test.d.ts +1 -0
- package/dist/core/http/types.d.ts +9 -0
- package/dist/core/http/webExchange.d.ts +18 -0
- package/dist/core/http/webExchange.js +53 -0
- package/dist/core/oauth/authorize.js +37 -13
- package/dist/core/oauth/consentPage.js +11 -0
- package/dist/core/oauth/issueContext.d.ts +14 -0
- package/dist/core/oauth/issueContext.js +32 -0
- package/dist/core/oauth/records.d.ts +7 -0
- package/dist/core/oauth/register.d.ts +2 -0
- package/dist/core/oauth/register.js +32 -11
- package/dist/core/oauth/register.test.d.ts +1 -0
- package/dist/core/oauth/token.js +21 -6
- package/dist/core/oauth/token.test.d.ts +1 -0
- package/dist/core/requestParser.d.ts +4 -0
- package/dist/core/requestParser.js +14 -7
- package/dist/core/sdkAssets.js +1 -1
- package/dist/core/server/baseServer.d.ts +14 -1
- package/dist/core/server/baseServer.js +43 -11
- package/dist/core/server/closeOnSignals.d.ts +28 -0
- package/dist/core/server/closeOnSignals.js +47 -0
- package/dist/core/server/closeOnSignals.test.d.ts +1 -0
- package/dist/core/server/drain.d.ts +18 -0
- package/dist/core/server/drain.js +62 -0
- package/dist/core/server/drain.test.d.ts +1 -0
- package/dist/core/server/fleet/channel.d.ts +103 -0
- package/dist/core/server/fleet/channel.js +191 -0
- package/dist/core/server/fleet/channel.test.d.ts +1 -0
- package/dist/core/server/fleet/fixtures/fleetServer.d.ts +1 -0
- package/dist/core/server/fleet/fleet.integration.test.d.ts +1 -0
- package/dist/core/server/fleet/link.d.ts +23 -0
- package/dist/core/server/fleet/link.js +70 -0
- package/dist/core/server/fleet/relay.d.ts +11 -0
- package/dist/core/server/fleet/relay.js +104 -0
- package/dist/core/server/fleet/relay.test.d.ts +1 -0
- package/dist/core/server/fleet/role.d.ts +46 -0
- package/dist/core/server/fleet/role.js +59 -0
- package/dist/core/server/fleet/role.test.d.ts +1 -0
- package/dist/core/server/fleet/stores.d.ts +7 -0
- package/dist/core/server/fleet/stores.js +19 -0
- package/dist/core/server/fleet/supervisor.d.ts +26 -0
- package/dist/core/server/fleet/supervisor.js +108 -0
- package/dist/core/server/fleet/supervisor.test.d.ts +1 -0
- package/dist/core/server/pageServer.js +36 -5
- package/dist/core/services/action.js +10 -2
- package/dist/core/services/functionRoutes.d.ts +9 -0
- package/dist/core/services/functionRoutes.js +47 -0
- package/dist/core/services/realtime.d.ts +8 -0
- package/dist/core/services/realtime.js +57 -0
- package/dist/core/services/registry.js +6 -2
- package/dist/core/staticFiles.js +12 -4
- package/dist/core/staticFiles.test.d.ts +1 -0
- package/dist/core/transports.d.ts +13 -3
- package/dist/core/transports.js +14 -5
- package/dist/functions-api.d.ts +281 -0
- package/dist/functions.d.ts +19 -0
- package/dist/functions.js +5 -0
- package/dist/functionsRunner.d.ts +15 -0
- package/dist/functionsRunner.js +6 -0
- package/dist/handlers.d.ts +2 -2
- package/dist/handlers.js +2 -2
- package/dist/helpers/atomicFile.d.ts +10 -0
- package/dist/helpers/atomicFile.js +33 -0
- package/dist/helpers/atomicFile.test.d.ts +1 -0
- package/dist/helpers/buildResponseHelpers.js +35 -11
- package/dist/helpers/buildServerInfo.js +4 -2
- package/dist/helpers/buildServerInfo.test.d.ts +1 -0
- package/dist/helpers/cache/CompressedFileCache.d.ts +18 -0
- package/dist/helpers/cache/CompressedFileCache.js +44 -0
- package/dist/helpers/cache/CompressedFileCache.test.d.ts +1 -0
- package/dist/helpers/cache/cacheManager.d.ts +1 -1
- package/dist/helpers/cache/cacheManager.js +2 -1
- package/dist/helpers/cache/cacheManager.test.d.ts +1 -0
- package/dist/helpers/cache/index.d.ts +1 -1
- package/dist/helpers/cache/keys.d.ts +15 -1
- package/dist/helpers/cache/keys.js +47 -5
- package/dist/helpers/cache/serverCaches.d.ts +7 -1
- package/dist/helpers/compress.d.ts +3 -1
- package/dist/helpers/compress.js +11 -3
- package/dist/helpers/hydrationPayload.d.ts +11 -0
- package/dist/helpers/hydrationPayload.js +51 -0
- package/dist/helpers/hydrationPayload.test.d.ts +1 -0
- package/dist/helpers/metrics.d.ts +1 -1
- package/dist/helpers/metrics.js +3 -2
- package/dist/helpers/outboundGuard.d.ts +14 -6
- package/dist/helpers/outboundGuard.js +132 -8
- package/dist/helpers/paintedCookie.d.ts +8 -0
- package/dist/helpers/paintedCookie.js +13 -0
- package/dist/helpers/reportReactBuild.d.ts +9 -0
- package/dist/helpers/reportReactBuild.js +18 -0
- package/dist/helpers/reportReactBuild.test.d.ts +1 -0
- package/dist/helpers/serverLog.d.ts +25 -1
- package/dist/helpers/serverLog.js +68 -3
- package/dist/helpers/serverLog.test.d.ts +1 -0
- package/dist/helpers/wsFrame.d.ts +3 -0
- package/dist/helpers/wsFrame.js +5 -0
- package/dist/index.d.ts +8 -1
- package/dist/index.js +6 -2
- package/dist/kernel.d.ts +2 -1
- package/dist/kernel.js +3 -2
- package/dist/modules/actions/connectorWrite.js +2 -1
- package/dist/modules/actions/index.d.ts +41 -13
- package/dist/modules/actions/index.js +52 -9
- package/dist/modules/actions/jobs/failureStreak.d.ts +17 -0
- package/dist/modules/actions/jobs/failureStreak.js +47 -0
- package/dist/modules/actions/jobs/failureStreak.test.d.ts +1 -0
- package/dist/modules/actions/jobs/index.d.ts +33 -0
- package/dist/modules/actions/jobs/index.js +59 -0
- package/dist/modules/actions/jobs/memoryQueue.d.ts +33 -0
- package/dist/modules/actions/jobs/memoryQueue.js +194 -0
- package/dist/modules/actions/jobs/memoryStores.test.d.ts +1 -0
- package/dist/modules/actions/jobs/scheduler.d.ts +64 -0
- package/dist/modules/actions/jobs/scheduler.js +130 -0
- package/dist/modules/actions/jobs/scheduler.test.d.ts +1 -0
- package/dist/modules/actions/jobs/schedules.d.ts +23 -0
- package/dist/modules/actions/jobs/schedules.js +47 -0
- package/dist/modules/actions/jobs/storeClock.d.ts +16 -0
- package/dist/modules/actions/jobs/storeClock.js +32 -0
- package/dist/modules/actions/jobs/testing/jobQueueContract.d.ts +25 -0
- package/dist/modules/actions/jobs/worker.d.ts +64 -0
- package/dist/modules/actions/jobs/worker.js +267 -0
- package/dist/modules/actions/jobs/worker.test.d.ts +1 -0
- package/dist/modules/actions/moduleFor.js +6 -2
- package/dist/modules/actions/runtime/errors.d.ts +12 -0
- package/dist/modules/actions/runtime/errors.js +16 -1
- package/dist/modules/actions/runtime/kvChange.d.ts +27 -0
- package/dist/modules/actions/runtime/kvChange.js +23 -0
- package/dist/modules/actions/runtime/kvList.d.ts +51 -0
- package/dist/modules/actions/runtime/kvList.js +52 -0
- package/dist/modules/actions/runtime/kvStore.d.ts +8 -0
- package/dist/modules/actions/runtime/kvStore.js +46 -20
- package/dist/modules/actions/runtime/memoryKv.d.ts +3 -0
- package/dist/modules/actions/runtime/memoryKv.js +19 -1
- package/dist/modules/actions/runtime/namespaceKv.d.ts +4 -6
- package/dist/modules/actions/runtime/namespaceKv.js +15 -3
- package/dist/modules/actions/runtime/namespaceKv.test.d.ts +1 -0
- package/dist/modules/actions/runtime/precheck.js +5 -9
- package/dist/modules/actions/runtime/rateLimit.d.ts +19 -0
- package/dist/modules/actions/runtime/rateLimit.js +19 -0
- package/dist/modules/actions/runtime/redisKv.d.ts +27 -0
- package/dist/modules/actions/runtime/redisKv.integration.test.d.ts +1 -0
- package/dist/modules/actions/runtime/redisKv.js +46 -0
- package/dist/modules/actions/runtime/renderResolver.js +2 -2
- package/dist/modules/actions/runtime/report.js +2 -1
- package/dist/modules/actions/runtime/runAction.d.ts +10 -2
- package/dist/modules/actions/runtime/runAction.js +52 -116
- package/dist/modules/actions/runtime/runLogger.d.ts +5 -0
- package/dist/modules/actions/runtime/runLogger.js +8 -2
- package/dist/modules/actions/runtime/signing.d.ts +20 -0
- package/dist/modules/actions/runtime/signing.js +25 -0
- package/dist/modules/actions/runtime/signing.test.d.ts +1 -0
- package/dist/modules/actions/runtime/taskContext.d.ts +33 -0
- package/dist/modules/actions/runtime/taskContext.js +124 -0
- package/dist/modules/actions/tasks/builtins.js +5 -1
- package/dist/modules/actions/tasks/flow.js +81 -10
- package/dist/modules/actions/tasks/helpers.js +2 -2
- package/dist/modules/actions/tasks/http.js +2 -3
- package/dist/modules/actions/tasks/kv.js +39 -0
- package/dist/modules/actions/tasks/list.d.ts +2 -0
- package/dist/modules/actions/tasks/list.js +145 -0
- package/dist/modules/actions/tasks/realtime.d.ts +2 -0
- package/dist/modules/actions/tasks/realtime.js +104 -0
- package/dist/modules/actions/tasks/registry.d.ts +10 -1
- package/dist/modules/actions/tasks/registry.js +17 -6
- package/dist/modules/actions/transport/callHandler.js +6 -3
- package/dist/modules/actions/transport/catalogHandler.d.ts +5 -2
- package/dist/modules/actions/transport/catalogHandler.js +3 -2
- package/dist/modules/actions/transport/stream.d.ts +4 -5
- package/dist/modules/actions/transport/stream.js +12 -76
- package/dist/modules/actions/transport/webhookHandler.js +5 -4
- package/dist/modules/actions/types.d.ts +162 -4
- package/dist/modules/connectors/engine.js +3 -5
- package/dist/modules/functions/build.d.ts +22 -0
- package/dist/modules/functions/build.js +122 -0
- package/dist/modules/functions/build.test.d.ts +1 -0
- package/dist/modules/functions/capabilities.d.ts +22 -0
- package/dist/modules/functions/capabilities.js +228 -0
- package/dist/modules/functions/config.d.ts +34 -0
- package/dist/modules/functions/config.js +24 -0
- package/dist/modules/functions/context.d.ts +17 -0
- package/dist/modules/functions/context.js +88 -0
- package/dist/modules/functions/contract.d.ts +121 -0
- package/dist/modules/functions/contract.js +9 -0
- package/dist/modules/functions/driver.d.ts +36 -0
- package/dist/modules/functions/driver.js +150 -0
- package/dist/modules/functions/load.d.ts +15 -0
- package/dist/modules/functions/load.js +45 -0
- package/dist/modules/functions/load.test.d.ts +1 -0
- package/dist/modules/functions/local.d.ts +29 -0
- package/dist/modules/functions/local.js +61 -0
- package/dist/modules/functions/local.test.d.ts +1 -0
- package/dist/modules/functions/manifest.d.ts +24 -0
- package/dist/modules/functions/manifest.js +163 -0
- package/dist/modules/functions/manifest.test.d.ts +1 -0
- package/dist/modules/functions/native.d.ts +8 -0
- package/dist/modules/functions/native.js +21 -0
- package/dist/modules/functions/native.test.d.ts +1 -0
- package/dist/modules/functions/protocol.d.ts +216 -0
- package/dist/modules/functions/protocol.js +18 -0
- package/dist/modules/functions/routes.d.ts +36 -0
- package/dist/modules/functions/routes.js +181 -0
- package/dist/modules/functions/routes.test.d.ts +1 -0
- package/dist/modules/functions/runner/messages.d.ts +5 -0
- package/dist/modules/functions/runner/messages.js +188 -0
- package/dist/modules/functions/runner/remote.d.ts +19 -0
- package/dist/modules/functions/runner/remote.js +116 -0
- package/dist/modules/functions/runner/remote.test.d.ts +1 -0
- package/dist/modules/functions/runner/service.d.ts +32 -0
- package/dist/modules/functions/runner/service.js +209 -0
- package/dist/modules/functions/sandbox/guest.d.ts +45 -0
- package/dist/modules/functions/sandbox/guest.js +660 -0
- package/dist/modules/functions/sandbox/isolate.d.ts +37 -0
- package/dist/modules/functions/sandbox/isolate.js +347 -0
- package/dist/modules/functions/sandbox/isolate.test.d.ts +1 -0
- package/dist/modules/functions/sandbox/prelude.d.ts +2 -0
- package/dist/modules/functions/sandbox/prelude.js +49 -0
- package/dist/modules/functions/space.d.ts +40 -0
- package/dist/modules/functions/space.js +162 -0
- package/dist/modules/functions/testing/tasksOf.d.ts +4 -0
- package/dist/modules/functions/tryEntry.d.ts +9 -0
- package/dist/modules/functions/tryEntry.js +49 -0
- package/dist/modules/mongo/index.d.ts +4 -0
- package/dist/modules/mongo/jobQueue.d.ts +38 -0
- package/dist/modules/mongo/jobQueue.js +335 -0
- package/dist/modules/mongo/kv.d.ts +15 -0
- package/dist/modules/mongo/kv.js +129 -0
- package/dist/modules/mongo/stores.integration.test.d.ts +1 -0
- package/dist/modules/mysql/accounts.js +19 -9
- package/dist/modules/mysql/index.d.ts +2 -1
- package/dist/modules/mysql/jobs/index.d.ts +5 -0
- package/dist/modules/mysql/jobs/jobQueue.d.ts +40 -0
- package/dist/modules/mysql/jobs/jobQueue.js +375 -0
- package/dist/modules/mysql/jobs/kv.d.ts +18 -0
- package/dist/modules/mysql/jobs/kv.js +108 -0
- package/dist/modules/mysql/jobs/schema.d.ts +33 -0
- package/dist/modules/mysql/jobs/schema.js +113 -0
- package/dist/modules/mysql/jobs/stores.integration.test.d.ts +1 -0
- package/dist/modules/mysql/schema.js +25 -0
- package/dist/modules/mysql/sessions.d.ts +1 -11
- package/dist/modules/mysql/sessions.js +6 -1
- package/dist/modules/realtime/declarations.d.ts +11 -0
- package/dist/modules/realtime/declarations.js +26 -0
- package/dist/modules/realtime/grants.d.ts +23 -0
- package/dist/modules/realtime/grants.js +61 -0
- package/dist/modules/realtime/handlers.d.ts +74 -0
- package/dist/modules/realtime/handlers.js +224 -0
- package/dist/modules/realtime/hub.d.ts +62 -0
- package/dist/modules/realtime/hub.js +141 -0
- package/dist/modules/realtime/index.d.ts +51 -0
- package/dist/modules/realtime/index.js +62 -0
- package/dist/modules/realtime/memoryPubSub.d.ts +10 -0
- package/dist/modules/realtime/memoryPubSub.js +43 -0
- package/dist/modules/realtime/realtime.test.d.ts +1 -0
- package/dist/modules/realtime/redisPubSub.d.ts +24 -0
- package/dist/modules/realtime/redisPubSub.js +36 -0
- package/dist/modules/realtime/redisPubSub.test.d.ts +1 -0
- package/dist/modules/realtime/socket.d.ts +25 -0
- package/dist/modules/realtime/socket.js +122 -0
- package/dist/modules/realtime/socket.test.d.ts +1 -0
- package/dist/modules/rsc/handler.d.ts +16 -0
- package/dist/modules/rsc/handler.js +17 -10
- package/dist/modules/rsc/handler.test.d.ts +1 -0
- package/dist/modules/rsc/resolveRscData.js +2 -1
- package/dist/modules/runtime/bundle.d.ts +24 -0
- package/dist/modules/runtime/bundle.js +102 -0
- package/dist/modules/runtime/contract.d.ts +37 -0
- package/dist/modules/runtime/contract.js +21 -0
- package/dist/modules/runtime/entry.test.d.ts +1 -0
- package/dist/modules/runtime/host.d.ts +38 -0
- package/dist/modules/runtime/host.js +68 -0
- package/dist/modules/runtime/inside.d.ts +15 -0
- package/dist/modules/runtime/inside.js +47 -0
- package/dist/modules/runtime/inside.test.d.ts +1 -0
- package/dist/modules/runtime/runner.d.ts +11 -0
- package/dist/modules/runtime/runner.js +38 -0
- package/dist/modules/runtime/runtime.test.d.ts +1 -0
- package/dist/modules/runtime/stages.d.ts +50 -0
- package/dist/modules/runtime/stages.js +117 -0
- package/dist/modules/ssr/Component.d.ts +4 -5
- package/dist/modules/ssr/Component.js +2 -2
- package/dist/modules/ssr/handler.js +10 -2
- package/dist/modules/ssr/loadPluginComponents.js +2 -1
- package/dist/modules/ssr/prepareRender.js +39 -13
- package/dist/modules/ssr/preview.d.ts +3 -0
- package/dist/modules/ssr/preview.js +17 -7
- package/dist/modules/ssr/redirect.test.d.ts +1 -0
- package/dist/modules/ssr/registerExternalPlugins.d.ts +6 -1
- package/dist/modules/ssr/registerExternalPlugins.js +19 -16
- package/dist/modules/ssr/registerExternalPlugins.test.d.ts +1 -0
- package/dist/modules/ssr/reportMissingPlugins.d.ts +9 -0
- package/dist/modules/ssr/reportMissingPlugins.js +24 -0
- package/dist/modules/ssr/reportMissingPlugins.test.d.ts +1 -0
- package/dist/modules/ssr/streamBody.d.ts +2 -2
- package/dist/modules/ssr/streamBody.js +4 -1
- package/dist/modules/ssr/template.js +1 -1
- package/dist/modules/ssr/views/template.ejs +8 -5
- package/dist/mongo.d.ts +25 -0
- package/dist/mongo.js +3 -0
- package/dist/mysql.d.ts +11 -0
- package/dist/mysql.js +4 -1
- package/dist/oauth.js +2 -2
- package/dist/plugins/compile.js +9 -4
- package/dist/plugins/copy.js +3 -3
- package/dist/plugins/manager.d.ts +35 -3
- package/dist/plugins/manager.js +109 -38
- package/dist/runtime.d.ts +15 -0
- package/dist/runtime.js +7 -0
- package/package.json +52 -11
- package/skills/plitzi-authoring/SKILL.md +153 -263
- package/skills/plitzi-authoring/reference/accessibility.md +111 -0
- package/skills/plitzi-authoring/reference/authoring-errors.md +70 -0
- package/skills/plitzi-authoring/reference/data-and-visibility.md +180 -0
- package/skills/plitzi-authoring/reference/elements-and-styles.md +161 -0
- package/skills/plitzi-authoring/reference/flows.md +171 -0
- package/skills/plitzi-authoring/reference/layouts.md +140 -0
- package/skills/plitzi-authoring/reference/lists.md +74 -0
- package/skills/plitzi-authoring/reference/performance.md +67 -0
- package/skills/plitzi-authoring/reference/plugins.md +167 -0
- package/skills/plitzi-authoring/reference/review-checklist.md +67 -0
- package/skills/plitzi-authoring/reference/structure.md +83 -0
- package/skills/plitzi-authoring/reference/templates-and-export.md +54 -0
- package/skills/plitzi-authoring/reference/templates.md +109 -0
- package/skills/plitzi-authoring/reference/testing.md +119 -0
- package/skills/plitzi-authoring/reference/validation.md +73 -0
- package/dist/modules/actions/runtime/schedule.d.ts +0 -35
- package/dist/modules/actions/runtime/schedule.js +0 -76
- /package/dist/{modules/actions/runtime/schedule.test.d.ts → adapters/jsonAdapters.test.d.ts} +0 -0
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
import { execute, selectOne, selectRows } from "../query.js";
|
|
2
|
+
import { NOW_MS, ensureJobTables, jobTables } from "./schema.js";
|
|
3
|
+
import { createStoreClock } from "../../actions/jobs/storeClock.js";
|
|
4
|
+
//#region src/modules/mysql/jobs/jobQueue.ts
|
|
5
|
+
var TERMINAL = [
|
|
6
|
+
"succeeded",
|
|
7
|
+
"failed",
|
|
8
|
+
"dead",
|
|
9
|
+
"cancelled"
|
|
10
|
+
];
|
|
11
|
+
var DAY_MS = 864e5;
|
|
12
|
+
/** How often finished jobs past their retention are removed — at most once a minute per process, on a write. */
|
|
13
|
+
var PURGE_EVERY_MS = 6e4;
|
|
14
|
+
/**
|
|
15
|
+
* A row as the contract shapes it.
|
|
16
|
+
*
|
|
17
|
+
* `BIGINT` arrives as a number or as a string depending on how the deployment configured its pool
|
|
18
|
+
* (`bigNumberStrings`), so every numeric column goes through `Number`. The text columns hold only what this queue
|
|
19
|
+
* wrote into them — a status, an environment, a trigger — which is why they can be read back as those types.
|
|
20
|
+
*/
|
|
21
|
+
var toJob = (row) => ({
|
|
22
|
+
id: row.id,
|
|
23
|
+
spaceId: Number(row.space_id),
|
|
24
|
+
actionId: row.action_id,
|
|
25
|
+
environment: row.environment,
|
|
26
|
+
trigger: row.trigger_type,
|
|
27
|
+
input: JSON.parse(row.input),
|
|
28
|
+
dueAt: Number(row.due_at),
|
|
29
|
+
maxAttempts: Number(row.max_attempts),
|
|
30
|
+
...row.missed === null ? {} : { missed: Number(row.missed) },
|
|
31
|
+
status: row.status,
|
|
32
|
+
runAt: Number(row.run_at),
|
|
33
|
+
attempts: Number(row.attempts),
|
|
34
|
+
...row.lease_until === null ? {} : { leaseUntil: Number(row.lease_until) },
|
|
35
|
+
...row.worker_id === null ? {} : { workerId: row.worker_id },
|
|
36
|
+
...row.run_id === null ? {} : { runId: row.run_id },
|
|
37
|
+
...Number(row.cancel_requested) === 1 ? { cancelRequested: true } : {},
|
|
38
|
+
createdAt: Number(row.created_at),
|
|
39
|
+
updatedAt: Number(row.updated_at),
|
|
40
|
+
...row.error === null ? {} : { error: row.error },
|
|
41
|
+
history: JSON.parse(row.history)
|
|
42
|
+
});
|
|
43
|
+
var toSchedule = (row) => ({
|
|
44
|
+
spaceId: Number(row.space_id),
|
|
45
|
+
actionId: row.action_id,
|
|
46
|
+
cron: row.cron,
|
|
47
|
+
...row.timezone === null ? {} : { timezone: row.timezone },
|
|
48
|
+
environment: row.environment,
|
|
49
|
+
enabled: Number(row.enabled) === 1,
|
|
50
|
+
nextRunAt: Number(row.next_run_at),
|
|
51
|
+
...row.last_fire_at === null ? {} : { lastFireAt: Number(row.last_fire_at) },
|
|
52
|
+
missed: Number(row.missed),
|
|
53
|
+
maxAttempts: Number(row.max_attempts),
|
|
54
|
+
updatedAt: Number(row.updated_at)
|
|
55
|
+
});
|
|
56
|
+
/** MySQL's answer when two transactions wait on each other: one is rolled back, and is meant to be run again. */
|
|
57
|
+
var DEADLOCK = "ER_LOCK_DEADLOCK";
|
|
58
|
+
var TRANSACTION_ATTEMPTS = 3;
|
|
59
|
+
/** Claim rounds per call: a worker that lost every row it read to other workers looks once more, then waits a poll. */
|
|
60
|
+
var CLAIM_ROUNDS = 3;
|
|
61
|
+
/**
|
|
62
|
+
* One connection, one transaction: what a settlement or a schedule write needs so the rows it read stay locked until
|
|
63
|
+
* it writes. A deadlock is retried — MySQL picks a victim and expects it to try again, which is what two replicas
|
|
64
|
+
* reconciling the same space at boot will occasionally make it do.
|
|
65
|
+
*/
|
|
66
|
+
var transaction = async (pool, work) => {
|
|
67
|
+
for (let attempt = 1;; attempt += 1) {
|
|
68
|
+
const connection = await pool.getConnection();
|
|
69
|
+
try {
|
|
70
|
+
await connection.beginTransaction();
|
|
71
|
+
const result = await work(connection);
|
|
72
|
+
await connection.commit();
|
|
73
|
+
return result;
|
|
74
|
+
} catch (error) {
|
|
75
|
+
await connection.rollback();
|
|
76
|
+
if (error.code !== DEADLOCK || attempt >= TRANSACTION_ATTEMPTS) throw error;
|
|
77
|
+
} finally {
|
|
78
|
+
connection.release();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
/** MySQL's code for a primary key that refused a second row with the same value. */
|
|
83
|
+
var DUPLICATE_ENTRY = "ER_DUP_ENTRY";
|
|
84
|
+
/**
|
|
85
|
+
* An {@link ActionJobQueue} over tables in a MySQL database the deployment owns. Tested on MySQL 8; nothing in it
|
|
86
|
+
* needs more than 5.7.
|
|
87
|
+
*
|
|
88
|
+
* The helper for a deployment that already runs MySQL: it borrows connections from the pool it is handed, and creates
|
|
89
|
+
* its three tables on first use unless told not to. Every replica pointed at the same database shares the jobs and the
|
|
90
|
+
* schedules.
|
|
91
|
+
*
|
|
92
|
+
* ```ts
|
|
93
|
+
* import mysql from 'mysql2/promise';
|
|
94
|
+
* import { createMysqlJobQueue, createMysqlKv } from '@plitzi/sdk-server/mysql';
|
|
95
|
+
*
|
|
96
|
+
* const pool = mysql.createPool(process.env.DATABASE_URL);
|
|
97
|
+
* createServer({ action: { lookups, kv: createMysqlKv({ pool }), jobs: { queue: createMysqlJobQueue({ pool }) } } });
|
|
98
|
+
* ```
|
|
99
|
+
*
|
|
100
|
+
* The contract's rules, as MySQL keeps them: every instant is the SERVER's (`NOW(3)`), `enqueue` is idempotent by the
|
|
101
|
+
* primary key, `claim` takes each job with a single-row compare-and-set — so two workers never take the same one —
|
|
102
|
+
* and reaps a lapsed lease in the same write, and a schedule advances by compare-and-set too.
|
|
103
|
+
*/
|
|
104
|
+
var createMysqlJobQueue = ({ pool, tablePrefix = "", createTables = true, retentionDays = 90 }) => {
|
|
105
|
+
const t = jobTables(tablePrefix);
|
|
106
|
+
const ready = ensureJobTables(pool, tablePrefix, createTables);
|
|
107
|
+
const clock = createStoreClock(async () => {
|
|
108
|
+
const row = await selectOne(pool, `SELECT ${NOW_MS} AS now`);
|
|
109
|
+
return row ? Number(row.now) : Date.now();
|
|
110
|
+
});
|
|
111
|
+
const db = async () => {
|
|
112
|
+
await ready();
|
|
113
|
+
return pool;
|
|
114
|
+
};
|
|
115
|
+
const expiryOf = (at) => at + retentionDays * DAY_MS;
|
|
116
|
+
let purgedAt = 0;
|
|
117
|
+
const purge = async (at) => {
|
|
118
|
+
if (at - purgedAt < PURGE_EVERY_MS) return;
|
|
119
|
+
purgedAt = at;
|
|
120
|
+
await execute(pool, `DELETE FROM ${t.jobs} WHERE expires_at IS NOT NULL AND expires_at < ? LIMIT 1000`, [at]);
|
|
121
|
+
};
|
|
122
|
+
const inSpaces = (spaceIds) => spaceIds.length > 0;
|
|
123
|
+
return {
|
|
124
|
+
resyncClock: clock.resync,
|
|
125
|
+
now: async () => new Date(await clock.now()),
|
|
126
|
+
enqueue: async (job) => {
|
|
127
|
+
const conn = await db();
|
|
128
|
+
const at = await clock.now();
|
|
129
|
+
try {
|
|
130
|
+
await execute(conn, `INSERT INTO ${t.jobs} (id, space_id, action_id, environment, trigger_type, input, due_at, max_attempts,
|
|
131
|
+
missed, status, run_at, attempts, created_at, updated_at, history)
|
|
132
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, 0, ?, ?, '[]')`, [
|
|
133
|
+
job.id,
|
|
134
|
+
job.spaceId,
|
|
135
|
+
job.actionId,
|
|
136
|
+
job.environment,
|
|
137
|
+
job.trigger,
|
|
138
|
+
JSON.stringify(job.input),
|
|
139
|
+
job.dueAt,
|
|
140
|
+
job.maxAttempts,
|
|
141
|
+
job.missed ?? null,
|
|
142
|
+
job.dueAt,
|
|
143
|
+
at,
|
|
144
|
+
at
|
|
145
|
+
]);
|
|
146
|
+
return true;
|
|
147
|
+
} catch (error) {
|
|
148
|
+
if (error.code === DUPLICATE_ENTRY) return false;
|
|
149
|
+
throw error;
|
|
150
|
+
}
|
|
151
|
+
},
|
|
152
|
+
claim: async ({ workerId, leaseMs, limit }) => {
|
|
153
|
+
const conn = await db();
|
|
154
|
+
const at = await clock.now();
|
|
155
|
+
const taken = [];
|
|
156
|
+
/**
|
|
157
|
+
* Read, then take each job with ONE single-row update that only lands if the row is still the version that was
|
|
158
|
+
* read — due, unclaimed or with its lease still lapsed, and not written since. Two workers reading the same
|
|
159
|
+
* candidates each get the rows they win and skip the rest.
|
|
160
|
+
*
|
|
161
|
+
* Not `SELECT … FOR UPDATE SKIP LOCKED` in a transaction: the locks that takes on the status index cross with
|
|
162
|
+
* the ones the updates need, and two replicas claiming at once deadlock. A single-row update by primary key
|
|
163
|
+
* holds one lock and cannot.
|
|
164
|
+
*/
|
|
165
|
+
for (let round = 0; round < CLAIM_ROUNDS && taken.length < limit; round += 1) {
|
|
166
|
+
const rows = await selectRows(conn, `SELECT * FROM ${t.jobs}
|
|
167
|
+
WHERE (status = 'pending' AND run_at <= ?) OR (status = 'running' AND lease_until < ?)
|
|
168
|
+
ORDER BY run_at LIMIT ?`, [
|
|
169
|
+
at,
|
|
170
|
+
at,
|
|
171
|
+
limit - taken.length
|
|
172
|
+
]);
|
|
173
|
+
if (rows.length === 0) break;
|
|
174
|
+
for (const row of rows) {
|
|
175
|
+
const job = toJob(row);
|
|
176
|
+
const history = job.status === "running" ? [...job.history, {
|
|
177
|
+
attempt: job.history.length + 1,
|
|
178
|
+
status: "lost",
|
|
179
|
+
startedAt: job.updatedAt,
|
|
180
|
+
endedAt: at,
|
|
181
|
+
workerId: job.workerId ?? "unknown",
|
|
182
|
+
error: "the worker holding this job stopped reporting"
|
|
183
|
+
}] : job.history;
|
|
184
|
+
if ((await execute(conn, `UPDATE ${t.jobs} SET status = 'running', worker_id = ?, lease_until = ?, updated_at = ?,
|
|
185
|
+
attempts = attempts + 1, history = ?
|
|
186
|
+
WHERE id = ? AND status = ? AND updated_at = ? AND attempts = ?
|
|
187
|
+
AND ((status = 'pending' AND run_at <= ?) OR (status = 'running' AND lease_until < ?))`, [
|
|
188
|
+
workerId,
|
|
189
|
+
at + leaseMs,
|
|
190
|
+
at,
|
|
191
|
+
JSON.stringify(history),
|
|
192
|
+
job.id,
|
|
193
|
+
job.status,
|
|
194
|
+
job.updatedAt,
|
|
195
|
+
job.attempts,
|
|
196
|
+
at,
|
|
197
|
+
at
|
|
198
|
+
])).affectedRows !== 1) continue;
|
|
199
|
+
taken.push({
|
|
200
|
+
...job,
|
|
201
|
+
status: "running",
|
|
202
|
+
workerId,
|
|
203
|
+
leaseUntil: at + leaseMs,
|
|
204
|
+
updatedAt: at,
|
|
205
|
+
attempts: job.attempts + 1,
|
|
206
|
+
history
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
return taken;
|
|
211
|
+
},
|
|
212
|
+
heartbeat: async ({ jobIds, workerId, leaseMs }) => {
|
|
213
|
+
if (jobIds.length === 0) return [];
|
|
214
|
+
const conn = await db();
|
|
215
|
+
const at = await clock.now();
|
|
216
|
+
await execute(conn, `UPDATE ${t.jobs} SET lease_until = ? WHERE id IN (?) AND worker_id = ? AND status = 'running'`, [
|
|
217
|
+
at + leaseMs,
|
|
218
|
+
jobIds,
|
|
219
|
+
workerId
|
|
220
|
+
]);
|
|
221
|
+
return (await selectRows(conn, `SELECT id FROM ${t.jobs} WHERE id IN (?) AND worker_id = ? AND status = 'running' AND cancel_requested = 1`, [jobIds, workerId])).map((row) => row.id);
|
|
222
|
+
},
|
|
223
|
+
settle: async ({ jobId, workerId, status, delayMs, runId, error, attempt }) => {
|
|
224
|
+
await db();
|
|
225
|
+
const at = await clock.now();
|
|
226
|
+
await transaction(pool, async (connection) => {
|
|
227
|
+
const row = await selectOne(connection, `SELECT * FROM ${t.jobs} WHERE id = ? AND worker_id = ? FOR UPDATE`, [jobId, workerId]);
|
|
228
|
+
if (!row) return;
|
|
229
|
+
const job = toJob(row);
|
|
230
|
+
const history = attempt ? [...job.history, {
|
|
231
|
+
...attempt,
|
|
232
|
+
attempt: job.history.length + 1,
|
|
233
|
+
startedAt: job.updatedAt,
|
|
234
|
+
endedAt: at
|
|
235
|
+
}] : job.history;
|
|
236
|
+
const pending = status === "pending";
|
|
237
|
+
const attempts = pending && !attempt ? Math.max(0, job.attempts - 1) : job.attempts;
|
|
238
|
+
await execute(connection, `UPDATE ${t.jobs} SET status = ?, updated_at = ?, lease_until = NULL, error = ?, run_id = COALESCE(?, run_id),
|
|
239
|
+
run_at = ?, worker_id = ?, attempts = ?, history = ?, expires_at = ? WHERE id = ?`, [
|
|
240
|
+
status,
|
|
241
|
+
at,
|
|
242
|
+
error ?? null,
|
|
243
|
+
runId ?? null,
|
|
244
|
+
pending ? at + (delayMs ?? 0) : job.runAt,
|
|
245
|
+
pending ? null : workerId,
|
|
246
|
+
attempts,
|
|
247
|
+
JSON.stringify(history),
|
|
248
|
+
TERMINAL.includes(status) ? expiryOf(at) : null,
|
|
249
|
+
jobId
|
|
250
|
+
]);
|
|
251
|
+
});
|
|
252
|
+
await purge(at);
|
|
253
|
+
},
|
|
254
|
+
dueSchedules: async (limit) => {
|
|
255
|
+
const conn = await db();
|
|
256
|
+
const at = await clock.now();
|
|
257
|
+
return (await selectRows(conn, `SELECT * FROM ${t.schedules} WHERE enabled = 1 AND next_run_at <= ? ORDER BY next_run_at LIMIT ?`, [at, limit])).map(toSchedule);
|
|
258
|
+
},
|
|
259
|
+
advanceSchedule: async ({ spaceId, actionId, from, to, missed }) => {
|
|
260
|
+
const conn = await db();
|
|
261
|
+
const at = await clock.now();
|
|
262
|
+
return (await execute(conn, `UPDATE ${t.schedules} SET next_run_at = ?, last_fire_at = ?, updated_at = ?, missed = missed + ?
|
|
263
|
+
WHERE space_id = ? AND action_id = ? AND next_run_at = ?`, [
|
|
264
|
+
to,
|
|
265
|
+
from,
|
|
266
|
+
at,
|
|
267
|
+
missed,
|
|
268
|
+
spaceId,
|
|
269
|
+
actionId,
|
|
270
|
+
from
|
|
271
|
+
])).affectedRows === 1;
|
|
272
|
+
},
|
|
273
|
+
putSchedules: async ({ spaceId, schedules: incoming }) => {
|
|
274
|
+
await db();
|
|
275
|
+
const at = await clock.now();
|
|
276
|
+
await transaction(pool, async (connection) => {
|
|
277
|
+
for (const schedule of incoming)
|
|
278
|
+
/**
|
|
279
|
+
* `next_run_at` FIRST: MySQL applies these assignments left to right, each seeing the ones before it, so
|
|
280
|
+
* the comparison has to read the old expression before `cron` is overwritten. A fire already promised for
|
|
281
|
+
* an unchanged expression is kept — recomputing it on every save would push a busy space's schedule forward
|
|
282
|
+
* forever. `VALUES()` rather than a row alias, which MariaDB does not have.
|
|
283
|
+
*/
|
|
284
|
+
await execute(connection, `INSERT INTO ${t.schedules} (space_id, action_id, cron, timezone, environment, enabled, next_run_at,
|
|
285
|
+
missed, max_attempts, updated_at)
|
|
286
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?)
|
|
287
|
+
ON DUPLICATE KEY UPDATE
|
|
288
|
+
next_run_at = IF(cron = VALUES(cron) AND timezone <=> VALUES(timezone)
|
|
289
|
+
AND environment = VALUES(environment), next_run_at, VALUES(next_run_at)),
|
|
290
|
+
cron = VALUES(cron), timezone = VALUES(timezone), environment = VALUES(environment),
|
|
291
|
+
enabled = VALUES(enabled), max_attempts = VALUES(max_attempts), updated_at = VALUES(updated_at)`, [
|
|
292
|
+
spaceId,
|
|
293
|
+
schedule.actionId,
|
|
294
|
+
schedule.cron,
|
|
295
|
+
schedule.timezone ?? null,
|
|
296
|
+
schedule.environment,
|
|
297
|
+
schedule.enabled ? 1 : 0,
|
|
298
|
+
schedule.nextRunAt,
|
|
299
|
+
schedule.maxAttempts,
|
|
300
|
+
at
|
|
301
|
+
]);
|
|
302
|
+
const kept = incoming.map((schedule) => schedule.actionId);
|
|
303
|
+
await execute(connection, kept.length > 0 ? `DELETE FROM ${t.schedules} WHERE space_id = ? AND action_id NOT IN (?)` : `DELETE FROM ${t.schedules} WHERE space_id = ?`, kept.length > 0 ? [spaceId, kept] : [spaceId]);
|
|
304
|
+
});
|
|
305
|
+
},
|
|
306
|
+
listJobs: async ({ spaceIds, actionId, status, limit, offset }) => {
|
|
307
|
+
if (!inSpaces(spaceIds)) return {
|
|
308
|
+
jobs: [],
|
|
309
|
+
total: 0
|
|
310
|
+
};
|
|
311
|
+
const conn = await db();
|
|
312
|
+
const where = [
|
|
313
|
+
"space_id IN (?)",
|
|
314
|
+
...actionId ? ["action_id = ?"] : [],
|
|
315
|
+
...status ? ["status = ?"] : []
|
|
316
|
+
].join(" AND ");
|
|
317
|
+
const params = [
|
|
318
|
+
spaceIds,
|
|
319
|
+
...actionId ? [actionId] : [],
|
|
320
|
+
...status ? [status] : []
|
|
321
|
+
];
|
|
322
|
+
const [rows, count] = await Promise.all([selectRows(conn, `SELECT * FROM ${t.jobs} WHERE ${where} ORDER BY updated_at DESC, id DESC LIMIT ? OFFSET ?`, [
|
|
323
|
+
...params,
|
|
324
|
+
limit,
|
|
325
|
+
offset
|
|
326
|
+
]), selectOne(conn, `SELECT COUNT(*) AS total FROM ${t.jobs} WHERE ${where}`, params)]);
|
|
327
|
+
return {
|
|
328
|
+
jobs: rows.map(toJob),
|
|
329
|
+
total: Number(count?.total ?? 0)
|
|
330
|
+
};
|
|
331
|
+
},
|
|
332
|
+
getJob: async (spaceIds, jobId) => {
|
|
333
|
+
if (!inSpaces(spaceIds)) return;
|
|
334
|
+
const row = await selectOne(await db(), `SELECT * FROM ${t.jobs} WHERE id = ? AND space_id IN (?)`, [jobId, spaceIds]);
|
|
335
|
+
return row ? toJob(row) : void 0;
|
|
336
|
+
},
|
|
337
|
+
listSchedules: async (spaceIds) => {
|
|
338
|
+
if (!inSpaces(spaceIds)) return [];
|
|
339
|
+
return (await selectRows(await db(), `SELECT * FROM ${t.schedules} WHERE space_id IN (?) ORDER BY next_run_at`, [spaceIds])).map(toSchedule);
|
|
340
|
+
},
|
|
341
|
+
requeue: async (spaceIds, jobId) => {
|
|
342
|
+
if (!inSpaces(spaceIds)) return false;
|
|
343
|
+
const conn = await db();
|
|
344
|
+
const at = await clock.now();
|
|
345
|
+
return (await execute(conn, `UPDATE ${t.jobs} SET status = 'pending', run_at = ?, attempts = 0, updated_at = ?, cancel_requested = 0,
|
|
346
|
+
worker_id = NULL, lease_until = NULL, error = NULL, expires_at = NULL
|
|
347
|
+
WHERE id = ? AND space_id IN (?) AND status IN (?)`, [
|
|
348
|
+
at,
|
|
349
|
+
at,
|
|
350
|
+
jobId,
|
|
351
|
+
spaceIds,
|
|
352
|
+
TERMINAL
|
|
353
|
+
])).affectedRows === 1;
|
|
354
|
+
},
|
|
355
|
+
cancel: async (spaceIds, jobId) => {
|
|
356
|
+
if (!inSpaces(spaceIds)) return false;
|
|
357
|
+
const conn = await db();
|
|
358
|
+
const at = await clock.now();
|
|
359
|
+
if ((await execute(conn, `UPDATE ${t.jobs} SET status = 'cancelled', updated_at = ?, expires_at = ?, worker_id = NULL
|
|
360
|
+
WHERE id = ? AND space_id IN (?) AND status = 'pending'`, [
|
|
361
|
+
at,
|
|
362
|
+
expiryOf(at),
|
|
363
|
+
jobId,
|
|
364
|
+
spaceIds
|
|
365
|
+
])).affectedRows === 1) return true;
|
|
366
|
+
return (await execute(conn, `UPDATE ${t.jobs} SET cancel_requested = 1, updated_at = ? WHERE id = ? AND space_id IN (?) AND status = 'running'`, [
|
|
367
|
+
at,
|
|
368
|
+
jobId,
|
|
369
|
+
spaceIds
|
|
370
|
+
])).affectedRows === 1;
|
|
371
|
+
}
|
|
372
|
+
};
|
|
373
|
+
};
|
|
374
|
+
//#endregion
|
|
375
|
+
export { createMysqlJobQueue };
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { ActionKvAdapter } from '../../actions/types';
|
|
2
|
+
import { Pool } from 'mysql2/promise';
|
|
3
|
+
export type MysqlKvOptions = {
|
|
4
|
+
/** A pool the deployment opened. Never ended here. */
|
|
5
|
+
pool: Pool;
|
|
6
|
+
/** Same prefix as the job queue's, when both are used: the three tables are created together. */
|
|
7
|
+
tablePrefix?: string;
|
|
8
|
+
/** Create the tables on first use. Default true. */
|
|
9
|
+
createTables?: boolean;
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* An {@link ActionKvAdapter} over a table in a MySQL database the deployment owns.
|
|
13
|
+
*
|
|
14
|
+
* What the run guards' single-flight keys and the `kv` tasks' counters need from a store the replicas share: a TTL the
|
|
15
|
+
* store enforces and an increment nobody can race. The increment is one `INSERT … ON DUPLICATE KEY UPDATE`, which
|
|
16
|
+
* MySQL applies to the row under its lock — so two replicas counting at once both count.
|
|
17
|
+
*/
|
|
18
|
+
export declare const createMysqlKv: ({ pool, tablePrefix, createTables }: MysqlKvOptions) => ActionKvAdapter;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { execute, selectOne } from "../query.js";
|
|
2
|
+
import { NOW_MS, ensureJobTables, jobTables } from "./schema.js";
|
|
3
|
+
//#region src/modules/mysql/jobs/kv.ts
|
|
4
|
+
/** Alive by the SERVER's clock: a key past its lifetime answers nothing, whether or not it has been removed yet. */
|
|
5
|
+
var LIVE = `(expires_at IS NULL OR expires_at > ${NOW_MS})`;
|
|
6
|
+
/**
|
|
7
|
+
* An {@link ActionKvAdapter} over a table in a MySQL database the deployment owns.
|
|
8
|
+
*
|
|
9
|
+
* What the run guards' single-flight keys and the `kv` tasks' counters need from a store the replicas share: a TTL the
|
|
10
|
+
* store enforces and an increment nobody can race. The increment is one `INSERT … ON DUPLICATE KEY UPDATE`, which
|
|
11
|
+
* MySQL applies to the row under its lock — so two replicas counting at once both count.
|
|
12
|
+
*/
|
|
13
|
+
var createMysqlKv = ({ pool, tablePrefix = "", createTables = true }) => {
|
|
14
|
+
const { kv } = jobTables(tablePrefix);
|
|
15
|
+
const ready = ensureJobTables(pool, tablePrefix, createTables);
|
|
16
|
+
const db = async () => {
|
|
17
|
+
await ready();
|
|
18
|
+
return pool;
|
|
19
|
+
};
|
|
20
|
+
/** Expired rows are removed as they are overwritten or read past; a table of used-once keys is swept here too. */
|
|
21
|
+
let sweptAt = 0;
|
|
22
|
+
const sweep = async () => {
|
|
23
|
+
if (Date.now() - sweptAt < 6e4) return;
|
|
24
|
+
sweptAt = Date.now();
|
|
25
|
+
await execute(pool, `DELETE FROM ${kv} WHERE expires_at IS NOT NULL AND expires_at <= ${NOW_MS} LIMIT 1000`);
|
|
26
|
+
};
|
|
27
|
+
return {
|
|
28
|
+
get: async (key) => {
|
|
29
|
+
return (await selectOne(await db(), `SELECT v FROM ${kv} WHERE k = ? AND ${LIVE}`, [key]))?.v;
|
|
30
|
+
},
|
|
31
|
+
set: async (key, value, ttlSeconds) => {
|
|
32
|
+
const expiry = ttlSeconds === void 0 ? "NULL" : `${NOW_MS} + ?`;
|
|
33
|
+
await execute(await db(), `INSERT INTO ${kv} (k, v, expires_at) VALUES (?, ?, ${expiry})
|
|
34
|
+
ON DUPLICATE KEY UPDATE v = VALUES(v), expires_at = VALUES(expires_at)`, ttlSeconds === void 0 ? [key, value] : [
|
|
35
|
+
key,
|
|
36
|
+
value,
|
|
37
|
+
ttlSeconds * 1e3
|
|
38
|
+
]);
|
|
39
|
+
await sweep();
|
|
40
|
+
},
|
|
41
|
+
delete: async (key) => {
|
|
42
|
+
await execute(await db(), `DELETE FROM ${kv} WHERE k = ?`, [key]);
|
|
43
|
+
},
|
|
44
|
+
increment: async (key, amount) => {
|
|
45
|
+
const connection = await (await db()).getConnection();
|
|
46
|
+
try {
|
|
47
|
+
await connection.beginTransaction();
|
|
48
|
+
/**
|
|
49
|
+
* `v` FIRST: the assignments run left to right and each sees the ones before it, so the value has to be
|
|
50
|
+
* computed while `expires_at` still says whether the old one is alive. A lapsed counter starts again from
|
|
51
|
+
* zero, and without the lifetime it had. `+ 0` reads the text as a number, as Redis reads its strings.
|
|
52
|
+
*/
|
|
53
|
+
await execute(connection, `INSERT INTO ${kv} (k, v, expires_at) VALUES (?, CAST(? AS CHAR), NULL)
|
|
54
|
+
ON DUPLICATE KEY UPDATE
|
|
55
|
+
v = CAST(IF(${LIVE}, (v + 0) + ?, ?) AS CHAR),
|
|
56
|
+
expires_at = IF(${LIVE}, expires_at, NULL)`, [
|
|
57
|
+
key,
|
|
58
|
+
amount,
|
|
59
|
+
amount,
|
|
60
|
+
amount
|
|
61
|
+
]);
|
|
62
|
+
const row = await selectOne(connection, `SELECT v FROM ${kv} WHERE k = ?`, [key]);
|
|
63
|
+
await connection.commit();
|
|
64
|
+
return Number(row?.v ?? amount);
|
|
65
|
+
} catch (error) {
|
|
66
|
+
await connection.rollback();
|
|
67
|
+
throw error;
|
|
68
|
+
} finally {
|
|
69
|
+
connection.release();
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
expire: async (key, ttlSeconds) => {
|
|
73
|
+
await execute(await db(), `UPDATE ${kv} SET expires_at = ${NOW_MS} + ? WHERE k = ? AND ${LIVE}`, [ttlSeconds * 1e3, key]);
|
|
74
|
+
},
|
|
75
|
+
/**
|
|
76
|
+
* Each branch is ONE statement MySQL applies under the row's lock, so two writers racing for the same key cannot
|
|
77
|
+
* both win. `affectedRows` counts the rows MATCHED because mysql2 connects with `FOUND_ROWS`: without it, writing
|
|
78
|
+
* back the value a key already holds would read as a refusal. The comparison is on bytes — the table's collation
|
|
79
|
+
* ignores case and trailing spaces, and "unchanged" has to mean the same bytes.
|
|
80
|
+
*/
|
|
81
|
+
swap: async (key, expected, next, ttlSeconds) => {
|
|
82
|
+
const expiry = ttlSeconds === void 0 ? "NULL" : `${NOW_MS} + ?`;
|
|
83
|
+
const lifetime = ttlSeconds === void 0 ? [] : [ttlSeconds * 1e3];
|
|
84
|
+
const pool = await db();
|
|
85
|
+
if (expected !== void 0) return (await execute(pool, `UPDATE ${kv} SET v = ?, expires_at = ${expiry}
|
|
86
|
+
WHERE k = ? AND ${LIVE} AND CAST(v AS BINARY) = CAST(? AS BINARY)`, [
|
|
87
|
+
next,
|
|
88
|
+
...lifetime,
|
|
89
|
+
key,
|
|
90
|
+
expected
|
|
91
|
+
])).affectedRows === 1;
|
|
92
|
+
if ((await execute(pool, `UPDATE ${kv} SET v = ?, expires_at = ${expiry} WHERE k = ? AND NOT ${LIVE}`, [
|
|
93
|
+
next,
|
|
94
|
+
...lifetime,
|
|
95
|
+
key
|
|
96
|
+
])).affectedRows === 1) return true;
|
|
97
|
+
const inserted = await execute(pool, `INSERT IGNORE INTO ${kv} (k, v, expires_at) VALUES (?, ?, ${expiry})`, [
|
|
98
|
+
key,
|
|
99
|
+
next,
|
|
100
|
+
...lifetime
|
|
101
|
+
]);
|
|
102
|
+
await sweep();
|
|
103
|
+
return inserted.affectedRows === 1;
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
};
|
|
107
|
+
//#endregion
|
|
108
|
+
export { createMysqlKv };
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { Queryable } from '../query';
|
|
2
|
+
/** The tables the job queue and the key/value store keep, backticked and prefixed. */
|
|
3
|
+
export type JobTables = {
|
|
4
|
+
jobs: string;
|
|
5
|
+
schedules: string;
|
|
6
|
+
kv: string;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Table names under a prefix. A prefix is part of an identifier, which no driver can bind as a parameter — so it is
|
|
10
|
+
* held to the characters an unquoted identifier may use, and refused otherwise, rather than escaped and hoped for.
|
|
11
|
+
*/
|
|
12
|
+
export declare const jobTables: (prefix?: string) => JobTables;
|
|
13
|
+
/**
|
|
14
|
+
* The tables, as SQL — for a deployment that runs its own migrations rather than letting the helpers create them.
|
|
15
|
+
*
|
|
16
|
+
* Instants are epoch milliseconds in `BIGINT`, read from the SERVER's clock; `history` and `input` are JSON kept as
|
|
17
|
+
* text, because they are only ever read whole. Separate from the account store's schema and its version: a deployment
|
|
18
|
+
* can use either without the other.
|
|
19
|
+
*
|
|
20
|
+
* A `kv` key is BYTES, as it is in Redis and in memory: under a collation `Board` and `board` — and `a` and `a ` — were
|
|
21
|
+
* one key here and two everywhere else. A value is a `MEDIUMTEXT`: a `TEXT` stops at 64 KB, and past it MySQL
|
|
22
|
+
* refuses the write, or cuts it short where it is not strict.
|
|
23
|
+
*/
|
|
24
|
+
export declare const mysqlJobSchemaStatements: (prefix?: string) => string[];
|
|
25
|
+
/**
|
|
26
|
+
* What brings a `kv` table made before the columns above up to them — for a deployment that runs its own migrations,
|
|
27
|
+
* once. `ensureJobTables` does it by itself, and only when the table needs it: the `ALTER` copies the table.
|
|
28
|
+
*/
|
|
29
|
+
export declare const mysqlJobSchemaUpgrades: (prefix?: string) => string[];
|
|
30
|
+
/** The server's now, in epoch ms, as an SQL expression — what every comparison reads instead of a caller's clock. */
|
|
31
|
+
export declare const NOW_MS = "CAST(ROUND(UNIX_TIMESTAMP(NOW(3)) * 1000) AS SIGNED)";
|
|
32
|
+
/** Creates the tables once per process, on first use; a failure is retried by the next call rather than remembered. */
|
|
33
|
+
export declare const ensureJobTables: (db: Queryable, prefix: string, create: boolean) => (() => Promise<void>);
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { execute, selectRows } from "../query.js";
|
|
2
|
+
//#region src/modules/mysql/jobs/schema.ts
|
|
3
|
+
var PREFIX = /^[A-Za-z0-9_]*$/;
|
|
4
|
+
/**
|
|
5
|
+
* Table names under a prefix. A prefix is part of an identifier, which no driver can bind as a parameter — so it is
|
|
6
|
+
* held to the characters an unquoted identifier may use, and refused otherwise, rather than escaped and hoped for.
|
|
7
|
+
*/
|
|
8
|
+
var jobTables = (prefix = "") => {
|
|
9
|
+
if (!PREFIX.test(prefix)) throw new Error(`@plitzi/sdk-server/mysql: tablePrefix "${prefix}" may only use letters, digits and underscores`);
|
|
10
|
+
return {
|
|
11
|
+
jobs: `\`${prefix}action_jobs\``,
|
|
12
|
+
schedules: `\`${prefix}action_schedules\``,
|
|
13
|
+
kv: `\`${prefix}action_kv\``
|
|
14
|
+
};
|
|
15
|
+
};
|
|
16
|
+
var CHARSET = "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci";
|
|
17
|
+
/**
|
|
18
|
+
* The tables, as SQL — for a deployment that runs its own migrations rather than letting the helpers create them.
|
|
19
|
+
*
|
|
20
|
+
* Instants are epoch milliseconds in `BIGINT`, read from the SERVER's clock; `history` and `input` are JSON kept as
|
|
21
|
+
* text, because they are only ever read whole. Separate from the account store's schema and its version: a deployment
|
|
22
|
+
* can use either without the other.
|
|
23
|
+
*
|
|
24
|
+
* A `kv` key is BYTES, as it is in Redis and in memory: under a collation `Board` and `board` — and `a` and `a ` — were
|
|
25
|
+
* one key here and two everywhere else. A value is a `MEDIUMTEXT`: a `TEXT` stops at 64 KB, and past it MySQL
|
|
26
|
+
* refuses the write, or cuts it short where it is not strict.
|
|
27
|
+
*/
|
|
28
|
+
var mysqlJobSchemaStatements = (prefix = "") => {
|
|
29
|
+
const t = jobTables(prefix);
|
|
30
|
+
return [
|
|
31
|
+
`CREATE TABLE IF NOT EXISTS ${t.jobs} (
|
|
32
|
+
id VARCHAR(191) NOT NULL,
|
|
33
|
+
space_id BIGINT NOT NULL,
|
|
34
|
+
action_id VARCHAR(191) NOT NULL,
|
|
35
|
+
environment VARCHAR(64) NOT NULL,
|
|
36
|
+
trigger_type VARCHAR(64) NOT NULL,
|
|
37
|
+
input MEDIUMTEXT NOT NULL,
|
|
38
|
+
due_at BIGINT NOT NULL,
|
|
39
|
+
max_attempts INT NOT NULL,
|
|
40
|
+
missed INT NULL,
|
|
41
|
+
status VARCHAR(16) NOT NULL,
|
|
42
|
+
run_at BIGINT NOT NULL,
|
|
43
|
+
attempts INT NOT NULL DEFAULT 0,
|
|
44
|
+
lease_until BIGINT NULL,
|
|
45
|
+
worker_id VARCHAR(191) NULL,
|
|
46
|
+
run_id VARCHAR(191) NULL,
|
|
47
|
+
cancel_requested TINYINT(1) NOT NULL DEFAULT 0,
|
|
48
|
+
created_at BIGINT NOT NULL,
|
|
49
|
+
updated_at BIGINT NOT NULL,
|
|
50
|
+
error TEXT NULL,
|
|
51
|
+
history MEDIUMTEXT NOT NULL,
|
|
52
|
+
expires_at BIGINT NULL,
|
|
53
|
+
PRIMARY KEY (id),
|
|
54
|
+
KEY action_jobs_pending (status, run_at),
|
|
55
|
+
KEY action_jobs_lease (status, lease_until),
|
|
56
|
+
KEY action_jobs_space (space_id, updated_at),
|
|
57
|
+
KEY action_jobs_expires (expires_at)
|
|
58
|
+
) ${CHARSET}`,
|
|
59
|
+
`CREATE TABLE IF NOT EXISTS ${t.schedules} (
|
|
60
|
+
space_id BIGINT NOT NULL,
|
|
61
|
+
action_id VARCHAR(191) NOT NULL,
|
|
62
|
+
cron VARCHAR(191) NOT NULL,
|
|
63
|
+
timezone VARCHAR(64) NULL,
|
|
64
|
+
environment VARCHAR(64) NOT NULL,
|
|
65
|
+
enabled TINYINT(1) NOT NULL,
|
|
66
|
+
next_run_at BIGINT NOT NULL,
|
|
67
|
+
last_fire_at BIGINT NULL,
|
|
68
|
+
missed INT NOT NULL DEFAULT 0,
|
|
69
|
+
max_attempts INT NOT NULL,
|
|
70
|
+
updated_at BIGINT NOT NULL,
|
|
71
|
+
PRIMARY KEY (space_id, action_id),
|
|
72
|
+
KEY action_schedules_due (enabled, next_run_at)
|
|
73
|
+
) ${CHARSET}`,
|
|
74
|
+
`CREATE TABLE IF NOT EXISTS ${t.kv} (
|
|
75
|
+
k VARBINARY(764) NOT NULL,
|
|
76
|
+
v MEDIUMTEXT NOT NULL,
|
|
77
|
+
expires_at BIGINT NULL,
|
|
78
|
+
PRIMARY KEY (k),
|
|
79
|
+
KEY action_kv_expires (expires_at)
|
|
80
|
+
) ${CHARSET}`
|
|
81
|
+
];
|
|
82
|
+
};
|
|
83
|
+
/**
|
|
84
|
+
* What brings a `kv` table made before the columns above up to them — for a deployment that runs its own migrations,
|
|
85
|
+
* once. `ensureJobTables` does it by itself, and only when the table needs it: the `ALTER` copies the table.
|
|
86
|
+
*/
|
|
87
|
+
var mysqlJobSchemaUpgrades = (prefix = "") => [`ALTER TABLE ${jobTables(prefix).kv} MODIFY k VARBINARY(764) NOT NULL, MODIFY v MEDIUMTEXT NOT NULL`];
|
|
88
|
+
/** Whether the `kv` table still has the columns it was first made with. */
|
|
89
|
+
var kvNeedsUpgrade = async (db, prefix) => {
|
|
90
|
+
const columns = await selectRows(db, `SELECT COLUMN_NAME AS name, DATA_TYPE AS type FROM INFORMATION_SCHEMA.COLUMNS
|
|
91
|
+
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME IN ('k', 'v')`, [`${prefix}action_kv`]);
|
|
92
|
+
const typeOf = (name) => columns.find((column) => column.name === name)?.type.toLowerCase();
|
|
93
|
+
return typeOf("k") !== "varbinary" || typeOf("v") !== "mediumtext";
|
|
94
|
+
};
|
|
95
|
+
/** The server's now, in epoch ms, as an SQL expression — what every comparison reads instead of a caller's clock. */
|
|
96
|
+
var NOW_MS = "CAST(ROUND(UNIX_TIMESTAMP(NOW(3)) * 1000) AS SIGNED)";
|
|
97
|
+
/** Creates the tables once per process, on first use; a failure is retried by the next call rather than remembered. */
|
|
98
|
+
var ensureJobTables = (db, prefix, create) => {
|
|
99
|
+
let ready;
|
|
100
|
+
return async () => {
|
|
101
|
+
if (!create) return;
|
|
102
|
+
ready ??= (async () => {
|
|
103
|
+
for (const statement of mysqlJobSchemaStatements(prefix)) await execute(db, statement);
|
|
104
|
+
if (await kvNeedsUpgrade(db, prefix)) for (const statement of mysqlJobSchemaUpgrades(prefix)) await execute(db, statement);
|
|
105
|
+
})().catch((error) => {
|
|
106
|
+
ready = void 0;
|
|
107
|
+
throw error;
|
|
108
|
+
});
|
|
109
|
+
await ready;
|
|
110
|
+
};
|
|
111
|
+
};
|
|
112
|
+
//#endregion
|
|
113
|
+
export { NOW_MS, ensureJobTables, jobTables, mysqlJobSchemaStatements, mysqlJobSchemaUpgrades };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|