@platformatic/watt-extra 1.21.0 → 1.22.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/app.js CHANGED
@@ -10,6 +10,7 @@ import update from './plugins/update.js'
10
10
  import alert from './plugins/alerts.js'
11
11
  import healthSignals from './plugins/health-signals.js'
12
12
  import flamegraphs from './plugins/flamegraphs.js'
13
+ import durableObjects from './plugins/durable-objects.js'
13
14
 
14
15
  async function buildApp (logger) {
15
16
  const app = {
@@ -27,6 +28,7 @@ async function buildApp (logger) {
27
28
  .use(metadata)
28
29
  .use(compliancy)
29
30
  .use(scheduler)
31
+ .use(durableObjects)
30
32
  .use(update)
31
33
  .use(flamegraphs)
32
34
 
@@ -0,0 +1,131 @@
1
+ import { readFile } from 'node:fs/promises'
2
+ import { join } from 'node:path'
3
+
4
+ // The Durable Objects of Lavoratore applications (@platformatic/lavoratore), orchestrated by ICC: see
5
+ // docs/durable-objects.md of Lavoratore. Everything here is optional: an ICC which knows nothing of Durable Objects
6
+ // sends no `durableObjects` with the instance config and no command, and nothing changes.
7
+
8
+ export const LAVORATORE_TYPES = ['@platformatic/lavoratore', 'lavoratore']
9
+
10
+ // The ITC commands handled by the Lavoratore applications, with the same names and payloads as their runtime messages
11
+ export const COMMANDS = {
12
+ 'durable-objects-promote': 'plt:lavoratore:promote',
13
+ 'durable-objects-drain': 'plt:lavoratore:drain'
14
+ }
15
+
16
+ // The names a runtime gives to the configuration file of an application which declares none
17
+ const CONFIG_FILES = ['watt.json', 'platformatic.json', 'platformatic.application.json']
18
+
19
+ // The configuration file of an application, parsed, or null when there is none or it cannot be read as JSON
20
+ async function readApplicationConfig (application) {
21
+ const candidates = []
22
+
23
+ if (application?.config) {
24
+ candidates.push(application.config)
25
+ } else if (application?.path) {
26
+ for (const file of CONFIG_FILES) {
27
+ candidates.push(join(application.path, file))
28
+ }
29
+ }
30
+
31
+ for (const candidate of candidates) {
32
+ try {
33
+ return JSON.parse(await readFile(candidate, 'utf8'))
34
+ } catch {}
35
+ }
36
+
37
+ return null
38
+ }
39
+
40
+ // Whether an application of the runtime configuration is a Lavoratore one. The runtime configuration reports the type
41
+ // only for some applications, and "unknown" for one whose capability it resolves when it starts, so the configuration
42
+ // file of the application decides too: its module, or its schema.
43
+ export async function isLavoratoreApplication (application) {
44
+ if (LAVORATORE_TYPES.includes(application?.type)) {
45
+ return true
46
+ }
47
+
48
+ const config = await readApplicationConfig(application)
49
+
50
+ if (!config) {
51
+ return false
52
+ }
53
+
54
+ return LAVORATORE_TYPES.includes(config.module) || /\/@platformatic\/lavoratore\/[^/]+\.json$/.test(config.$schema ?? '')
55
+ }
56
+
57
+ // The configuration patch of a Lavoratore application under ICC, or null when ICC said nothing about Durable Objects:
58
+ // a pod of a staged version receives no objects until ICC promotes it, with the generation of the approval
59
+ export function durableObjectsPatch (instanceConfig) {
60
+ const durableObjects = instanceConfig?.durableObjects
61
+
62
+ if (!durableObjects || typeof durableObjects !== 'object') {
63
+ return null
64
+ }
65
+
66
+ return [
67
+ {
68
+ op: 'merge',
69
+ path: '/lavoratore/durableObjects/placement',
70
+ value: { promoted: durableObjects.staged !== true }
71
+ }
72
+ ]
73
+ }
74
+
75
+ // Whether the patch applies to a Lavoratore application: only one which places Durable Objects across pods, i.e. whose
76
+ // configuration has a placement. A configuration which cannot be read gets the patch: the application ignores it
77
+ // without Durable Objects.
78
+ export async function placesDurableObjects (application) {
79
+ const config = await readApplicationConfig(application)
80
+ return config ? config.lavoratore?.durableObjects?.placement != null : true
81
+ }
82
+
83
+ // What ICC said about Durable Objects when it refused the registration of the pod, or null. The error of the client
84
+ // carries the body of the response: an ICC which orchestrates Durable Objects adds `durableObjects` to its refusals,
85
+ // i.e. the 4xx answers, since a pod it does not accept must not receive objects. Any other failure, an ICC which cannot
86
+ // be reached, which fails, or which knows nothing of Durable Objects, says nothing, and nothing changes.
87
+ export function refusedDurableObjects (err) {
88
+ let body
89
+
90
+ try {
91
+ body = JSON.parse(err?.message)
92
+ } catch {
93
+ return null
94
+ }
95
+
96
+ if (!body || typeof body !== 'object' || !(body.statusCode >= 400 && body.statusCode < 500)) {
97
+ return null
98
+ }
99
+
100
+ const durableObjects = body.durableObjects
101
+ return durableObjects && typeof durableObjects === 'object' ? { durableObjects } : null
102
+ }
103
+
104
+ // The result of a promotion of the Lavoratore applications of the pod: promoted when all those which place Durable
105
+ // Objects across pods are. An application which does not answers { status: 'unsupported' }, or has no handler with a
106
+ // version of Lavoratore which predates that answer: it has nothing to promote, and a pod with no other has none either.
107
+ export function aggregatePromotion (results) {
108
+ return {
109
+ promoted: Object.values(results).every(result => result.promoted === true || result.status === 'unsupported'),
110
+ applications: results
111
+ }
112
+ }
113
+
114
+ // The status of a drain of the Lavoratore applications of the pod which handle it: draining while any is, or waits for
115
+ // the newer version; drained when all are, or cannot drain at all, or none handles it; blocked when none is partial and
116
+ // some are blocked; partial otherwise.
117
+ export function aggregateDrain (results) {
118
+ const statuses = Object.values(results).map(result => result.status)
119
+ const done = value => value === 'drained' || value === 'unsupported'
120
+ let status = 'partial'
121
+
122
+ if (statuses.some(value => value === 'draining' || value === 'waiting')) {
123
+ status = 'draining'
124
+ } else if (statuses.every(done)) {
125
+ status = 'drained'
126
+ } else if (statuses.every(value => done(value) || value === 'blocked')) {
127
+ status = 'blocked'
128
+ }
129
+
130
+ return { status, applications: results }
131
+ }
package/lib/watt.js CHANGED
@@ -5,6 +5,7 @@ import { join, resolve } from 'node:path'
5
5
  import { createRequire } from 'node:module'
6
6
  import { pathToFileURL } from 'node:url'
7
7
  import semver from 'semver'
8
+ import { durableObjectsPatch, isLavoratoreApplication, placesDurableObjects } from './durable-objects.js'
8
9
 
9
10
  const require = createRequire(import.meta.url)
10
11
 
@@ -37,6 +38,7 @@ class Watt {
37
38
  #applicationName
38
39
  #instanceId
39
40
  #instanceConfig
41
+ #registrationRefusal
40
42
  #originalConfig
41
43
  #config
42
44
  #sharedContext
@@ -51,6 +53,8 @@ class Watt {
51
53
  this.runtime = null
52
54
  this.#sharedContext = {}
53
55
  this.#instanceConfig = app.instanceConfig
56
+ // What ICC said about Durable Objects when it refused the registration, see refusedDurableObjects()
57
+ this.#registrationRefusal = app.registrationRefusal ?? null
54
58
  }
55
59
 
56
60
  async spawn () {
@@ -104,6 +108,13 @@ class Watt {
104
108
  const previousConfig = this.#instanceConfig
105
109
  this.#instanceConfig = instanceConfig
106
110
 
111
+ // The configuration of the applications was patched at boot, without ICC: a pod of a staged version started as
112
+ // any other, and receives Durable Objects until it restarts. Not when ICC refused the pod at boot: it started
113
+ // staged then, and ICC promotes it once its version is active.
114
+ if (!previousConfig && instanceConfig?.durableObjects?.staged === true && this.#registrationRefusal?.durableObjects?.staged !== true) {
115
+ this.#logger.warn('ICC was unreachable when this pod started: its Lavoratore applications were not started as staged, and receive Durable Objects before the approval of their version. Restart the pod to stage it.')
116
+ }
117
+
107
118
  // If we didn't have a config before and now we do, apply runtime updates
108
119
  if (!previousConfig && instanceConfig && this.runtime) {
109
120
  // Update undici interceptors
@@ -602,10 +613,23 @@ class Watt {
602
613
  await this.#configurePlatformaticServices(runtime, app)
603
614
  } else if (app.type === '@platformatic/regina') {
604
615
  await this.#configureReginaService(runtime, app)
616
+ } else if (await isLavoratoreApplication(app)) {
617
+ await this.#configureLavoratoreService(runtime, app)
605
618
  }
606
619
  }
607
620
  }
608
621
 
622
+ // Under an ICC which orchestrates Durable Objects, its approvals order the deployments of the application, and a pod
623
+ // of a staged version starts without receiving objects, until ICC promotes it (see lib/durable-objects.js). An ICC
624
+ // which says nothing about Durable Objects leaves the configuration of the application alone.
625
+ async #configureLavoratoreService (runtime, service) {
626
+ const patches = durableObjectsPatch(this.#instanceConfig ?? this.#registrationRefusal)
627
+
628
+ if (patches && await placesDurableObjects(service)) {
629
+ await this.#patchService(runtime, service.id, patches)
630
+ }
631
+ }
632
+
609
633
  async #configureNextService (runtime, service) {
610
634
  let nextPackage
611
635
  let nextSchema
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@platformatic/watt-extra",
3
- "version": "1.21.0",
3
+ "version": "1.22.0-alpha.1",
4
4
  "description": "The Platformatic runtime manager",
5
5
  "type": "module",
6
6
  "scripts": {
@@ -0,0 +1,96 @@
1
+ import { aggregateDrain, aggregatePromotion, COMMANDS, isLavoratoreApplication } from '../lib/durable-objects.js'
2
+
3
+ // Forwards the Durable Objects commands of ICC to the Lavoratore applications of the runtime, see lib/durable-objects.js
4
+ async function durableObjects (app, _opts) {
5
+ // The Lavoratore applications of the runtime: the configuration does not change while it runs
6
+ let applications
7
+
8
+ async function lavoratoreApplications () {
9
+ const runtime = app.watt?.runtime
10
+
11
+ if (!runtime) {
12
+ throw new Error('Runtime not started, cannot reach the Durable Objects applications')
13
+ }
14
+
15
+ if (!applications) {
16
+ const found = []
17
+
18
+ for (const application of runtime.getRuntimeConfig(true).applications ?? []) {
19
+ if (await isLavoratoreApplication(application)) {
20
+ found.push(application.id)
21
+ }
22
+ }
23
+
24
+ applications = found
25
+ }
26
+
27
+ // ICC sends these commands only to the pods it knows to run Lavoratore: finding none is a failure to recognize the
28
+ // applications, not a pod with nothing to promote or drain, and ICC must not take it for done
29
+ if (applications.length === 0) {
30
+ throw new Error('No Lavoratore application found in the runtime')
31
+ }
32
+
33
+ return applications
34
+ }
35
+
36
+ // The commands in flight, by command and parameters: ICC repeats a command which did not answer in time, and the
37
+ // repetition joins the one still running instead of starting another
38
+ const inflight = new Map()
39
+
40
+ // Runs a command on every Lavoratore application: a promotion with { generation } or a drain. Returns the aggregate
41
+ // of their answers, see lib/durable-objects.js. An application which has no handler for the command, i.e. without
42
+ // Durable Objects placed across pods, or with a version of Lavoratore which predates the commands, is skipped: it has
43
+ // nothing to promote or drain, and ICC must not wait for it. The applications are all asked even when one fails, and
44
+ // the failure is reported with what the others answered.
45
+ function sendDurableObjectsCommand (command, params = {}) {
46
+ const name = COMMANDS[command]
47
+
48
+ if (!name) {
49
+ return Promise.reject(new Error(`Unknown Durable Objects command "${command}"`))
50
+ }
51
+
52
+ const key = `${command}:${JSON.stringify(params)}`
53
+ let running = inflight.get(key)
54
+
55
+ if (!running) {
56
+ running = runCommand(command, name, params).finally(() => inflight.delete(key))
57
+ inflight.set(key, running)
58
+ }
59
+
60
+ return running
61
+ }
62
+
63
+ async function runCommand (command, name, params) {
64
+ const results = {}
65
+ const skipped = []
66
+ const errors = []
67
+
68
+ for (const id of await lavoratoreApplications()) {
69
+ try {
70
+ const result = await app.watt.runtime.sendCommandToApplication(id, name, params)
71
+
72
+ // The runtime answers {} when the application has no handler for the command
73
+ if (!result || typeof result !== 'object' || Object.keys(result).length === 0) {
74
+ skipped.push(id)
75
+ } else {
76
+ results[id] = result
77
+ }
78
+ } catch (err) {
79
+ errors.push(`${id}: ${err.message}`)
80
+ }
81
+ }
82
+
83
+ if (errors.length > 0) {
84
+ const error = new Error(`The ${command} command failed in the applications ${errors.join('; ')}`)
85
+ error.results = results
86
+ throw error
87
+ }
88
+
89
+ const aggregate = command === 'durable-objects-promote' ? aggregatePromotion(results) : aggregateDrain(results)
90
+ return skipped.length > 0 ? { ...aggregate, skipped } : aggregate
91
+ }
92
+
93
+ app.sendDurableObjectsCommand = sendDurableObjectsCommand
94
+ }
95
+
96
+ export default durableObjects
package/plugins/init.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import Watt from '../lib/watt.js'
2
+ import { refusedDurableObjects } from '../lib/durable-objects.js'
2
3
  import os from 'node:os'
3
4
 
4
5
  async function initPlugin (app) {
@@ -95,6 +96,13 @@ async function initPlugin (app) {
95
96
  // and nothing here should block the app from start
96
97
  app.log.error(err, 'Failed to get application information')
97
98
 
99
+ // ICC answered, and refused the pod: under an ICC which orchestrates Durable Objects, its Lavoratore applications
100
+ // start staged, see lib/durable-objects.js. The registration is retried, and ICC promotes the pod if it accepts it.
101
+ app.registrationRefusal = refusedDurableObjects(err)
102
+ if (app.registrationRefusal?.durableObjects?.staged === true) {
103
+ app.log.warn('ICC refused the registration of this pod: its Lavoratore applications start staged, and receive no Durable Objects until ICC accepts the pod and promotes it')
104
+ }
105
+
98
106
  // Set fallback values when ICC connection fails
99
107
  if (!app.applicationName) {
100
108
  app.applicationName = app.env.PLT_APP_NAME
package/plugins/update.js CHANGED
@@ -113,6 +113,36 @@ async function updatePlugin (app) {
113
113
  return
114
114
  }
115
115
 
116
+ // Handle the Durable Objects commands of ICC: the promotion of a staged version, with the generation of the
117
+ // approval, and the drain of a retiring one. Both are forwarded to the Lavoratore applications and answered
118
+ // with their result, so that ICC knows what happened (see lib/durable-objects.js).
119
+ if (command === 'durable-objects-promote' || command === 'durable-objects-drain') {
120
+ app.log.info({ command, params: message.params }, 'Received Durable Objects command from ICC')
121
+ try {
122
+ const result = await app.sendDurableObjectsCommand(command, message.params ?? {})
123
+ // What the applications answered: a promotion or a drain which did nothing is found here
124
+ app.log.info({ command, params: message.params, result }, 'Answered the Durable Objects command of ICC')
125
+ if (message.requestId) {
126
+ socket.send(JSON.stringify({
127
+ requestId: message.requestId,
128
+ // The command ran: whether it promoted the pod is in the result, which ICC reads
129
+ success: true,
130
+ result
131
+ }))
132
+ }
133
+ } catch (error) {
134
+ if (message.requestId) {
135
+ socket.send(JSON.stringify({
136
+ requestId: message.requestId,
137
+ success: false,
138
+ error: { message: error.message, name: error.name }
139
+ }))
140
+ }
141
+ throw error
142
+ }
143
+ return
144
+ }
145
+
116
146
  // Handle updates websocket format: { type: '...', topic: '...', data: {...} }
117
147
  if (!topic || !type) {
118
148
  app.log.warn({ message }, 'Received invalid message from updates websocket')
@@ -0,0 +1,101 @@
1
+ import assert from 'node:assert'
2
+ import { test } from 'node:test'
3
+ import { randomUUID } from 'node:crypto'
4
+ import { join } from 'node:path'
5
+ import { setUpEnvironment, startICC } from './helper.js'
6
+ import pino from 'pino'
7
+ import buildApp from '../app.js'
8
+ import { start } from '../index.js'
9
+ import { LAVORATORE_TYPES } from '../lib/durable-objects.js'
10
+
11
+ // With a real runtime between watt-extra and the application: the mocks of durable-objects.test.js give the
12
+ // applications a type which the runtime does not report. The application of the fixture stands for a Lavoratore one:
13
+ // it answers the commands, and reports the placement it started with.
14
+ const applicationName = 'test-application'
15
+
16
+ async function startWith (t, controlPlaneResponse, accepted = true) {
17
+ const applicationId = randomUUID()
18
+ const icc = await startICC(t, { applicationId, applicationName, controlPlaneResponse })
19
+
20
+ setUpEnvironment({
21
+ PLT_APP_NAME: applicationName,
22
+ PLT_APP_DIR: join(import.meta.dirname, 'fixtures', 'runtime-lavoratore'),
23
+ PLT_ICC_URL: icc.iccUrl,
24
+ PLT_APP_PORT: 3047,
25
+ PLT_METRICS_PORT: 9097
26
+ })
27
+
28
+ // start() retries the registration until ICC accepts it: a refused pod stops at the runtime, as a pod does while
29
+ // it retries
30
+ let app
31
+
32
+ if (accepted) {
33
+ app = await start()
34
+ } else {
35
+ app = await buildApp(pino({ level: 'silent' }))
36
+ await app.startRuntime()
37
+ }
38
+
39
+ t.after(async () => {
40
+ await app.close()
41
+ await icc.close()
42
+ })
43
+
44
+ return { app, applicationId }
45
+ }
46
+
47
+ async function placementOf (app) {
48
+ const { body } = await app.watt.runtime.inject('objects', { method: 'GET', url: '/' })
49
+ return JSON.parse(body).placement
50
+ }
51
+
52
+ function refusal (reply, extra = {}) {
53
+ reply.code(409)
54
+ return { statusCode: 409, code: 'PLT_CONTROL_PLANE_VERSION_DEPLOYMENT_MISMATCH', error: 'Conflict', message: 'Version "v1" is already bound to another controller or image', ...extra }
55
+ }
56
+
57
+ test('the Durable Objects commands reach a Lavoratore application of a real runtime', async (t) => {
58
+ const { app } = await startWith(t, {
59
+ applicationId: randomUUID(),
60
+ applicationName,
61
+ deploymentVersion: 'v1',
62
+ iccServices: {},
63
+ config: {},
64
+ durableObjects: { staged: false }
65
+ })
66
+
67
+ // The premise: the runtime configuration does not say that the application is a Lavoratore one
68
+ const applications = app.watt.runtime.getRuntimeConfig(true).applications
69
+ assert.deepStrictEqual(applications.map(application => application.id).sort(), ['objects', 'plain'])
70
+ assert.ok(applications.every(application => !LAVORATORE_TYPES.includes(application.type)))
71
+
72
+ // ICC said that the version is not staged
73
+ assert.deepStrictEqual(await placementOf(app), { type: 'coordinator', promoted: true })
74
+
75
+ assert.deepStrictEqual(await app.sendDurableObjectsCommand('durable-objects-promote', { generation: 4 }), {
76
+ promoted: true,
77
+ applications: { objects: { promoted: true, status: 'applied', generation: 4, admission: 1 } }
78
+ })
79
+
80
+ assert.deepStrictEqual(await app.sendDurableObjectsCommand('durable-objects-drain', { wait: false, generation: 5 }), {
81
+ status: 'drained',
82
+ applications: { objects: { status: 'drained', moved: 2 } }
83
+ })
84
+ })
85
+
86
+ test('a pod refused by an ICC which orchestrates Durable Objects starts its Lavoratore applications staged', async (t) => {
87
+ const { app } = await startWith(t, (_req, reply) => refusal(reply, { durableObjects: { staged: true } }), false)
88
+
89
+ assert.deepStrictEqual(app.registrationRefusal, { durableObjects: { staged: true } })
90
+ assert.deepStrictEqual(await placementOf(app), { type: 'coordinator', promoted: false })
91
+
92
+ // The pod still answers the promotion ICC sends once it accepts it
93
+ assert.strictEqual((await app.sendDurableObjectsCommand('durable-objects-promote', { generation: 2 })).promoted, true)
94
+ })
95
+
96
+ test('a pod refused by an ICC which knows nothing of Durable Objects starts as before', async (t) => {
97
+ const { app } = await startWith(t, (_req, reply) => refusal(reply), false)
98
+
99
+ assert.strictEqual(app.registrationRefusal, null)
100
+ assert.deepStrictEqual(await placementOf(app), { type: 'coordinator' })
101
+ })
@@ -0,0 +1,317 @@
1
+ import { test } from 'node:test'
2
+ import assert from 'node:assert'
3
+ import { once } from 'node:events'
4
+ import { WebSocketServer } from 'ws'
5
+ import { setUpEnvironment } from './helper.js'
6
+ import { aggregateDrain, aggregatePromotion, durableObjectsPatch, isLavoratoreApplication, placesDurableObjects, refusedDurableObjects } from '../lib/durable-objects.js'
7
+ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
8
+ import { tmpdir } from 'node:os'
9
+ import { join } from 'node:path'
10
+ import durableObjectsPlugin from '../plugins/durable-objects.js'
11
+ import updatePlugin from '../plugins/update.js'
12
+
13
+ const logger = { info () {}, error () {}, warn () {}, debug () {} }
14
+
15
+ function createMockRuntime (applications, results = {}) {
16
+ const commands = []
17
+
18
+ return {
19
+ commands,
20
+ getRuntimeConfig () {
21
+ return { applications }
22
+ },
23
+ async sendCommandToApplication (id, name, params) {
24
+ commands.push({ id, name, params })
25
+ const result = results[id]
26
+ return typeof result === 'function' ? result() : result ?? {}
27
+ }
28
+ }
29
+ }
30
+
31
+ async function createApp (runtime) {
32
+ const app = { log: logger, watt: { runtime } }
33
+ await durableObjectsPlugin(app)
34
+ return app
35
+ }
36
+
37
+ test('the patch of a Lavoratore application follows what ICC says about Durable Objects', () => {
38
+ // An ICC which knows nothing of Durable Objects changes nothing
39
+ assert.strictEqual(durableObjectsPatch(null), null)
40
+ assert.strictEqual(durableObjectsPatch({ applicationId: 'app' }), null)
41
+ assert.strictEqual(durableObjectsPatch({ durableObjects: 'yes' }), null)
42
+
43
+ // A staged version receives no objects until ICC promotes it
44
+ assert.deepStrictEqual(durableObjectsPatch({ durableObjects: { staged: true } }), [
45
+ { op: 'merge', path: '/lavoratore/durableObjects/placement', value: { promoted: false } }
46
+ ])
47
+ assert.deepStrictEqual(durableObjectsPatch({ durableObjects: { staged: false } }), [
48
+ { op: 'merge', path: '/lavoratore/durableObjects/placement', value: { promoted: true } }
49
+ ])
50
+ assert.deepStrictEqual(durableObjectsPatch({ durableObjects: {} })[0].value, { promoted: true })
51
+ })
52
+
53
+ test('Lavoratore applications are recognized by their type or by their configuration file', async (t) => {
54
+ const dir = await mkdtemp(join(tmpdir(), 'watt-extra-lavoratore-'))
55
+ t.after(() => rm(dir, { recursive: true, force: true }))
56
+
57
+ async function application (name, file, content) {
58
+ const path = join(dir, name)
59
+ await mkdir(path, { recursive: true })
60
+ await writeFile(join(path, file), typeof content === 'string' ? content : JSON.stringify(content))
61
+ return path
62
+ }
63
+
64
+ assert.strictEqual(await isLavoratoreApplication({ type: '@platformatic/lavoratore' }), true)
65
+ assert.strictEqual(await isLavoratoreApplication({ type: 'lavoratore' }), true)
66
+ assert.strictEqual(await isLavoratoreApplication({ type: '@platformatic/service' }), false)
67
+ assert.strictEqual(await isLavoratoreApplication(undefined), false)
68
+
69
+ // The runtime configuration says "unknown" for an application whose capability is resolved when it starts
70
+ const byModule = await application('module', 'watt.json', { module: '@platformatic/lavoratore' })
71
+ const bySchema = await application('schema', 'platformatic.json', { $schema: 'https://schemas.platformatic.dev/@platformatic/lavoratore/0.6.0.json' })
72
+ const sibling = await application('sibling', 'watt.json', { $schema: 'https://schemas.platformatic.dev/@platformatic/lavoratore-caposquadra/0.6.0.json', module: '@platformatic/lavoratore-caposquadra' })
73
+ const broken = await application('broken', 'watt.json', '{ not json')
74
+
75
+ assert.strictEqual(await isLavoratoreApplication({ type: 'unknown', path: byModule }), true)
76
+ assert.strictEqual(await isLavoratoreApplication({ type: 'unknown', path: bySchema }), true)
77
+ assert.strictEqual(await isLavoratoreApplication({ type: 'unknown', config: join(byModule, 'watt.json') }), true)
78
+ assert.strictEqual(await isLavoratoreApplication({ type: 'unknown', path: sibling }), false)
79
+ assert.strictEqual(await isLavoratoreApplication({ type: 'unknown', path: broken }), false)
80
+ assert.strictEqual(await isLavoratoreApplication({ type: 'unknown', path: join(dir, 'missing') }), false)
81
+
82
+ // The placement is read from the same file
83
+ const placed = await application('placed', 'watt.json', { module: '@platformatic/lavoratore', lavoratore: { durableObjects: { placement: { type: 'coordinator' } } } })
84
+ assert.strictEqual(await placesDurableObjects({ path: placed }), true)
85
+ assert.strictEqual(await placesDurableObjects({ path: byModule }), false)
86
+ })
87
+
88
+ test('only a refusal of ICC which names Durable Objects stages the pod', () => {
89
+ const body = { statusCode: 409, code: 'PLT_CONTROL_PLANE_VERSION_DEPLOYMENT_MISMATCH', error: 'Conflict', message: 'refused' }
90
+
91
+ assert.deepStrictEqual(refusedDurableObjects(new Error(JSON.stringify({ ...body, durableObjects: { staged: true } }))), { durableObjects: { staged: true } })
92
+ assert.deepStrictEqual(durableObjectsPatch(refusedDurableObjects(new Error(JSON.stringify({ ...body, durableObjects: { staged: true } })))), [
93
+ { op: 'merge', path: '/lavoratore/durableObjects/placement', value: { promoted: false } }
94
+ ])
95
+
96
+ // An ICC which knows nothing of Durable Objects, one which fails, and one which cannot be reached
97
+ assert.strictEqual(refusedDurableObjects(new Error(JSON.stringify(body))), null)
98
+ assert.strictEqual(refusedDurableObjects(new Error(JSON.stringify({ ...body, statusCode: 503, durableObjects: { staged: true } }))), null)
99
+ assert.strictEqual(refusedDurableObjects(new Error(JSON.stringify({ ...body, durableObjects: 'yes' }))), null)
100
+ assert.strictEqual(refusedDurableObjects(new Error('connect ECONNREFUSED 127.0.0.1:3000')), null)
101
+ assert.strictEqual(refusedDurableObjects(new Error('null')), null)
102
+ assert.strictEqual(refusedDurableObjects(undefined), null)
103
+ })
104
+
105
+ test('the answers of the applications are aggregated', () => {
106
+ assert.deepStrictEqual(aggregatePromotion({ a: { promoted: true }, b: { promoted: true } }), { promoted: true, applications: { a: { promoted: true }, b: { promoted: true } } })
107
+ assert.strictEqual(aggregatePromotion({ a: { promoted: true }, b: { promoted: false, status: 'duplicate' } }).promoted, false)
108
+ // An application which places nothing across pods has nothing to promote
109
+ assert.strictEqual(aggregatePromotion({ a: { promoted: true }, b: { promoted: false, status: 'unsupported' } }).promoted, true)
110
+ assert.strictEqual(aggregatePromotion({ b: { promoted: false, status: 'unsupported' } }).promoted, true)
111
+ // Nothing to promote or drain
112
+ assert.strictEqual(aggregatePromotion({}).promoted, true)
113
+ assert.strictEqual(aggregateDrain({}).status, 'drained')
114
+ assert.strictEqual(aggregateDrain({ a: { status: 'unsupported' }, b: { status: 'drained' } }).status, 'drained')
115
+ assert.strictEqual(aggregateDrain({ a: { status: 'waiting' }, b: { status: 'drained' } }).status, 'draining')
116
+ assert.strictEqual(aggregateDrain({ a: { status: 'superseded' }, b: { status: 'drained' } }).status, 'partial')
117
+
118
+ assert.strictEqual(aggregateDrain({ a: { status: 'drained' }, b: { status: 'drained' } }).status, 'drained')
119
+ assert.strictEqual(aggregateDrain({ a: { status: 'drained' }, b: { status: 'draining' } }).status, 'draining')
120
+ assert.strictEqual(aggregateDrain({ a: { status: 'partial' }, b: { status: 'draining' } }).status, 'draining')
121
+ assert.strictEqual(aggregateDrain({ a: { status: 'drained' }, b: { status: 'blocked' } }).status, 'blocked')
122
+ assert.strictEqual(aggregateDrain({ a: { status: 'blocked' }, b: { status: 'partial' } }).status, 'partial')
123
+ })
124
+
125
+ test('the commands are forwarded to every Lavoratore application of the runtime', async () => {
126
+ const applications = [
127
+ { id: 'front', type: '@platformatic/service' },
128
+ { id: 'objects', type: '@platformatic/lavoratore' },
129
+ { id: 'legacy', type: 'lavoratore' }
130
+ ]
131
+ const runtime = createMockRuntime(applications, {
132
+ objects: { promoted: true, status: 'applied', generation: 3, admission: 2 },
133
+ legacy: { promoted: true, status: 'duplicate', generation: 3, admission: 1 }
134
+ })
135
+ const app = await createApp(runtime)
136
+
137
+ const promotion = await app.sendDurableObjectsCommand('durable-objects-promote', { generation: 3 })
138
+ assert.strictEqual(promotion.promoted, true)
139
+ assert.deepStrictEqual(Object.keys(promotion.applications), ['objects', 'legacy'])
140
+ assert.deepStrictEqual(runtime.commands, [
141
+ { id: 'objects', name: 'plt:lavoratore:promote', params: { generation: 3 } },
142
+ { id: 'legacy', name: 'plt:lavoratore:promote', params: { generation: 3 } }
143
+ ])
144
+
145
+ const draining = createMockRuntime(applications, {
146
+ objects: { status: 'drained', moved: 2 },
147
+ legacy: { status: 'blocked', blocked: 1 }
148
+ })
149
+ const drain = await (await createApp(draining)).sendDurableObjectsCommand('durable-objects-drain')
150
+ assert.strictEqual(drain.status, 'blocked')
151
+ assert.deepStrictEqual(drain.applications.objects, { status: 'drained', moved: 2 })
152
+ assert.deepStrictEqual(draining.commands.map(command => command.name), ['plt:lavoratore:drain', 'plt:lavoratore:drain'])
153
+ })
154
+
155
+ test('applications without the handler are skipped, failures are reported with the other answers, and repeated commands join', async () => {
156
+ // The runtime answers {} when the application has no handler: a plain Worker, or an older Lavoratore
157
+ const mixed = createMockRuntime([{ id: 'front', type: '@platformatic/lavoratore' }, { id: 'objects', type: '@platformatic/lavoratore' }], {
158
+ objects: { status: 'drained', moved: 1 }
159
+ })
160
+ assert.deepStrictEqual(await (await createApp(mixed)).sendDurableObjectsCommand('durable-objects-drain', { wait: false }), {
161
+ status: 'drained', applications: { objects: { status: 'drained', moved: 1 } }, skipped: ['front']
162
+ })
163
+
164
+ const plain = await createApp(createMockRuntime([{ id: 'front', type: '@platformatic/lavoratore' }]))
165
+ assert.deepStrictEqual(await plain.sendDurableObjectsCommand('durable-objects-promote', { generation: 1 }), { promoted: true, applications: {}, skipped: ['front'] })
166
+ assert.deepStrictEqual(await plain.sendDurableObjectsCommand('durable-objects-drain'), { status: 'drained', applications: {}, skipped: ['front'] })
167
+
168
+ // ICC asks only the pods it knows to run Lavoratore: finding none is a failure, never an answer it takes for done
169
+ const none = await createApp(createMockRuntime([{ id: 'api', type: '@platformatic/service' }]))
170
+ await assert.rejects(none.sendDurableObjectsCommand('durable-objects-drain'), /No Lavoratore application found/)
171
+ await assert.rejects(none.sendDurableObjectsCommand('durable-objects-promote', { generation: 1 }), /No Lavoratore application found/)
172
+
173
+ // One application fails: the other is still asked, and the error carries its answer
174
+ const failing = createMockRuntime([{ id: 'a', type: '@platformatic/lavoratore' }, { id: 'b', type: '@platformatic/lavoratore' }], {
175
+ a: () => { throw new Error('boom') },
176
+ b: { promoted: true, status: 'applied' }
177
+ })
178
+ await assert.rejects((await createApp(failing)).sendDurableObjectsCommand('durable-objects-promote', { generation: 2 }), err => {
179
+ assert.match(err.message, /durable-objects-promote command failed in the applications a: boom/)
180
+ assert.deepStrictEqual(err.results, { b: { promoted: true, status: 'applied' } })
181
+ return true
182
+ })
183
+ assert.strictEqual(failing.commands.length, 2)
184
+
185
+ // The same command repeated while it runs joins it; another generation does not
186
+ let release
187
+ const slow = createMockRuntime([{ id: 'objects', type: '@platformatic/lavoratore' }], {
188
+ objects: () => new Promise(resolve => { release = () => resolve({ promoted: true }) })
189
+ })
190
+ const app = await createApp(slow)
191
+ const first = app.sendDurableObjectsCommand('durable-objects-promote', { generation: 3 })
192
+ const second = app.sendDurableObjectsCommand('durable-objects-promote', { generation: 3 })
193
+ await new Promise(resolve => setImmediate(resolve))
194
+ assert.strictEqual(slow.commands.length, 1)
195
+ release()
196
+ assert.deepStrictEqual(await first, await second)
197
+ const third = app.sendDurableObjectsCommand('durable-objects-promote', { generation: 3 })
198
+ await new Promise(resolve => setImmediate(resolve))
199
+ assert.strictEqual(slow.commands.length, 2)
200
+ release()
201
+ await third
202
+
203
+ await assert.rejects(app.sendDurableObjectsCommand('durable-objects-restart'), /Unknown Durable Objects command/)
204
+
205
+ const stopped = { log: logger, watt: { runtime: null } }
206
+ await durableObjectsPlugin(stopped)
207
+ await assert.rejects(stopped.sendDurableObjectsCommand('durable-objects-drain'), /Runtime not started/)
208
+ })
209
+
210
+ test('the patch applies only to the applications which place Durable Objects', async (t) => {
211
+ const dir = await mkdtemp(join(tmpdir(), 'watt-extra-do-'))
212
+ t.after(() => rm(dir, { recursive: true, force: true }))
213
+ const write = async (name, content) => {
214
+ const path = join(dir, name)
215
+ await writeFile(path, typeof content === 'string' ? content : JSON.stringify(content))
216
+ return { config: path }
217
+ }
218
+
219
+ assert.strictEqual(await placesDurableObjects(await write('placed.json', { lavoratore: { durableObjects: { placement: { type: 'coordinator' } } } })), true)
220
+ assert.strictEqual(await placesDurableObjects(await write('local.json', { lavoratore: { durableObjects: {} } })), false)
221
+ assert.strictEqual(await placesDurableObjects(await write('plain.json', { lavoratore: {} })), false)
222
+ // A configuration which cannot be read gets the patch
223
+ assert.strictEqual(await placesDurableObjects(await write('broken.json', '{ not json')), true)
224
+ assert.strictEqual(await placesDurableObjects({ config: join(dir, 'missing.json') }), true)
225
+ assert.strictEqual(await placesDurableObjects({}), true)
226
+ })
227
+
228
+ function setupMockIccServer (wss) {
229
+ let ws = null
230
+
231
+ const waitForClientSubscription = once(wss, 'connection').then(([socket]) => {
232
+ ws = socket
233
+
234
+ return new Promise((resolve) => {
235
+ socket.on('message', (data) => {
236
+ const message = JSON.parse(data.toString())
237
+
238
+ if (message.command === 'subscribe' && message.topic === '/config') {
239
+ socket.send(JSON.stringify({ command: 'ack' }))
240
+ resolve()
241
+ }
242
+ })
243
+ })
244
+ })
245
+
246
+ return { waitForClientSubscription, getWs: () => ws }
247
+ }
248
+
249
+ async function reply (ws, frame) {
250
+ ws.send(JSON.stringify(frame))
251
+ return JSON.parse((await once(ws, 'message'))[0].toString())
252
+ }
253
+
254
+ test('the Durable Objects commands of ICC are answered over the websocket with their result', async (t) => {
255
+ setUpEnvironment()
256
+ const port = 14300
257
+ const wss = new WebSocketServer({ port })
258
+ t.after(() => wss.close())
259
+ const { waitForClientSubscription, getWs } = setupMockIccServer(wss)
260
+
261
+ const received = []
262
+ const app = {
263
+ log: logger,
264
+ instanceConfig: { applicationId: 'test-application-id' },
265
+ getAuthorizationHeaders: async () => ({ Authorization: 'Bearer test-token' }),
266
+ getRuntimeId: () => 'test-runtime-id',
267
+ env: { PLT_APP_NAME: 'test-app', PLT_APP_DIR: '/path/to/app', PLT_ICC_URL: `http://localhost:${port}`, PLT_UPDATES_RECONNECT_INTERVAL_SEC: 1 },
268
+ async sendDurableObjectsCommand (command, params) {
269
+ received.push({ command, params })
270
+
271
+ if (params.generation === 7) {
272
+ return { promoted: false, applications: { objects: { promoted: false, status: 'superseded', generation: 7, admission: null } } }
273
+ }
274
+
275
+ if (params.generation === 8) {
276
+ throw new Error('The application "objects" does not handle the durable-objects-promote command')
277
+ }
278
+
279
+ return command === 'durable-objects-promote'
280
+ ? { promoted: true, applications: { objects: { promoted: true, status: 'applied', generation: params.generation, admission: 1 } } }
281
+ : { status: 'drained', applications: { objects: { status: 'drained', moved: 3 } } }
282
+ }
283
+ }
284
+
285
+ await updatePlugin(app)
286
+ await app.connectToUpdates()
287
+ t.after(() => app.closeUpdates())
288
+ await waitForClientSubscription
289
+
290
+ assert.deepStrictEqual(await reply(getWs(), { command: 'durable-objects-promote', requestId: 'promote-1', params: { generation: 2 } }), {
291
+ requestId: 'promote-1',
292
+ success: true,
293
+ result: { promoted: true, applications: { objects: { promoted: true, status: 'applied', generation: 2, admission: 1 } } }
294
+ })
295
+
296
+ // A promotion which did not promote the pod says so in its result: the frame stays valid for ICC, which reads it
297
+ const superseded = await reply(getWs(), { command: 'durable-objects-promote', requestId: 'promote-2', params: { generation: 7 } })
298
+ assert.strictEqual(superseded.success, true)
299
+ assert.strictEqual(superseded.result.promoted, false)
300
+ assert.strictEqual(superseded.result.applications.objects.status, 'superseded')
301
+
302
+ const failed = await reply(getWs(), { command: 'durable-objects-promote', requestId: 'promote-3', params: { generation: 8 } })
303
+ assert.deepStrictEqual(failed, { requestId: 'promote-3', success: false, error: { message: 'The application "objects" does not handle the durable-objects-promote command', name: 'Error' } })
304
+
305
+ assert.deepStrictEqual(await reply(getWs(), { command: 'durable-objects-drain', requestId: 'drain-1' }), {
306
+ requestId: 'drain-1',
307
+ success: true,
308
+ result: { status: 'drained', applications: { objects: { status: 'drained', moved: 3 } } }
309
+ })
310
+
311
+ assert.deepStrictEqual(received, [
312
+ { command: 'durable-objects-promote', params: { generation: 2 } },
313
+ { command: 'durable-objects-promote', params: { generation: 7 } },
314
+ { command: 'durable-objects-promote', params: { generation: 8 } },
315
+ { command: 'durable-objects-drain', params: {} }
316
+ ])
317
+ })
@@ -0,0 +1,19 @@
1
+ // A stand-in for the Lavoratore capability: the Node.js one, accepting the "lavoratore" key of the configuration, which
2
+ // watt-extra patches
3
+ import * as node from '@platformatic/node'
4
+
5
+ export const { Generator, loadConfiguration, packageJson, schemaComponents, transform, version } = node
6
+
7
+ export const schema = {
8
+ ...node.schema,
9
+ properties: { ...node.schema.properties, lavoratore: { type: 'object', additionalProperties: true } }
10
+ }
11
+
12
+ export async function create (configOrRoot, sourceOrConfig, context) {
13
+ const capability = await node.create(configOrRoot, sourceOrConfig, { ...context, schema })
14
+
15
+ // For the application, which reports the placement it started with
16
+ globalThis.lavoratoreStandIn = { placement: () => capability.config?.lavoratore?.durableObjects?.placement }
17
+
18
+ return capability
19
+ }
@@ -0,0 +1,23 @@
1
+ // Stands for a Lavoratore application: the runtime reports no type for it, as for the real one, and it answers the
2
+ // commands of ICC through the ITC of the runtime
3
+ import { createServer } from 'node:http'
4
+
5
+ const commands = []
6
+
7
+ globalThis.platformatic.itc.handle('plt:lavoratore:promote', message => {
8
+ commands.push({ name: 'promote', message })
9
+ return { promoted: true, status: 'applied', generation: message.generation, admission: 1 }
10
+ })
11
+
12
+ globalThis.platformatic.itc.handle('plt:lavoratore:drain', message => {
13
+ commands.push({ name: 'drain', message })
14
+ return { status: 'drained', moved: commands.length }
15
+ })
16
+
17
+ export function create () {
18
+ return createServer((_req, res) => {
19
+ res.setHeader('content-type', 'application/json')
20
+ // With the configuration the application started with, i.e. patched by watt-extra
21
+ res.end(JSON.stringify({ commands, placement: globalThis.lavoratoreStandIn.placement() }))
22
+ })
23
+ }
@@ -0,0 +1,3 @@
1
+ {
2
+ "type": "module"
3
+ }
@@ -0,0 +1,14 @@
1
+ {
2
+ "$schema": "https://schemas.platformatic.dev/@platformatic/lavoratore/0.6.0.json",
3
+ "module": "./capability.js",
4
+ "node": {
5
+ "main": "index.js"
6
+ },
7
+ "lavoratore": {
8
+ "durableObjects": {
9
+ "placement": {
10
+ "type": "coordinator"
11
+ }
12
+ }
13
+ }
14
+ }
@@ -0,0 +1,7 @@
1
+ import { createServer } from 'node:http'
2
+
3
+ export function create () {
4
+ return createServer((_req, res) => {
5
+ res.end('ok')
6
+ })
7
+ }
@@ -0,0 +1,3 @@
1
+ {
2
+ "type": "module"
3
+ }
@@ -0,0 +1,11 @@
1
+ {
2
+ "$schema": "https://schemas.platformatic.dev/@platformatic/runtime/3.0.0.json",
3
+ "entrypoint": "plain",
4
+ "applications": [
5
+ { "id": "objects", "path": "./objects" },
6
+ { "id": "plain", "path": "./plain" }
7
+ ],
8
+ "logger": {
9
+ "level": "error"
10
+ }
11
+ }
package/test/helper.js CHANGED
@@ -89,9 +89,9 @@ async function startICC (t, opts = {}) {
89
89
 
90
90
  // Control Plane
91
91
  await icc.register(async (icc) => {
92
- icc.post('/pods/:podId/instance', async (req) => {
92
+ icc.post('/pods/:podId/instance', async (req, reply) => {
93
93
  if (typeof controlPlaneResponse === 'function') {
94
- return controlPlaneResponse(req)
94
+ return controlPlaneResponse(req, reply)
95
95
  }
96
96
  return controlPlaneResponse || {
97
97
  applicationId,