@platformatic/watt-extra 1.21.0 → 1.22.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app.js +2 -0
- package/lib/durable-objects.js +161 -0
- package/lib/watt.js +24 -0
- package/package.json +1 -1
- package/plugins/durable-objects.js +106 -0
- package/plugins/init.js +8 -0
- package/plugins/update.js +41 -0
- package/test/durable-objects-runtime.test.js +101 -0
- package/test/durable-objects.test.js +353 -0
- package/test/fixtures/runtime-lavoratore/objects/capability.js +19 -0
- package/test/fixtures/runtime-lavoratore/objects/index.js +23 -0
- package/test/fixtures/runtime-lavoratore/objects/package.json +3 -0
- package/test/fixtures/runtime-lavoratore/objects/watt.json +14 -0
- package/test/fixtures/runtime-lavoratore/package.json +1 -0
- package/test/fixtures/runtime-lavoratore/plain/index.js +7 -0
- package/test/fixtures/runtime-lavoratore/plain/package.json +3 -0
- package/test/fixtures/runtime-lavoratore/watt.json +11 -0
- package/test/helper.js +2 -2
package/app.js
CHANGED
|
@@ -10,6 +10,7 @@ import update from './plugins/update.js'
|
|
|
10
10
|
import alert from './plugins/alerts.js'
|
|
11
11
|
import healthSignals from './plugins/health-signals.js'
|
|
12
12
|
import flamegraphs from './plugins/flamegraphs.js'
|
|
13
|
+
import durableObjects from './plugins/durable-objects.js'
|
|
13
14
|
|
|
14
15
|
async function buildApp (logger) {
|
|
15
16
|
const app = {
|
|
@@ -27,6 +28,7 @@ async function buildApp (logger) {
|
|
|
27
28
|
.use(metadata)
|
|
28
29
|
.use(compliancy)
|
|
29
30
|
.use(scheduler)
|
|
31
|
+
.use(durableObjects)
|
|
30
32
|
.use(update)
|
|
31
33
|
.use(flamegraphs)
|
|
32
34
|
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises'
|
|
2
|
+
import { join } from 'node:path'
|
|
3
|
+
|
|
4
|
+
// The Durable Objects of Lavoratore applications (@platformatic/lavoratore), orchestrated by ICC: see
|
|
5
|
+
// docs/durable-objects.md of Lavoratore. Everything here is optional: an ICC which knows nothing of Durable Objects
|
|
6
|
+
// sends no `durableObjects` with the instance config and no command, and nothing changes.
|
|
7
|
+
|
|
8
|
+
export const LAVORATORE_TYPES = ['@platformatic/lavoratore', 'lavoratore']
|
|
9
|
+
|
|
10
|
+
// The ITC commands handled by the Lavoratore applications, with the same names and payloads as their runtime messages
|
|
11
|
+
export const COMMANDS = {
|
|
12
|
+
'durable-objects-promote': 'plt:lavoratore:promote',
|
|
13
|
+
'durable-objects-drain': 'plt:lavoratore:drain',
|
|
14
|
+
'durable-objects-list': 'plt:lavoratore:objects'
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// The names a runtime gives to the configuration file of an application which declares none
|
|
18
|
+
const CONFIG_FILES = ['watt.json', 'platformatic.json', 'platformatic.application.json']
|
|
19
|
+
|
|
20
|
+
// The configuration file of an application, parsed, or null when there is none or it cannot be read as JSON
|
|
21
|
+
async function readApplicationConfig (application) {
|
|
22
|
+
const candidates = []
|
|
23
|
+
|
|
24
|
+
if (application?.config) {
|
|
25
|
+
candidates.push(application.config)
|
|
26
|
+
} else if (application?.path) {
|
|
27
|
+
for (const file of CONFIG_FILES) {
|
|
28
|
+
candidates.push(join(application.path, file))
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
for (const candidate of candidates) {
|
|
33
|
+
try {
|
|
34
|
+
return JSON.parse(await readFile(candidate, 'utf8'))
|
|
35
|
+
} catch {}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
return null
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Whether an application of the runtime configuration is a Lavoratore one. The runtime configuration reports the type
|
|
42
|
+
// only for some applications, and "unknown" for one whose capability it resolves when it starts, so the configuration
|
|
43
|
+
// file of the application decides too: its module, or its schema.
|
|
44
|
+
export async function isLavoratoreApplication (application) {
|
|
45
|
+
if (LAVORATORE_TYPES.includes(application?.type)) {
|
|
46
|
+
return true
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const config = await readApplicationConfig(application)
|
|
50
|
+
|
|
51
|
+
if (!config) {
|
|
52
|
+
return false
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return LAVORATORE_TYPES.includes(config.module) || /\/@platformatic\/lavoratore\/[^/]+\.json$/.test(config.$schema ?? '')
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// The configuration patch of a Lavoratore application under ICC, or null when ICC said nothing about Durable Objects:
|
|
59
|
+
// a pod of a staged version receives no objects until ICC promotes it, with the generation of the approval
|
|
60
|
+
export function durableObjectsPatch (instanceConfig) {
|
|
61
|
+
const durableObjects = instanceConfig?.durableObjects
|
|
62
|
+
|
|
63
|
+
if (!durableObjects || typeof durableObjects !== 'object') {
|
|
64
|
+
return null
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return [
|
|
68
|
+
{
|
|
69
|
+
op: 'merge',
|
|
70
|
+
path: '/lavoratore/durableObjects/placement',
|
|
71
|
+
value: { promoted: durableObjects.staged !== true }
|
|
72
|
+
}
|
|
73
|
+
]
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Whether the patch applies to a Lavoratore application: only one which places Durable Objects across pods, i.e. whose
|
|
77
|
+
// configuration has a placement. A configuration which cannot be read gets the patch: the application ignores it
|
|
78
|
+
// without Durable Objects.
|
|
79
|
+
export async function placesDurableObjects (application) {
|
|
80
|
+
const config = await readApplicationConfig(application)
|
|
81
|
+
return config ? config.lavoratore?.durableObjects?.placement != null : true
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// What ICC said about Durable Objects when it refused the registration of the pod, or null. The error of the client
|
|
85
|
+
// carries the body of the response: an ICC which orchestrates Durable Objects adds `durableObjects` to its refusals,
|
|
86
|
+
// i.e. the 4xx answers, since a pod it does not accept must not receive objects. Any other failure, an ICC which cannot
|
|
87
|
+
// be reached, which fails, or which knows nothing of Durable Objects, says nothing, and nothing changes.
|
|
88
|
+
export function refusedDurableObjects (err) {
|
|
89
|
+
let body
|
|
90
|
+
|
|
91
|
+
try {
|
|
92
|
+
body = JSON.parse(err?.message)
|
|
93
|
+
} catch {
|
|
94
|
+
return null
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
if (!body || typeof body !== 'object' || !(body.statusCode >= 400 && body.statusCode < 500)) {
|
|
98
|
+
return null
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const durableObjects = body.durableObjects
|
|
102
|
+
return durableObjects && typeof durableObjects === 'object' ? { durableObjects } : null
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// The result of a promotion of the Lavoratore applications of the pod: promoted when all those which place Durable
|
|
106
|
+
// Objects across pods are. An application which does not answers { status: 'unsupported' }, or has no handler with a
|
|
107
|
+
// version of Lavoratore which predates that answer: it has nothing to promote, and a pod with no other has none either.
|
|
108
|
+
export function aggregatePromotion (results) {
|
|
109
|
+
return {
|
|
110
|
+
promoted: Object.values(results).every(result => result.promoted === true || result.status === 'unsupported'),
|
|
111
|
+
applications: results
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// The status of a drain of the Lavoratore applications of the pod which handle it: draining while any is, or waits for
|
|
116
|
+
// the newer version; drained when all are, or cannot drain at all, or none handles it; blocked when none is partial and
|
|
117
|
+
// some are blocked; partial otherwise.
|
|
118
|
+
export function aggregateDrain (results) {
|
|
119
|
+
const statuses = Object.values(results).map(result => result.status)
|
|
120
|
+
const done = value => value === 'drained' || value === 'unsupported'
|
|
121
|
+
let status = 'partial'
|
|
122
|
+
|
|
123
|
+
if (statuses.some(value => value === 'draining' || value === 'waiting')) {
|
|
124
|
+
status = 'draining'
|
|
125
|
+
} else if (statuses.every(done)) {
|
|
126
|
+
status = 'drained'
|
|
127
|
+
} else if (statuses.every(value => done(value) || value === 'blocked')) {
|
|
128
|
+
status = 'blocked'
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
return { status, applications: results }
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// The objects the Lavoratore applications of the pod hold, each with its application: active, or hibernated. An
|
|
135
|
+
// application without Durable Objects answers { status: 'unsupported' } and has none. Truncated when an application
|
|
136
|
+
// holds more objects than it lists: total counts them all. applications are those which listed their objects, even
|
|
137
|
+
// none: ICC tells from them a pod which holds no object from one which cannot list, and from the skipped ones a list
|
|
138
|
+
// which misses the objects of some applications.
|
|
139
|
+
export function aggregateObjects (results) {
|
|
140
|
+
const objects = []
|
|
141
|
+
const applications = []
|
|
142
|
+
let total = 0
|
|
143
|
+
let truncated = false
|
|
144
|
+
|
|
145
|
+
for (const [application, result] of Object.entries(results)) {
|
|
146
|
+
if (!Array.isArray(result.objects)) {
|
|
147
|
+
continue
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
applications.push(application)
|
|
151
|
+
|
|
152
|
+
for (const object of result.objects) {
|
|
153
|
+
objects.push({ application, ...object })
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
total += result.total ?? result.objects.length
|
|
157
|
+
truncated ||= result.truncated === true
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
return { total, truncated, applications, objects }
|
|
161
|
+
}
|
package/lib/watt.js
CHANGED
|
@@ -5,6 +5,7 @@ import { join, resolve } from 'node:path'
|
|
|
5
5
|
import { createRequire } from 'node:module'
|
|
6
6
|
import { pathToFileURL } from 'node:url'
|
|
7
7
|
import semver from 'semver'
|
|
8
|
+
import { durableObjectsPatch, isLavoratoreApplication, placesDurableObjects } from './durable-objects.js'
|
|
8
9
|
|
|
9
10
|
const require = createRequire(import.meta.url)
|
|
10
11
|
|
|
@@ -37,6 +38,7 @@ class Watt {
|
|
|
37
38
|
#applicationName
|
|
38
39
|
#instanceId
|
|
39
40
|
#instanceConfig
|
|
41
|
+
#registrationRefusal
|
|
40
42
|
#originalConfig
|
|
41
43
|
#config
|
|
42
44
|
#sharedContext
|
|
@@ -51,6 +53,8 @@ class Watt {
|
|
|
51
53
|
this.runtime = null
|
|
52
54
|
this.#sharedContext = {}
|
|
53
55
|
this.#instanceConfig = app.instanceConfig
|
|
56
|
+
// What ICC said about Durable Objects when it refused the registration, see refusedDurableObjects()
|
|
57
|
+
this.#registrationRefusal = app.registrationRefusal ?? null
|
|
54
58
|
}
|
|
55
59
|
|
|
56
60
|
async spawn () {
|
|
@@ -104,6 +108,13 @@ class Watt {
|
|
|
104
108
|
const previousConfig = this.#instanceConfig
|
|
105
109
|
this.#instanceConfig = instanceConfig
|
|
106
110
|
|
|
111
|
+
// The configuration of the applications was patched at boot, without ICC: a pod of a staged version started as
|
|
112
|
+
// any other, and receives Durable Objects until it restarts. Not when ICC refused the pod at boot: it started
|
|
113
|
+
// staged then, and ICC promotes it once its version is active.
|
|
114
|
+
if (!previousConfig && instanceConfig?.durableObjects?.staged === true && this.#registrationRefusal?.durableObjects?.staged !== true) {
|
|
115
|
+
this.#logger.warn('ICC was unreachable when this pod started: its Lavoratore applications were not started as staged, and receive Durable Objects before the approval of their version. Restart the pod to stage it.')
|
|
116
|
+
}
|
|
117
|
+
|
|
107
118
|
// If we didn't have a config before and now we do, apply runtime updates
|
|
108
119
|
if (!previousConfig && instanceConfig && this.runtime) {
|
|
109
120
|
// Update undici interceptors
|
|
@@ -602,10 +613,23 @@ class Watt {
|
|
|
602
613
|
await this.#configurePlatformaticServices(runtime, app)
|
|
603
614
|
} else if (app.type === '@platformatic/regina') {
|
|
604
615
|
await this.#configureReginaService(runtime, app)
|
|
616
|
+
} else if (await isLavoratoreApplication(app)) {
|
|
617
|
+
await this.#configureLavoratoreService(runtime, app)
|
|
605
618
|
}
|
|
606
619
|
}
|
|
607
620
|
}
|
|
608
621
|
|
|
622
|
+
// Under an ICC which orchestrates Durable Objects, its approvals order the deployments of the application, and a pod
|
|
623
|
+
// of a staged version starts without receiving objects, until ICC promotes it (see lib/durable-objects.js). An ICC
|
|
624
|
+
// which says nothing about Durable Objects leaves the configuration of the application alone.
|
|
625
|
+
async #configureLavoratoreService (runtime, service) {
|
|
626
|
+
const patches = durableObjectsPatch(this.#instanceConfig ?? this.#registrationRefusal)
|
|
627
|
+
|
|
628
|
+
if (patches && await placesDurableObjects(service)) {
|
|
629
|
+
await this.#patchService(runtime, service.id, patches)
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
609
633
|
async #configureNextService (runtime, service) {
|
|
610
634
|
let nextPackage
|
|
611
635
|
let nextSchema
|
package/package.json
CHANGED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { aggregateDrain, aggregateObjects, aggregatePromotion, COMMANDS, isLavoratoreApplication } from '../lib/durable-objects.js'
|
|
2
|
+
|
|
3
|
+
// Forwards the Durable Objects commands of ICC to the Lavoratore applications of the runtime, see lib/durable-objects.js
|
|
4
|
+
async function durableObjects (app, _opts) {
|
|
5
|
+
// The Lavoratore applications of the runtime: the configuration does not change while it runs
|
|
6
|
+
let applications
|
|
7
|
+
|
|
8
|
+
async function lavoratoreApplications () {
|
|
9
|
+
const runtime = app.watt?.runtime
|
|
10
|
+
|
|
11
|
+
if (!runtime) {
|
|
12
|
+
throw new Error('Runtime not started, cannot reach the Durable Objects applications')
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
if (!applications) {
|
|
16
|
+
const found = []
|
|
17
|
+
|
|
18
|
+
for (const application of runtime.getRuntimeConfig(true).applications ?? []) {
|
|
19
|
+
if (await isLavoratoreApplication(application)) {
|
|
20
|
+
found.push(application.id)
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
applications = found
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// ICC sends these commands only to the pods it knows to run Lavoratore: finding none is a failure to recognize the
|
|
28
|
+
// applications, not a pod with nothing to promote or drain, and ICC must not take it for done
|
|
29
|
+
if (applications.length === 0) {
|
|
30
|
+
throw new Error('No Lavoratore application found in the runtime')
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
return applications
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// The commands in flight, by command and parameters: ICC repeats a command which did not answer in time, and the
|
|
37
|
+
// repetition joins the one still running instead of starting another
|
|
38
|
+
const inflight = new Map()
|
|
39
|
+
|
|
40
|
+
// Runs a command on every Lavoratore application: a promotion with { generation }, a drain, or the list of the
|
|
41
|
+
// objects the pod holds. Returns the aggregate
|
|
42
|
+
// of their answers, see lib/durable-objects.js. An application which has no handler for the command, i.e. without
|
|
43
|
+
// Durable Objects placed across pods, or with a version of Lavoratore which predates the commands, is skipped: it has
|
|
44
|
+
// nothing to promote or drain, and ICC must not wait for it. The applications are all asked even when one fails, and
|
|
45
|
+
// the failure is reported with what the others answered.
|
|
46
|
+
function sendDurableObjectsCommand (command, params = {}) {
|
|
47
|
+
const name = COMMANDS[command]
|
|
48
|
+
|
|
49
|
+
if (!name) {
|
|
50
|
+
return Promise.reject(new Error(`Unknown Durable Objects command "${command}"`))
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const key = `${command}:${JSON.stringify(params)}`
|
|
54
|
+
let running = inflight.get(key)
|
|
55
|
+
|
|
56
|
+
if (!running) {
|
|
57
|
+
running = runCommand(command, name, params).finally(() => inflight.delete(key))
|
|
58
|
+
inflight.set(key, running)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return running
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async function runCommand (command, name, params) {
|
|
65
|
+
const results = {}
|
|
66
|
+
const skipped = []
|
|
67
|
+
const errors = []
|
|
68
|
+
|
|
69
|
+
for (const id of await lavoratoreApplications()) {
|
|
70
|
+
try {
|
|
71
|
+
const result = await app.watt.runtime.sendCommandToApplication(id, name, params)
|
|
72
|
+
|
|
73
|
+
// The runtime answers {} when the application has no handler for the command
|
|
74
|
+
if (!result || typeof result !== 'object' || Object.keys(result).length === 0) {
|
|
75
|
+
skipped.push(id)
|
|
76
|
+
} else {
|
|
77
|
+
results[id] = result
|
|
78
|
+
}
|
|
79
|
+
} catch (err) {
|
|
80
|
+
errors.push(`${id}: ${err.message}`)
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (errors.length > 0) {
|
|
85
|
+
const error = new Error(`The ${command} command failed in the applications ${errors.join('; ')}`)
|
|
86
|
+
error.results = results
|
|
87
|
+
throw error
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
let aggregate
|
|
91
|
+
|
|
92
|
+
if (command === 'durable-objects-promote') {
|
|
93
|
+
aggregate = aggregatePromotion(results)
|
|
94
|
+
} else if (command === 'durable-objects-list') {
|
|
95
|
+
aggregate = aggregateObjects(results)
|
|
96
|
+
} else {
|
|
97
|
+
aggregate = aggregateDrain(results)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return skipped.length > 0 ? { ...aggregate, skipped } : aggregate
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
app.sendDurableObjectsCommand = sendDurableObjectsCommand
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export default durableObjects
|
package/plugins/init.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import Watt from '../lib/watt.js'
|
|
2
|
+
import { refusedDurableObjects } from '../lib/durable-objects.js'
|
|
2
3
|
import os from 'node:os'
|
|
3
4
|
|
|
4
5
|
async function initPlugin (app) {
|
|
@@ -95,6 +96,13 @@ async function initPlugin (app) {
|
|
|
95
96
|
// and nothing here should block the app from start
|
|
96
97
|
app.log.error(err, 'Failed to get application information')
|
|
97
98
|
|
|
99
|
+
// ICC answered, and refused the pod: under an ICC which orchestrates Durable Objects, its Lavoratore applications
|
|
100
|
+
// start staged, see lib/durable-objects.js. The registration is retried, and ICC promotes the pod if it accepts it.
|
|
101
|
+
app.registrationRefusal = refusedDurableObjects(err)
|
|
102
|
+
if (app.registrationRefusal?.durableObjects?.staged === true) {
|
|
103
|
+
app.log.warn('ICC refused the registration of this pod: its Lavoratore applications start staged, and receive no Durable Objects until ICC accepts the pod and promotes it')
|
|
104
|
+
}
|
|
105
|
+
|
|
98
106
|
// Set fallback values when ICC connection fails
|
|
99
107
|
if (!app.applicationName) {
|
|
100
108
|
app.applicationName = app.env.PLT_APP_NAME
|
package/plugins/update.js
CHANGED
|
@@ -113,6 +113,47 @@ async function updatePlugin (app) {
|
|
|
113
113
|
return
|
|
114
114
|
}
|
|
115
115
|
|
|
116
|
+
// Handle the Durable Objects commands of ICC: the promotion of a staged version, with the generation of the
|
|
117
|
+
// approval, the drain of a retiring one, and the list of the objects the pod holds. They are forwarded to the
|
|
118
|
+
// Lavoratore applications and answered with their result, so that ICC knows what happened (see
|
|
119
|
+
// lib/durable-objects.js).
|
|
120
|
+
if (command === 'durable-objects-promote' || command === 'durable-objects-drain' || command === 'durable-objects-list') {
|
|
121
|
+
// The list is read again and again while somebody looks at it, and changes nothing: it is not logged
|
|
122
|
+
const logged = command !== 'durable-objects-list'
|
|
123
|
+
|
|
124
|
+
if (logged) {
|
|
125
|
+
app.log.info({ command, params: message.params }, 'Received Durable Objects command from ICC')
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
try {
|
|
129
|
+
const result = await app.sendDurableObjectsCommand(command, message.params ?? {})
|
|
130
|
+
|
|
131
|
+
if (logged) {
|
|
132
|
+
// What the applications answered: a promotion or a drain which did nothing is found here
|
|
133
|
+
app.log.info({ command, params: message.params, result }, 'Answered the Durable Objects command of ICC')
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (message.requestId) {
|
|
137
|
+
socket.send(JSON.stringify({
|
|
138
|
+
requestId: message.requestId,
|
|
139
|
+
// The command ran: whether it promoted the pod is in the result, which ICC reads
|
|
140
|
+
success: true,
|
|
141
|
+
result
|
|
142
|
+
}))
|
|
143
|
+
}
|
|
144
|
+
} catch (error) {
|
|
145
|
+
if (message.requestId) {
|
|
146
|
+
socket.send(JSON.stringify({
|
|
147
|
+
requestId: message.requestId,
|
|
148
|
+
success: false,
|
|
149
|
+
error: { message: error.message, name: error.name }
|
|
150
|
+
}))
|
|
151
|
+
}
|
|
152
|
+
throw error
|
|
153
|
+
}
|
|
154
|
+
return
|
|
155
|
+
}
|
|
156
|
+
|
|
116
157
|
// Handle updates websocket format: { type: '...', topic: '...', data: {...} }
|
|
117
158
|
if (!topic || !type) {
|
|
118
159
|
app.log.warn({ message }, 'Received invalid message from updates websocket')
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import assert from 'node:assert'
|
|
2
|
+
import { test } from 'node:test'
|
|
3
|
+
import { randomUUID } from 'node:crypto'
|
|
4
|
+
import { join } from 'node:path'
|
|
5
|
+
import { setUpEnvironment, startICC } from './helper.js'
|
|
6
|
+
import pino from 'pino'
|
|
7
|
+
import buildApp from '../app.js'
|
|
8
|
+
import { start } from '../index.js'
|
|
9
|
+
import { LAVORATORE_TYPES } from '../lib/durable-objects.js'
|
|
10
|
+
|
|
11
|
+
// With a real runtime between watt-extra and the application: the mocks of durable-objects.test.js give the
|
|
12
|
+
// applications a type which the runtime does not report. The application of the fixture stands for a Lavoratore one:
|
|
13
|
+
// it answers the commands, and reports the placement it started with.
|
|
14
|
+
const applicationName = 'test-application'
|
|
15
|
+
|
|
16
|
+
async function startWith (t, controlPlaneResponse, accepted = true) {
|
|
17
|
+
const applicationId = randomUUID()
|
|
18
|
+
const icc = await startICC(t, { applicationId, applicationName, controlPlaneResponse })
|
|
19
|
+
|
|
20
|
+
setUpEnvironment({
|
|
21
|
+
PLT_APP_NAME: applicationName,
|
|
22
|
+
PLT_APP_DIR: join(import.meta.dirname, 'fixtures', 'runtime-lavoratore'),
|
|
23
|
+
PLT_ICC_URL: icc.iccUrl,
|
|
24
|
+
PLT_APP_PORT: 3047,
|
|
25
|
+
PLT_METRICS_PORT: 9097
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
// start() retries the registration until ICC accepts it: a refused pod stops at the runtime, as a pod does while
|
|
29
|
+
// it retries
|
|
30
|
+
let app
|
|
31
|
+
|
|
32
|
+
if (accepted) {
|
|
33
|
+
app = await start()
|
|
34
|
+
} else {
|
|
35
|
+
app = await buildApp(pino({ level: 'silent' }))
|
|
36
|
+
await app.startRuntime()
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
t.after(async () => {
|
|
40
|
+
await app.close()
|
|
41
|
+
await icc.close()
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
return { app, applicationId }
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async function placementOf (app) {
|
|
48
|
+
const { body } = await app.watt.runtime.inject('objects', { method: 'GET', url: '/' })
|
|
49
|
+
return JSON.parse(body).placement
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function refusal (reply, extra = {}) {
|
|
53
|
+
reply.code(409)
|
|
54
|
+
return { statusCode: 409, code: 'PLT_CONTROL_PLANE_VERSION_DEPLOYMENT_MISMATCH', error: 'Conflict', message: 'Version "v1" is already bound to another controller or image', ...extra }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
test('the Durable Objects commands reach a Lavoratore application of a real runtime', async (t) => {
|
|
58
|
+
const { app } = await startWith(t, {
|
|
59
|
+
applicationId: randomUUID(),
|
|
60
|
+
applicationName,
|
|
61
|
+
deploymentVersion: 'v1',
|
|
62
|
+
iccServices: {},
|
|
63
|
+
config: {},
|
|
64
|
+
durableObjects: { staged: false }
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
// The premise: the runtime configuration does not say that the application is a Lavoratore one
|
|
68
|
+
const applications = app.watt.runtime.getRuntimeConfig(true).applications
|
|
69
|
+
assert.deepStrictEqual(applications.map(application => application.id).sort(), ['objects', 'plain'])
|
|
70
|
+
assert.ok(applications.every(application => !LAVORATORE_TYPES.includes(application.type)))
|
|
71
|
+
|
|
72
|
+
// ICC said that the version is not staged
|
|
73
|
+
assert.deepStrictEqual(await placementOf(app), { type: 'coordinator', promoted: true })
|
|
74
|
+
|
|
75
|
+
assert.deepStrictEqual(await app.sendDurableObjectsCommand('durable-objects-promote', { generation: 4 }), {
|
|
76
|
+
promoted: true,
|
|
77
|
+
applications: { objects: { promoted: true, status: 'applied', generation: 4, admission: 1 } }
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
assert.deepStrictEqual(await app.sendDurableObjectsCommand('durable-objects-drain', { wait: false, generation: 5 }), {
|
|
81
|
+
status: 'drained',
|
|
82
|
+
applications: { objects: { status: 'drained', moved: 2 } }
|
|
83
|
+
})
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
test('a pod refused by an ICC which orchestrates Durable Objects starts its Lavoratore applications staged', async (t) => {
|
|
87
|
+
const { app } = await startWith(t, (_req, reply) => refusal(reply, { durableObjects: { staged: true } }), false)
|
|
88
|
+
|
|
89
|
+
assert.deepStrictEqual(app.registrationRefusal, { durableObjects: { staged: true } })
|
|
90
|
+
assert.deepStrictEqual(await placementOf(app), { type: 'coordinator', promoted: false })
|
|
91
|
+
|
|
92
|
+
// The pod still answers the promotion ICC sends once it accepts it
|
|
93
|
+
assert.strictEqual((await app.sendDurableObjectsCommand('durable-objects-promote', { generation: 2 })).promoted, true)
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
test('a pod refused by an ICC which knows nothing of Durable Objects starts as before', async (t) => {
|
|
97
|
+
const { app } = await startWith(t, (_req, reply) => refusal(reply), false)
|
|
98
|
+
|
|
99
|
+
assert.strictEqual(app.registrationRefusal, null)
|
|
100
|
+
assert.deepStrictEqual(await placementOf(app), { type: 'coordinator' })
|
|
101
|
+
})
|
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
import { test } from 'node:test'
|
|
2
|
+
import assert from 'node:assert'
|
|
3
|
+
import { once } from 'node:events'
|
|
4
|
+
import { WebSocketServer } from 'ws'
|
|
5
|
+
import { setUpEnvironment } from './helper.js'
|
|
6
|
+
import { aggregateDrain, aggregatePromotion, durableObjectsPatch, isLavoratoreApplication, placesDurableObjects, refusedDurableObjects } from '../lib/durable-objects.js'
|
|
7
|
+
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
8
|
+
import { tmpdir } from 'node:os'
|
|
9
|
+
import { join } from 'node:path'
|
|
10
|
+
import durableObjectsPlugin from '../plugins/durable-objects.js'
|
|
11
|
+
import updatePlugin from '../plugins/update.js'
|
|
12
|
+
|
|
13
|
+
const logger = { info () {}, error () {}, warn () {}, debug () {} }
|
|
14
|
+
|
|
15
|
+
function createMockRuntime (applications, results = {}) {
|
|
16
|
+
const commands = []
|
|
17
|
+
|
|
18
|
+
return {
|
|
19
|
+
commands,
|
|
20
|
+
getRuntimeConfig () {
|
|
21
|
+
return { applications }
|
|
22
|
+
},
|
|
23
|
+
async sendCommandToApplication (id, name, params) {
|
|
24
|
+
commands.push({ id, name, params })
|
|
25
|
+
const result = results[id]
|
|
26
|
+
return typeof result === 'function' ? result() : result ?? {}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async function createApp (runtime) {
|
|
32
|
+
const app = { log: logger, watt: { runtime } }
|
|
33
|
+
await durableObjectsPlugin(app)
|
|
34
|
+
return app
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
test('the patch of a Lavoratore application follows what ICC says about Durable Objects', () => {
|
|
38
|
+
// An ICC which knows nothing of Durable Objects changes nothing
|
|
39
|
+
assert.strictEqual(durableObjectsPatch(null), null)
|
|
40
|
+
assert.strictEqual(durableObjectsPatch({ applicationId: 'app' }), null)
|
|
41
|
+
assert.strictEqual(durableObjectsPatch({ durableObjects: 'yes' }), null)
|
|
42
|
+
|
|
43
|
+
// A staged version receives no objects until ICC promotes it
|
|
44
|
+
assert.deepStrictEqual(durableObjectsPatch({ durableObjects: { staged: true } }), [
|
|
45
|
+
{ op: 'merge', path: '/lavoratore/durableObjects/placement', value: { promoted: false } }
|
|
46
|
+
])
|
|
47
|
+
assert.deepStrictEqual(durableObjectsPatch({ durableObjects: { staged: false } }), [
|
|
48
|
+
{ op: 'merge', path: '/lavoratore/durableObjects/placement', value: { promoted: true } }
|
|
49
|
+
])
|
|
50
|
+
assert.deepStrictEqual(durableObjectsPatch({ durableObjects: {} })[0].value, { promoted: true })
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
test('Lavoratore applications are recognized by their type or by their configuration file', async (t) => {
|
|
54
|
+
const dir = await mkdtemp(join(tmpdir(), 'watt-extra-lavoratore-'))
|
|
55
|
+
t.after(() => rm(dir, { recursive: true, force: true }))
|
|
56
|
+
|
|
57
|
+
async function application (name, file, content) {
|
|
58
|
+
const path = join(dir, name)
|
|
59
|
+
await mkdir(path, { recursive: true })
|
|
60
|
+
await writeFile(join(path, file), typeof content === 'string' ? content : JSON.stringify(content))
|
|
61
|
+
return path
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
assert.strictEqual(await isLavoratoreApplication({ type: '@platformatic/lavoratore' }), true)
|
|
65
|
+
assert.strictEqual(await isLavoratoreApplication({ type: 'lavoratore' }), true)
|
|
66
|
+
assert.strictEqual(await isLavoratoreApplication({ type: '@platformatic/service' }), false)
|
|
67
|
+
assert.strictEqual(await isLavoratoreApplication(undefined), false)
|
|
68
|
+
|
|
69
|
+
// The runtime configuration says "unknown" for an application whose capability is resolved when it starts
|
|
70
|
+
const byModule = await application('module', 'watt.json', { module: '@platformatic/lavoratore' })
|
|
71
|
+
const bySchema = await application('schema', 'platformatic.json', { $schema: 'https://schemas.platformatic.dev/@platformatic/lavoratore/0.6.0.json' })
|
|
72
|
+
const sibling = await application('sibling', 'watt.json', { $schema: 'https://schemas.platformatic.dev/@platformatic/lavoratore-caposquadra/0.6.0.json', module: '@platformatic/lavoratore-caposquadra' })
|
|
73
|
+
const broken = await application('broken', 'watt.json', '{ not json')
|
|
74
|
+
|
|
75
|
+
assert.strictEqual(await isLavoratoreApplication({ type: 'unknown', path: byModule }), true)
|
|
76
|
+
assert.strictEqual(await isLavoratoreApplication({ type: 'unknown', path: bySchema }), true)
|
|
77
|
+
assert.strictEqual(await isLavoratoreApplication({ type: 'unknown', config: join(byModule, 'watt.json') }), true)
|
|
78
|
+
assert.strictEqual(await isLavoratoreApplication({ type: 'unknown', path: sibling }), false)
|
|
79
|
+
assert.strictEqual(await isLavoratoreApplication({ type: 'unknown', path: broken }), false)
|
|
80
|
+
assert.strictEqual(await isLavoratoreApplication({ type: 'unknown', path: join(dir, 'missing') }), false)
|
|
81
|
+
|
|
82
|
+
// The placement is read from the same file
|
|
83
|
+
const placed = await application('placed', 'watt.json', { module: '@platformatic/lavoratore', lavoratore: { durableObjects: { placement: { type: 'coordinator' } } } })
|
|
84
|
+
assert.strictEqual(await placesDurableObjects({ path: placed }), true)
|
|
85
|
+
assert.strictEqual(await placesDurableObjects({ path: byModule }), false)
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
test('only a refusal of ICC which names Durable Objects stages the pod', () => {
|
|
89
|
+
const body = { statusCode: 409, code: 'PLT_CONTROL_PLANE_VERSION_DEPLOYMENT_MISMATCH', error: 'Conflict', message: 'refused' }
|
|
90
|
+
|
|
91
|
+
assert.deepStrictEqual(refusedDurableObjects(new Error(JSON.stringify({ ...body, durableObjects: { staged: true } }))), { durableObjects: { staged: true } })
|
|
92
|
+
assert.deepStrictEqual(durableObjectsPatch(refusedDurableObjects(new Error(JSON.stringify({ ...body, durableObjects: { staged: true } })))), [
|
|
93
|
+
{ op: 'merge', path: '/lavoratore/durableObjects/placement', value: { promoted: false } }
|
|
94
|
+
])
|
|
95
|
+
|
|
96
|
+
// An ICC which knows nothing of Durable Objects, one which fails, and one which cannot be reached
|
|
97
|
+
assert.strictEqual(refusedDurableObjects(new Error(JSON.stringify(body))), null)
|
|
98
|
+
assert.strictEqual(refusedDurableObjects(new Error(JSON.stringify({ ...body, statusCode: 503, durableObjects: { staged: true } }))), null)
|
|
99
|
+
assert.strictEqual(refusedDurableObjects(new Error(JSON.stringify({ ...body, durableObjects: 'yes' }))), null)
|
|
100
|
+
assert.strictEqual(refusedDurableObjects(new Error('connect ECONNREFUSED 127.0.0.1:3000')), null)
|
|
101
|
+
assert.strictEqual(refusedDurableObjects(new Error('null')), null)
|
|
102
|
+
assert.strictEqual(refusedDurableObjects(undefined), null)
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
test('the answers of the applications are aggregated', () => {
|
|
106
|
+
assert.deepStrictEqual(aggregatePromotion({ a: { promoted: true }, b: { promoted: true } }), { promoted: true, applications: { a: { promoted: true }, b: { promoted: true } } })
|
|
107
|
+
assert.strictEqual(aggregatePromotion({ a: { promoted: true }, b: { promoted: false, status: 'duplicate' } }).promoted, false)
|
|
108
|
+
// An application which places nothing across pods has nothing to promote
|
|
109
|
+
assert.strictEqual(aggregatePromotion({ a: { promoted: true }, b: { promoted: false, status: 'unsupported' } }).promoted, true)
|
|
110
|
+
assert.strictEqual(aggregatePromotion({ b: { promoted: false, status: 'unsupported' } }).promoted, true)
|
|
111
|
+
// Nothing to promote or drain
|
|
112
|
+
assert.strictEqual(aggregatePromotion({}).promoted, true)
|
|
113
|
+
assert.strictEqual(aggregateDrain({}).status, 'drained')
|
|
114
|
+
assert.strictEqual(aggregateDrain({ a: { status: 'unsupported' }, b: { status: 'drained' } }).status, 'drained')
|
|
115
|
+
assert.strictEqual(aggregateDrain({ a: { status: 'waiting' }, b: { status: 'drained' } }).status, 'draining')
|
|
116
|
+
assert.strictEqual(aggregateDrain({ a: { status: 'superseded' }, b: { status: 'drained' } }).status, 'partial')
|
|
117
|
+
|
|
118
|
+
assert.strictEqual(aggregateDrain({ a: { status: 'drained' }, b: { status: 'drained' } }).status, 'drained')
|
|
119
|
+
assert.strictEqual(aggregateDrain({ a: { status: 'drained' }, b: { status: 'draining' } }).status, 'draining')
|
|
120
|
+
assert.strictEqual(aggregateDrain({ a: { status: 'partial' }, b: { status: 'draining' } }).status, 'draining')
|
|
121
|
+
assert.strictEqual(aggregateDrain({ a: { status: 'drained' }, b: { status: 'blocked' } }).status, 'blocked')
|
|
122
|
+
assert.strictEqual(aggregateDrain({ a: { status: 'blocked' }, b: { status: 'partial' } }).status, 'partial')
|
|
123
|
+
})
|
|
124
|
+
|
|
125
|
+
test('the commands are forwarded to every Lavoratore application of the runtime', async () => {
|
|
126
|
+
const applications = [
|
|
127
|
+
{ id: 'front', type: '@platformatic/service' },
|
|
128
|
+
{ id: 'objects', type: '@platformatic/lavoratore' },
|
|
129
|
+
{ id: 'legacy', type: 'lavoratore' }
|
|
130
|
+
]
|
|
131
|
+
const runtime = createMockRuntime(applications, {
|
|
132
|
+
objects: { promoted: true, status: 'applied', generation: 3, admission: 2 },
|
|
133
|
+
legacy: { promoted: true, status: 'duplicate', generation: 3, admission: 1 }
|
|
134
|
+
})
|
|
135
|
+
const app = await createApp(runtime)
|
|
136
|
+
|
|
137
|
+
const promotion = await app.sendDurableObjectsCommand('durable-objects-promote', { generation: 3 })
|
|
138
|
+
assert.strictEqual(promotion.promoted, true)
|
|
139
|
+
assert.deepStrictEqual(Object.keys(promotion.applications), ['objects', 'legacy'])
|
|
140
|
+
assert.deepStrictEqual(runtime.commands, [
|
|
141
|
+
{ id: 'objects', name: 'plt:lavoratore:promote', params: { generation: 3 } },
|
|
142
|
+
{ id: 'legacy', name: 'plt:lavoratore:promote', params: { generation: 3 } }
|
|
143
|
+
])
|
|
144
|
+
|
|
145
|
+
const draining = createMockRuntime(applications, {
|
|
146
|
+
objects: { status: 'drained', moved: 2 },
|
|
147
|
+
legacy: { status: 'blocked', blocked: 1 }
|
|
148
|
+
})
|
|
149
|
+
const drain = await (await createApp(draining)).sendDurableObjectsCommand('durable-objects-drain')
|
|
150
|
+
assert.strictEqual(drain.status, 'blocked')
|
|
151
|
+
assert.deepStrictEqual(drain.applications.objects, { status: 'drained', moved: 2 })
|
|
152
|
+
assert.deepStrictEqual(draining.commands.map(command => command.name), ['plt:lavoratore:drain', 'plt:lavoratore:drain'])
|
|
153
|
+
})
|
|
154
|
+
|
|
155
|
+
test('applications without the handler are skipped, failures are reported with the other answers, and repeated commands join', async () => {
|
|
156
|
+
// The runtime answers {} when the application has no handler: a plain Worker, or an older Lavoratore
|
|
157
|
+
const mixed = createMockRuntime([{ id: 'front', type: '@platformatic/lavoratore' }, { id: 'objects', type: '@platformatic/lavoratore' }], {
|
|
158
|
+
objects: { status: 'drained', moved: 1 }
|
|
159
|
+
})
|
|
160
|
+
assert.deepStrictEqual(await (await createApp(mixed)).sendDurableObjectsCommand('durable-objects-drain', { wait: false }), {
|
|
161
|
+
status: 'drained', applications: { objects: { status: 'drained', moved: 1 } }, skipped: ['front']
|
|
162
|
+
})
|
|
163
|
+
|
|
164
|
+
const plain = await createApp(createMockRuntime([{ id: 'front', type: '@platformatic/lavoratore' }]))
|
|
165
|
+
assert.deepStrictEqual(await plain.sendDurableObjectsCommand('durable-objects-promote', { generation: 1 }), { promoted: true, applications: {}, skipped: ['front'] })
|
|
166
|
+
assert.deepStrictEqual(await plain.sendDurableObjectsCommand('durable-objects-drain'), { status: 'drained', applications: {}, skipped: ['front'] })
|
|
167
|
+
|
|
168
|
+
// ICC asks only the pods it knows to run Lavoratore: finding none is a failure, never an answer it takes for done
|
|
169
|
+
const none = await createApp(createMockRuntime([{ id: 'api', type: '@platformatic/service' }]))
|
|
170
|
+
await assert.rejects(none.sendDurableObjectsCommand('durable-objects-drain'), /No Lavoratore application found/)
|
|
171
|
+
await assert.rejects(none.sendDurableObjectsCommand('durable-objects-promote', { generation: 1 }), /No Lavoratore application found/)
|
|
172
|
+
|
|
173
|
+
// One application fails: the other is still asked, and the error carries its answer
|
|
174
|
+
const failing = createMockRuntime([{ id: 'a', type: '@platformatic/lavoratore' }, { id: 'b', type: '@platformatic/lavoratore' }], {
|
|
175
|
+
a: () => { throw new Error('boom') },
|
|
176
|
+
b: { promoted: true, status: 'applied' }
|
|
177
|
+
})
|
|
178
|
+
await assert.rejects((await createApp(failing)).sendDurableObjectsCommand('durable-objects-promote', { generation: 2 }), err => {
|
|
179
|
+
assert.match(err.message, /durable-objects-promote command failed in the applications a: boom/)
|
|
180
|
+
assert.deepStrictEqual(err.results, { b: { promoted: true, status: 'applied' } })
|
|
181
|
+
return true
|
|
182
|
+
})
|
|
183
|
+
assert.strictEqual(failing.commands.length, 2)
|
|
184
|
+
|
|
185
|
+
// The same command repeated while it runs joins it; another generation does not
|
|
186
|
+
let release
|
|
187
|
+
const slow = createMockRuntime([{ id: 'objects', type: '@platformatic/lavoratore' }], {
|
|
188
|
+
objects: () => new Promise(resolve => { release = () => resolve({ promoted: true }) })
|
|
189
|
+
})
|
|
190
|
+
const app = await createApp(slow)
|
|
191
|
+
const first = app.sendDurableObjectsCommand('durable-objects-promote', { generation: 3 })
|
|
192
|
+
const second = app.sendDurableObjectsCommand('durable-objects-promote', { generation: 3 })
|
|
193
|
+
await new Promise(resolve => setImmediate(resolve))
|
|
194
|
+
assert.strictEqual(slow.commands.length, 1)
|
|
195
|
+
release()
|
|
196
|
+
assert.deepStrictEqual(await first, await second)
|
|
197
|
+
const third = app.sendDurableObjectsCommand('durable-objects-promote', { generation: 3 })
|
|
198
|
+
await new Promise(resolve => setImmediate(resolve))
|
|
199
|
+
assert.strictEqual(slow.commands.length, 2)
|
|
200
|
+
release()
|
|
201
|
+
await third
|
|
202
|
+
|
|
203
|
+
await assert.rejects(app.sendDurableObjectsCommand('durable-objects-restart'), /Unknown Durable Objects command/)
|
|
204
|
+
|
|
205
|
+
const stopped = { log: logger, watt: { runtime: null } }
|
|
206
|
+
await durableObjectsPlugin(stopped)
|
|
207
|
+
await assert.rejects(stopped.sendDurableObjectsCommand('durable-objects-drain'), /Runtime not started/)
|
|
208
|
+
})
|
|
209
|
+
|
|
210
|
+
test('the patch applies only to the applications which place Durable Objects', async (t) => {
|
|
211
|
+
const dir = await mkdtemp(join(tmpdir(), 'watt-extra-do-'))
|
|
212
|
+
t.after(() => rm(dir, { recursive: true, force: true }))
|
|
213
|
+
const write = async (name, content) => {
|
|
214
|
+
const path = join(dir, name)
|
|
215
|
+
await writeFile(path, typeof content === 'string' ? content : JSON.stringify(content))
|
|
216
|
+
return { config: path }
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
assert.strictEqual(await placesDurableObjects(await write('placed.json', { lavoratore: { durableObjects: { placement: { type: 'coordinator' } } } })), true)
|
|
220
|
+
assert.strictEqual(await placesDurableObjects(await write('local.json', { lavoratore: { durableObjects: {} } })), false)
|
|
221
|
+
assert.strictEqual(await placesDurableObjects(await write('plain.json', { lavoratore: {} })), false)
|
|
222
|
+
// A configuration which cannot be read gets the patch
|
|
223
|
+
assert.strictEqual(await placesDurableObjects(await write('broken.json', '{ not json')), true)
|
|
224
|
+
assert.strictEqual(await placesDurableObjects({ config: join(dir, 'missing.json') }), true)
|
|
225
|
+
assert.strictEqual(await placesDurableObjects({}), true)
|
|
226
|
+
})
|
|
227
|
+
|
|
228
|
+
function setupMockIccServer (wss) {
|
|
229
|
+
let ws = null
|
|
230
|
+
|
|
231
|
+
const waitForClientSubscription = once(wss, 'connection').then(([socket]) => {
|
|
232
|
+
ws = socket
|
|
233
|
+
|
|
234
|
+
return new Promise((resolve) => {
|
|
235
|
+
socket.on('message', (data) => {
|
|
236
|
+
const message = JSON.parse(data.toString())
|
|
237
|
+
|
|
238
|
+
if (message.command === 'subscribe' && message.topic === '/config') {
|
|
239
|
+
socket.send(JSON.stringify({ command: 'ack' }))
|
|
240
|
+
resolve()
|
|
241
|
+
}
|
|
242
|
+
})
|
|
243
|
+
})
|
|
244
|
+
})
|
|
245
|
+
|
|
246
|
+
return { waitForClientSubscription, getWs: () => ws }
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
async function reply (ws, frame) {
|
|
250
|
+
ws.send(JSON.stringify(frame))
|
|
251
|
+
return JSON.parse((await once(ws, 'message'))[0].toString())
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
test('the Durable Objects commands of ICC are answered over the websocket with their result', async (t) => {
|
|
255
|
+
setUpEnvironment()
|
|
256
|
+
const port = 14300
|
|
257
|
+
const wss = new WebSocketServer({ port })
|
|
258
|
+
t.after(() => wss.close())
|
|
259
|
+
const { waitForClientSubscription, getWs } = setupMockIccServer(wss)
|
|
260
|
+
|
|
261
|
+
const received = []
|
|
262
|
+
const app = {
|
|
263
|
+
log: logger,
|
|
264
|
+
instanceConfig: { applicationId: 'test-application-id' },
|
|
265
|
+
getAuthorizationHeaders: async () => ({ Authorization: 'Bearer test-token' }),
|
|
266
|
+
getRuntimeId: () => 'test-runtime-id',
|
|
267
|
+
env: { PLT_APP_NAME: 'test-app', PLT_APP_DIR: '/path/to/app', PLT_ICC_URL: `http://localhost:${port}`, PLT_UPDATES_RECONNECT_INTERVAL_SEC: 1 },
|
|
268
|
+
async sendDurableObjectsCommand (command, params) {
|
|
269
|
+
received.push({ command, params })
|
|
270
|
+
|
|
271
|
+
if (params.generation === 7) {
|
|
272
|
+
return { promoted: false, applications: { objects: { promoted: false, status: 'superseded', generation: 7, admission: null } } }
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
if (params.generation === 8) {
|
|
276
|
+
throw new Error('The application "objects" does not handle the durable-objects-promote command')
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
return command === 'durable-objects-promote'
|
|
280
|
+
? { promoted: true, applications: { objects: { promoted: true, status: 'applied', generation: params.generation, admission: 1 } } }
|
|
281
|
+
: { status: 'drained', applications: { objects: { status: 'drained', moved: 3 } } }
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
await updatePlugin(app)
|
|
286
|
+
await app.connectToUpdates()
|
|
287
|
+
t.after(() => app.closeUpdates())
|
|
288
|
+
await waitForClientSubscription
|
|
289
|
+
|
|
290
|
+
assert.deepStrictEqual(await reply(getWs(), { command: 'durable-objects-promote', requestId: 'promote-1', params: { generation: 2 } }), {
|
|
291
|
+
requestId: 'promote-1',
|
|
292
|
+
success: true,
|
|
293
|
+
result: { promoted: true, applications: { objects: { promoted: true, status: 'applied', generation: 2, admission: 1 } } }
|
|
294
|
+
})
|
|
295
|
+
|
|
296
|
+
// A promotion which did not promote the pod says so in its result: the frame stays valid for ICC, which reads it
|
|
297
|
+
const superseded = await reply(getWs(), { command: 'durable-objects-promote', requestId: 'promote-2', params: { generation: 7 } })
|
|
298
|
+
assert.strictEqual(superseded.success, true)
|
|
299
|
+
assert.strictEqual(superseded.result.promoted, false)
|
|
300
|
+
assert.strictEqual(superseded.result.applications.objects.status, 'superseded')
|
|
301
|
+
|
|
302
|
+
const failed = await reply(getWs(), { command: 'durable-objects-promote', requestId: 'promote-3', params: { generation: 8 } })
|
|
303
|
+
assert.deepStrictEqual(failed, { requestId: 'promote-3', success: false, error: { message: 'The application "objects" does not handle the durable-objects-promote command', name: 'Error' } })
|
|
304
|
+
|
|
305
|
+
assert.deepStrictEqual(await reply(getWs(), { command: 'durable-objects-drain', requestId: 'drain-1' }), {
|
|
306
|
+
requestId: 'drain-1',
|
|
307
|
+
success: true,
|
|
308
|
+
result: { status: 'drained', applications: { objects: { status: 'drained', moved: 3 } } }
|
|
309
|
+
})
|
|
310
|
+
|
|
311
|
+
assert.deepStrictEqual(received, [
|
|
312
|
+
{ command: 'durable-objects-promote', params: { generation: 2 } },
|
|
313
|
+
{ command: 'durable-objects-promote', params: { generation: 7 } },
|
|
314
|
+
{ command: 'durable-objects-promote', params: { generation: 8 } },
|
|
315
|
+
{ command: 'durable-objects-drain', params: {} }
|
|
316
|
+
])
|
|
317
|
+
})
|
|
318
|
+
|
|
319
|
+
test('the objects of the pod are listed over its Lavoratore applications', async () => {
|
|
320
|
+
const counter = { className: 'Counter', id: 'a1', name: 'a', state: 'active', storageBytes: 4096 }
|
|
321
|
+
const room = { className: 'Room', id: 'b2', name: null, state: 'hibernated', storageBytes: null }
|
|
322
|
+
const runtime = createMockRuntime([
|
|
323
|
+
{ id: 'objects', type: '@platformatic/lavoratore' },
|
|
324
|
+
{ id: 'rooms', type: '@platformatic/lavoratore' },
|
|
325
|
+
{ id: 'front', type: '@platformatic/lavoratore' },
|
|
326
|
+
{ id: 'legacy', type: '@platformatic/lavoratore' }
|
|
327
|
+
], {
|
|
328
|
+
objects: { status: 'ok', memberId: 'pod-1', total: 5, truncated: true, objects: [counter] },
|
|
329
|
+
rooms: { status: 'ok', memberId: 'pod-1', total: 1, truncated: false, objects: [room] },
|
|
330
|
+
// Without Durable Objects
|
|
331
|
+
front: { status: 'unsupported' }
|
|
332
|
+
// legacy has no handler: a Lavoratore which predates the list
|
|
333
|
+
})
|
|
334
|
+
const app = await createApp(runtime)
|
|
335
|
+
|
|
336
|
+
assert.deepStrictEqual(await app.sendDurableObjectsCommand('durable-objects-list', { limit: 1 }), {
|
|
337
|
+
total: 6,
|
|
338
|
+
truncated: true,
|
|
339
|
+
applications: ['objects', 'rooms'],
|
|
340
|
+
objects: [{ application: 'objects', ...counter }, { application: 'rooms', ...room }],
|
|
341
|
+
skipped: ['legacy']
|
|
342
|
+
})
|
|
343
|
+
assert.deepStrictEqual(runtime.commands[0], { id: 'objects', name: 'plt:lavoratore:objects', params: { limit: 1 } })
|
|
344
|
+
|
|
345
|
+
// An application which lists no object listed: the pod holds none, which is not a pod which cannot list
|
|
346
|
+
const empty = await createApp(createMockRuntime([{ id: 'objects', type: '@platformatic/lavoratore' }, { id: 'legacy', type: '@platformatic/lavoratore' }], {
|
|
347
|
+
objects: { status: 'ok', memberId: 'pod-1', total: 0, truncated: false, objects: [] }
|
|
348
|
+
}))
|
|
349
|
+
assert.deepStrictEqual(await empty.sendDurableObjectsCommand('durable-objects-list'), { total: 0, truncated: false, applications: ['objects'], objects: [], skipped: ['legacy'] })
|
|
350
|
+
|
|
351
|
+
const legacy = await createApp(createMockRuntime([{ id: 'legacy', type: '@platformatic/lavoratore' }]))
|
|
352
|
+
assert.deepStrictEqual(await legacy.sendDurableObjectsCommand('durable-objects-list'), { total: 0, truncated: false, applications: [], objects: [], skipped: ['legacy'] })
|
|
353
|
+
})
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// A stand-in for the Lavoratore capability: the Node.js one, accepting the "lavoratore" key of the configuration, which
|
|
2
|
+
// watt-extra patches
|
|
3
|
+
import * as node from '@platformatic/node'
|
|
4
|
+
|
|
5
|
+
export const { Generator, loadConfiguration, packageJson, schemaComponents, transform, version } = node
|
|
6
|
+
|
|
7
|
+
export const schema = {
|
|
8
|
+
...node.schema,
|
|
9
|
+
properties: { ...node.schema.properties, lavoratore: { type: 'object', additionalProperties: true } }
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export async function create (configOrRoot, sourceOrConfig, context) {
|
|
13
|
+
const capability = await node.create(configOrRoot, sourceOrConfig, { ...context, schema })
|
|
14
|
+
|
|
15
|
+
// For the application, which reports the placement it started with
|
|
16
|
+
globalThis.lavoratoreStandIn = { placement: () => capability.config?.lavoratore?.durableObjects?.placement }
|
|
17
|
+
|
|
18
|
+
return capability
|
|
19
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// Stands for a Lavoratore application: the runtime reports no type for it, as for the real one, and it answers the
|
|
2
|
+
// commands of ICC through the ITC of the runtime
|
|
3
|
+
import { createServer } from 'node:http'
|
|
4
|
+
|
|
5
|
+
const commands = []
|
|
6
|
+
|
|
7
|
+
globalThis.platformatic.itc.handle('plt:lavoratore:promote', message => {
|
|
8
|
+
commands.push({ name: 'promote', message })
|
|
9
|
+
return { promoted: true, status: 'applied', generation: message.generation, admission: 1 }
|
|
10
|
+
})
|
|
11
|
+
|
|
12
|
+
globalThis.platformatic.itc.handle('plt:lavoratore:drain', message => {
|
|
13
|
+
commands.push({ name: 'drain', message })
|
|
14
|
+
return { status: 'drained', moved: commands.length }
|
|
15
|
+
})
|
|
16
|
+
|
|
17
|
+
export function create () {
|
|
18
|
+
return createServer((_req, res) => {
|
|
19
|
+
res.setHeader('content-type', 'application/json')
|
|
20
|
+
// With the configuration the application started with, i.e. patched by watt-extra
|
|
21
|
+
res.end(JSON.stringify({ commands, placement: globalThis.lavoratoreStandIn.placement() }))
|
|
22
|
+
})
|
|
23
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://schemas.platformatic.dev/@platformatic/lavoratore/0.6.0.json",
|
|
3
|
+
"module": "./capability.js",
|
|
4
|
+
"node": {
|
|
5
|
+
"main": "index.js"
|
|
6
|
+
},
|
|
7
|
+
"lavoratore": {
|
|
8
|
+
"durableObjects": {
|
|
9
|
+
"placement": {
|
|
10
|
+
"type": "coordinator"
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://schemas.platformatic.dev/@platformatic/runtime/3.0.0.json",
|
|
3
|
+
"entrypoint": "plain",
|
|
4
|
+
"applications": [
|
|
5
|
+
{ "id": "objects", "path": "./objects" },
|
|
6
|
+
{ "id": "plain", "path": "./plain" }
|
|
7
|
+
],
|
|
8
|
+
"logger": {
|
|
9
|
+
"level": "error"
|
|
10
|
+
}
|
|
11
|
+
}
|
package/test/helper.js
CHANGED
|
@@ -89,9 +89,9 @@ async function startICC (t, opts = {}) {
|
|
|
89
89
|
|
|
90
90
|
// Control Plane
|
|
91
91
|
await icc.register(async (icc) => {
|
|
92
|
-
icc.post('/pods/:podId/instance', async (req) => {
|
|
92
|
+
icc.post('/pods/:podId/instance', async (req, reply) => {
|
|
93
93
|
if (typeof controlPlaneResponse === 'function') {
|
|
94
|
-
return controlPlaneResponse(req)
|
|
94
|
+
return controlPlaneResponse(req, reply)
|
|
95
95
|
}
|
|
96
96
|
return controlPlaneResponse || {
|
|
97
97
|
applicationId,
|