@bongos/core 1.19.398 → 1.19.400
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +47 -27
- package/clients/bongos-client/README.md +1 -1
- package/clients/bongos-client/bongos-client.global.js +2 -0
- package/clients/bongos-client/index.cjs +2 -0
- package/clients/bongos-client/index.d.ts +3 -0
- package/clients/bongos-client/index.mjs +2 -0
- package/docs/adr/0199-restart-an-unreachable-active-project.md +49 -0
- package/docs/adr/README.md +1 -0
- package/docs/api/openapi.json +74 -3
- package/docs/api-reference.md +3 -2
- package/modules/dev-box/app/src/vendor/bongos-client.cjs +2 -0
- package/modules/provisioning/migrations/provisioning_011_restart.sql +28 -0
- package/modules/provisioning/provisioning.js +42 -4
- package/modules/provisioning/routes/provisioning.js +59 -0
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/public-landing/projects.html +124 -6
- package/scripts/gds/provision.js +104 -0
- package/src/module-api.js +5 -1
- package/tests/projects_hub_pre_uat.mjs +77 -0
- package/tests/provision_restart.mjs +242 -0
- package/tests/provisioning_restart.mjs +285 -0
- package/tests/provisioning_settings_apply.mjs +8 -3
- package/tests/provisioning_teardown_intent.mjs +9 -5
|
@@ -46,7 +46,11 @@ const PROJECT_TYPES = ['game', 'research', 'business', 'non-profit', 'not-sure']
|
|
|
46
46
|
// per-project setting to an already-ACTIVE instance — patch its web.env settings
|
|
47
47
|
// lines in place + restart, since the branding pack is memoized for the process
|
|
48
48
|
// lifetime (task 1003139 built only the standup leg).
|
|
49
|
-
|
|
49
|
+
// 'restart' (task 1003238, migration provisioning_011): restart an already-ACTIVE
|
|
50
|
+
// instance whose liveness probe says it is down — the settings-apply leg minus the
|
|
51
|
+
// env patch. Its own action because no existing one fits: provision refuses
|
|
52
|
+
// 'active' and settings-apply requires a value that actually changed.
|
|
53
|
+
const INTENT_ACTIONS = ['provision', 'teardown', 'oauth-manifest', 'domain-attach', 'settings-apply', 'restart'];
|
|
50
54
|
// Every state a teardown can be asked for — i.e. everything except the one
|
|
51
55
|
// state where there is provably nothing left to release. DERIVED from
|
|
52
56
|
// INSTANCE_STATES on purpose (task 1003042, walk finding N5): the old hand-kept
|
|
@@ -84,6 +88,19 @@ const SETTINGS_APPLY_FROM = new Set(['active']);
|
|
|
84
88
|
// to it (the placeManifestCreds refusal) — so the route refuses the push HONESTLY
|
|
85
89
|
// rather than queue one that never lands.
|
|
86
90
|
const SETTINGS_APPLY_SHAPES = new Set(['co-tenant', 'standalone']);
|
|
91
|
+
// The ONE state a restart is meaningful from (task 1003238). Liveness is an
|
|
92
|
+
// OVERLAY, not a seventh state — an instance the heartbeat cannot reach is still
|
|
93
|
+
// 'active', and 'active' is exactly where a restart applies. Every earlier state
|
|
94
|
+
// has nothing running to restart and is already served by the provision path
|
|
95
|
+
// (PROVISION_ENQUEUE_FROM covers 'requested'/'provisioning'/'error'/'torn_down');
|
|
96
|
+
// 'tearing_down' is mid-reclaim. Kept as its own Set rather than reusing
|
|
97
|
+
// SETTINGS_APPLY_FROM: each action declaring its own gate is this module's
|
|
98
|
+
// convention, and the two answer different questions.
|
|
99
|
+
const RESTART_FROM = new Set(['active']);
|
|
100
|
+
// Same reachability rule as the settings push, for the same reason: a 'dedicated'
|
|
101
|
+
// droplet is stood up by cloud-init and the runner has no post-hoc exec path to
|
|
102
|
+
// it, so the route refuses HONESTLY rather than queue a restart that never lands.
|
|
103
|
+
const RESTART_SHAPES = new Set(['co-tenant', 'standalone']);
|
|
87
104
|
|
|
88
105
|
// Per-project settings (task 1003040, goal 1000058): the known keys + their
|
|
89
106
|
// vocabularies, enforced at the API layer — the jsonb `settings` column
|
|
@@ -632,7 +649,10 @@ async function enqueueIntent(db, instanceId, action, requestedBy) {
|
|
|
632
649
|
// (the state CHECK has no 'superseded'; last_error says why) so the runner never
|
|
633
650
|
// stands it up. A PENDING settings push is superseded the same way (task 1003140):
|
|
634
651
|
// a push into a project about to be torn down is dead work, and letting it hold the
|
|
635
|
-
// delete hostage would repeat 1002691. A
|
|
652
|
+
// delete hostage would repeat 1002691. A pending 'restart' joins them for the same
|
|
653
|
+
// reason (task 1003238) — and more sharply, since the owner reaches for Restart
|
|
654
|
+
// precisely when a project looks broken, which is also when they give up and delete
|
|
655
|
+
// it. A RUNNING intent is mid-flight and never yanked —
|
|
636
656
|
// the caller gets { conflict: 'open_intent' } with the blocking intent so the route
|
|
637
657
|
// can answer truthfully. Each statement is atomic and the index stays the arbiter:
|
|
638
658
|
// any interleaving with the runner or a concurrent enqueue lands on one of these
|
|
@@ -642,7 +662,7 @@ async function enqueueTeardown(db, instanceId, requestedBy) {
|
|
|
642
662
|
`UPDATE provisioning_intents
|
|
643
663
|
SET state = 'error', last_error = 'superseded: teardown requested',
|
|
644
664
|
updated_at = now(), resolved_at = now()
|
|
645
|
-
WHERE instance_id = $1 AND action IN ('provision', 'settings-apply') AND state = 'pending'
|
|
665
|
+
WHERE instance_id = $1 AND action IN ('provision', 'settings-apply', 'restart') AND state = 'pending'
|
|
646
666
|
RETURNING id`,
|
|
647
667
|
[instanceId]
|
|
648
668
|
);
|
|
@@ -668,6 +688,23 @@ async function enqueueSettingsApply(db, instanceId, requestedBy) {
|
|
|
668
688
|
return { intent, created: false, conflict: 'open_intent' };
|
|
669
689
|
}
|
|
670
690
|
|
|
691
|
+
// Queue a restart for an ACTIVE instance (task 1003238). Idempotent against its
|
|
692
|
+
// own kind in EITHER open state, which is where it differs from the settings push:
|
|
693
|
+
// that one must refuse to ride a RUNNING sibling because the runner may have read
|
|
694
|
+
// the settings row before this write, so the in-flight run would not carry the
|
|
695
|
+
// change. A restart carries no payload — a restart already running IS the thing
|
|
696
|
+
// the owner asked for, so reporting "already under way" is the truth, not a
|
|
697
|
+
// swallowed request. Anything else holding the single open slot is reported as
|
|
698
|
+
// { conflict: 'open_intent' } so the route can name it. The state gate and the
|
|
699
|
+
// shape gate are the ROUTE's job (RESTART_FROM / RESTART_SHAPES); this only
|
|
700
|
+
// arbitrates the open slot, like enqueueTeardown.
|
|
701
|
+
async function enqueueRestart(db, instanceId, requestedBy) {
|
|
702
|
+
const { intent, created } = await enqueueIntent(db, instanceId, 'restart', requestedBy);
|
|
703
|
+
if (created) return { intent, created: true };
|
|
704
|
+
if (intent.action === 'restart') return { intent, created: false };
|
|
705
|
+
return { intent, created: false, conflict: 'open_intent' };
|
|
706
|
+
}
|
|
707
|
+
|
|
671
708
|
// The instance's currently-open (pending|running) intent, if any.
|
|
672
709
|
async function getOpenIntent(db, instanceId) {
|
|
673
710
|
const { rows } = await db.query(
|
|
@@ -860,6 +897,7 @@ module.exports = {
|
|
|
860
897
|
INSTANCE_STATES, HOSTING_SHAPES, ONBOARD_MODES, PROJECT_TYPES, INTENT_ACTIONS, TEARDOWN_FROM,
|
|
861
898
|
settingsEnvLines, settingsEnvVars,
|
|
862
899
|
PROVISION_ENQUEUE_FROM, DOMAIN_ATTACH_FROM, SETTINGS_APPLY_FROM, SETTINGS_APPLY_SHAPES,
|
|
900
|
+
RESTART_FROM, RESTART_SHAPES,
|
|
863
901
|
TIER_MONTHLY_USD, CERT_ELIGIBLE_STATES,
|
|
864
902
|
SETTINGS_VOCAB, DEFAULT_SETTINGS,
|
|
865
903
|
// pure
|
|
@@ -873,7 +911,7 @@ module.exports = {
|
|
|
873
911
|
// data access — verification overlay (task 1002696)
|
|
874
912
|
listActiveInstancesForLiveness, recordLivenessProbe, recordDnsNote, clearErrorNote,
|
|
875
913
|
createInstance, setInstanceStatus, updateInstanceDomain, updateInstanceSettings,
|
|
876
|
-
enqueueIntent, enqueueTeardown, enqueueSettingsApply, getOpenIntent, claimNextIntent, resolveIntent,
|
|
914
|
+
enqueueIntent, enqueueTeardown, enqueueSettingsApply, enqueueRestart, getOpenIntent, claimNextIntent, resolveIntent,
|
|
877
915
|
recordEvent, instanceStateForSlug, costLedger,
|
|
878
916
|
// data access — GitHub App Manifest attempts (task 2080)
|
|
879
917
|
createOAuthManifest, getOAuthManifestByState, attachManifestCode,
|
|
@@ -795,6 +795,65 @@ module.exports = function provisioningRoutes() {
|
|
|
795
795
|
}
|
|
796
796
|
});
|
|
797
797
|
|
|
798
|
+
// POST /provisioning/instances/:id/restart — restart an ACTIVE project the
|
|
799
|
+
// heartbeat cannot reach (task 1003238). rank: any authenticated builder (own
|
|
800
|
+
// resource; archon may restart any — the override is audited via the actor).
|
|
801
|
+
//
|
|
802
|
+
// WHY it is its own route. The liveness overlay (task 1002696) can say a project
|
|
803
|
+
// is unreachable while its lifecycle status is still 'active' — and until now
|
|
804
|
+
// that reading came with no action the owner could take. Re-provision is not it:
|
|
805
|
+
// PROVISION_ENQUEUE_FROM deliberately excludes 'active' (task 1002691), so a
|
|
806
|
+
// Restart wired to that path would have refused on every press — the finding this
|
|
807
|
+
// route closes. The web tier only ENQUEUES; the control-plane runner performs the
|
|
808
|
+
// restart and the patient healthz verify (ADR 0111 §2).
|
|
809
|
+
//
|
|
810
|
+
// Every refusal names the action that IS available in that state rather than
|
|
811
|
+
// hiding behind a code, and no branch promises a duration — a restart's downtime
|
|
812
|
+
// is exactly what this route cannot know (the F16 rule, ADR 0190 §7).
|
|
813
|
+
router.post('/provisioning/instances/:id/restart', auth.requireBuilder, async (req, res) => {
|
|
814
|
+
const id = parseId(req, res); // (req, res) — parseId sends 400 bad_id + returns null on failure
|
|
815
|
+
if (!id) return;
|
|
816
|
+
try {
|
|
817
|
+
const inst = await provisioning.getInstanceById(pool, id);
|
|
818
|
+
if (instanceAccessDenied(inst, req.builder)) return res.fail('instance_not_found', 404);
|
|
819
|
+
// The archon override carries WHICH archon (the force-teardown convention).
|
|
820
|
+
const actor = String(inst.owner_builder_id) === String(req.builder.id) ? 'api:self' : `api:archon:${req.builder.id}`;
|
|
821
|
+
if (!provisioning.RESTART_FROM.has(inst.status)) {
|
|
822
|
+
const offline = inst.status === 'torn_down' || inst.status === 'tearing_down';
|
|
823
|
+
return res.fail('not_restartable', { status: 409, details: { status: inst.status }, message: offline
|
|
824
|
+
? 'This project is offline, so there is nothing running to restart — stand it up again instead.'
|
|
825
|
+
: inst.status === 'error'
|
|
826
|
+
? 'This project’s standup stopped partway, so there is nothing running to restart — stand it up again to run the standup afresh.'
|
|
827
|
+
: 'This project hasn’t finished standing up yet, so there is nothing to restart.' });
|
|
828
|
+
}
|
|
829
|
+
// A dedicated droplet is stood up by cloud-init and the runner has no post-hoc
|
|
830
|
+
// exec path to it — refuse honestly rather than queue a restart that never
|
|
831
|
+
// lands (the F16 lie), and name the command whoever runs that machine needs.
|
|
832
|
+
if (!provisioning.RESTART_SHAPES.has(inst.hosting_shape)) {
|
|
833
|
+
return res.fail('restart_unreachable', { status: 409, details: { hosting_shape: inst.hosting_shape },
|
|
834
|
+
message: `This project runs on a machine the platform doesn’t manage, so it can’t be restarted from here — whoever runs that machine has to run 'systemctl restart ${inst.slug}' there.` });
|
|
835
|
+
}
|
|
836
|
+
const r = await provisioning.enqueueRestart(pool, id, actor);
|
|
837
|
+
if (r.conflict) {
|
|
838
|
+
return res.fail('intent_conflict', { status: 409,
|
|
839
|
+
message: `A '${r.intent.action}' run is currently ${r.intent.state} for this project — the restart could not be queued; try again once that settles.`,
|
|
840
|
+
details: { action: r.intent.action, state: r.intent.state } });
|
|
841
|
+
}
|
|
842
|
+
if (r.created) {
|
|
843
|
+
await provisioning.recordEvent(pool, {
|
|
844
|
+
instanceId: id, ownerBuilderId: inst.owner_builder_id, event: 'restart',
|
|
845
|
+
detail: 'restart requested by owner', actor,
|
|
846
|
+
}).catch(() => {});
|
|
847
|
+
}
|
|
848
|
+
res.json({ ok: true, queued: r.created, action: 'restart', status: inst.status, message: r.created
|
|
849
|
+
? 'Restart queued. Your project will stop and start again — this page says whether it answered afterwards.'
|
|
850
|
+
: 'A restart is already under way for this project.' });
|
|
851
|
+
} catch (err) {
|
|
852
|
+
log.error('[provisioning] POST /provisioning/instances/:id/restart', err);
|
|
853
|
+
res.fail('restart_failed', { status: 500, message: 'internal error' });
|
|
854
|
+
}
|
|
855
|
+
});
|
|
856
|
+
|
|
798
857
|
// POST /provisioning/instances/:id/domain — give the caller's OWN instance its
|
|
799
858
|
// address (task 1002692, gap 5: no path existed to add a domain after creation).
|
|
800
859
|
// Validates + records the domain; for an already-ACTIVE instance it also enqueues
|
package/package-lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.19.
|
|
3
|
+
"version": "1.19.400",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@bongos/core",
|
|
9
|
-
"version": "1.19.
|
|
9
|
+
"version": "1.19.400",
|
|
10
10
|
"license": "AGPL-3.0-or-later",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"express": "^4.21.2",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.19.
|
|
3
|
+
"version": "1.19.400",
|
|
4
4
|
"description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
|
|
5
5
|
"license": "AGPL-3.0-or-later",
|
|
6
6
|
"main": "src/platform-server.js",
|
|
@@ -1086,6 +1086,10 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
1086
1086
|
(direction §3 — an active card must be able to
|
|
1087
1087
|
self-correct, so the list re-reads once a minute) */
|
|
1088
1088
|
var FRESH_MS = 12 * 60 * 1000; /* "checked Xm ago" freshness ≈ 2 heartbeat sweeps */
|
|
1089
|
+
/* how long the card keeps showing the moment a standup LANDED (task 1003243).
|
|
1090
|
+
Shorter than FRESH_MS on purpose: "live" is a standing fact the chip carries
|
|
1091
|
+
forever, while "it just came up" is an event that stops being news. */
|
|
1092
|
+
var LIVE_MOMENT_MS = 5 * 60 * 1000;
|
|
1089
1093
|
|
|
1090
1094
|
var b = (typeof window !== 'undefined' && window.__BRANDING__) || {};
|
|
1091
1095
|
var productName = (b.identity && b.identity.productName) || 'Cloud Bongos';
|
|
@@ -3950,6 +3954,20 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
3950
3954
|
so the page can never say "Answering at …" beside a chip that says "live?". */
|
|
3951
3955
|
function verifiedAt(inst) { return inst.last_verified_at ? new Date(inst.last_verified_at).getTime() : 0; }
|
|
3952
3956
|
function freshUp(inst) { var at = verifiedAt(inst); return !!(inst.last_verified_up && at && (Date.now() - at) <= FRESH_MS); }
|
|
3957
|
+
/* the landing moment (task 1003243): a standup or revive that VERIFIABLY came
|
|
3958
|
+
up, still recent enough to be news. Two gates, both already on the card —
|
|
3959
|
+
freshUp() so "live" stays ONE rule (a lit third stage beside a "live?" chip
|
|
3960
|
+
would be the exact contradiction direction §3 forbids), and provisioned_at,
|
|
3961
|
+
which the runner re-stamps every time it flips an instance to active, so a
|
|
3962
|
+
revive earns the moment too. No sticky per-card flag is invented: the whole
|
|
3963
|
+
window is derived from data the card already holds, which is why a reload
|
|
3964
|
+
mid-window still shows it and a long-live project never does. */
|
|
3965
|
+
function provisionedAt(inst) { return inst.provisioned_at ? new Date(inst.provisioned_at).getTime() : 0; }
|
|
3966
|
+
function justLive(inst) {
|
|
3967
|
+
var at = provisionedAt(inst);
|
|
3968
|
+
return inst.status === 'active' && !inst.open_intent && freshUp(inst) &&
|
|
3969
|
+
!!at && (Date.now() - at) <= LIVE_MOMENT_MS;
|
|
3970
|
+
}
|
|
3953
3971
|
|
|
3954
3972
|
function chipMarkup(cls, label, rec) {
|
|
3955
3973
|
return '<span class="chip ' + cls + '"><span class="cdot"></span>' + esc(label) +
|
|
@@ -4005,6 +4023,17 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
4005
4023
|
both re-POST the slug (the server's PROVISION_ENQUEUE_FROM gate) */
|
|
4006
4024
|
function canRetry(inst) { return inst.status === 'error' && !inst.open_intent; }
|
|
4007
4025
|
function canRevive(inst) { return inst.status === 'torn_down' && !inst.open_intent; }
|
|
4026
|
+
/* the restart door (task 1003238): client mirror of RESTART_FROM {active} +
|
|
4027
|
+
RESTART_SHAPES (a dedicated droplet has no runner exec path), shown ONLY
|
|
4028
|
+
where it has a reason to exist — a project the heartbeat cannot reach.
|
|
4029
|
+
last_verified_up === false is the verdict; null means never probed and is
|
|
4030
|
+
deliberately NOT the same claim (publicInstance keeps the two apart), so a
|
|
4031
|
+
brand-new project never grows a Restart button it has no use for. A door on
|
|
4032
|
+
every healthy card would only invite needless downtime. */
|
|
4033
|
+
function canRestart(inst) {
|
|
4034
|
+
return inst.status === 'active' && inst.last_verified_up === false &&
|
|
4035
|
+
inst.hosting_shape !== 'dedicated' && !inst.open_intent;
|
|
4036
|
+
}
|
|
4008
4037
|
/* keep watching while the platform is doing something to this instance */
|
|
4009
4038
|
function shouldWatch(inst) {
|
|
4010
4039
|
if (inst.status === 'requested' || inst.status === 'provisioning' || inst.status === 'tearing_down') return true;
|
|
@@ -4035,11 +4064,12 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
4035
4064
|
revive: { names: ['Revive queued', 'Standing it up', 'Back live'], cls: '' },
|
|
4036
4065
|
teardown: { names: ['Teardown queued', 'Winding down', 'Torn down'], cls: 'flowDown' },
|
|
4037
4066
|
address: { names: ['Address queued', 'Wiring the address', 'Answering there'], cls: '' },
|
|
4038
|
-
settings: { names: ['Change saved', 'Restarting to apply it', 'Applied'], cls: '' }
|
|
4067
|
+
settings: { names: ['Change saved', 'Restarting to apply it', 'Applied'], cls: '' },
|
|
4068
|
+
restart: { names: ['Restart queued', 'Restarting', 'Answering again'], cls: '' }
|
|
4039
4069
|
};
|
|
4040
|
-
function stripMarkup(flow, stageNow, stagesDone, note) {
|
|
4070
|
+
function stripMarkup(flow, stageNow, stagesDone, note, extraCls) {
|
|
4041
4071
|
var f = FLOWS[flow];
|
|
4042
|
-
return '<div class="strip ' + f.cls + '" role="status" aria-live="polite">' +
|
|
4072
|
+
return '<div class="strip ' + f.cls + (extraCls ? ' ' + extraCls : '') + '" role="status" aria-live="polite">' +
|
|
4043
4073
|
'<ul class="stages">' + f.names.map(function (n, i) {
|
|
4044
4074
|
var c = stagesDone > i ? 'done' : (stageNow === i ? 'now' : '');
|
|
4045
4075
|
return '<li class="' + c + '"><span class="sdot"></span><span class="slbl">' + n + '</span></li>';
|
|
@@ -4072,9 +4102,25 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
4072
4102
|
if (a === 'settings-apply') return inst.open_intent.state === 'running'
|
|
4073
4103
|
? stripMarkup('settings', 1, 1, 'Applying a settings change — the project restarts to pick it up.')
|
|
4074
4104
|
: stripMarkup('settings', 0, 0, 'A settings change is queued — this card updates when it starts.');
|
|
4105
|
+
/* task 1003238: the owner asked for a restart — say so, and never claim
|
|
4106
|
+
it came back before a probe has said it did */
|
|
4107
|
+
if (a === 'restart') return inst.open_intent.state === 'running'
|
|
4108
|
+
? stripMarkup('restart', 1, 1, 'Restarting the project — it goes down and comes back up.')
|
|
4109
|
+
: stripMarkup('restart', 0, 0, 'A restart is queued — this card updates when it starts.');
|
|
4075
4110
|
if (a === 'provision') return stripMarkup(s === 'torn_down' ? 'revive' : 'standup', 0, 0, (s === 'torn_down' ? 'A revive is queued' : 'A standup is queued') + ' — this card updates when it starts.');
|
|
4076
4111
|
return stripMarkup('standup', 2, 2, 'A ' + esc(a) + ' run is queued — this card updates when it lands.');
|
|
4077
4112
|
}
|
|
4113
|
+
/* the moment it landed (task 1003243). Until now this fell straight through to
|
|
4114
|
+
'' — stages 1 and 2 lit while the work ran, then the strip vanished, so the
|
|
4115
|
+
founder watching their own card never saw the third stage the wizard's done
|
|
4116
|
+
panel shows them. stagesDone=3 lights all three; stageNow=-1 because nothing
|
|
4117
|
+
is in progress any more. The address row carries the address, so this line
|
|
4118
|
+
says the EVENT and does not repeat it. */
|
|
4119
|
+
if (justLive(inst)) {
|
|
4120
|
+
return stripMarkup(revive ? 'revive' : 'standup', -1, 3,
|
|
4121
|
+
revive ? 'It’s back — your project is answering again.' : 'Standing up finished — your project is answering.',
|
|
4122
|
+
'isLive');
|
|
4123
|
+
}
|
|
4078
4124
|
return '';
|
|
4079
4125
|
}
|
|
4080
4126
|
|
|
@@ -4147,7 +4193,17 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
4147
4193
|
/* the runner had no DNS hook (F16): the CAUSE beats the bare symptom —
|
|
4148
4194
|
a successful A record flips last_verified_up and retires the note */
|
|
4149
4195
|
if (inst.dns_note) return 'Its address is ' + link + '. ' + esc(inst.dns_note);
|
|
4150
|
-
|
|
4196
|
+
/* task 1003238: there IS an action now, so the sentence names it — the old
|
|
4197
|
+
copy could only counsel patience because nothing else was on offer. It
|
|
4198
|
+
names it through canRestart(), the SAME predicate the button renders on:
|
|
4199
|
+
a dedicated-shape project (no runner exec path) or one with work already
|
|
4200
|
+
queued gets no button, and pointing it at one would be the exact
|
|
4201
|
+
misdirection ADR 0199 §8 forbids — a refusal must name the action that IS
|
|
4202
|
+
available. Where there is no button the sentence stops at the full stop. */
|
|
4203
|
+
if (inst.last_verified_up === false) {
|
|
4204
|
+
return 'Not answering at ' + link + ' right now. We keep checking every few minutes and update this on its own if it comes back' +
|
|
4205
|
+
(canRestart(inst) ? ' — or restart it yourself below.' : '.');
|
|
4206
|
+
}
|
|
4151
4207
|
if (inst.last_verified_up) return 'Its address is ' + link + ' — last answered ' + esc(relTime(verifiedAt(inst))) + '.';
|
|
4152
4208
|
return 'Its address is ' + link + ' — confirming it answers.';
|
|
4153
4209
|
}
|
|
@@ -4234,6 +4290,7 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
4234
4290
|
if (inst.status === 'active' && inst.domain) acts.unshift('<a class="btn small ghost" href="https://' + esc(inst.domain) + '" target="_blank" rel="noopener">Open site →</a>');
|
|
4235
4291
|
if (canRetry(inst)) acts.push('<button type="button" class="btn small" data-retry="' + iid + '">Try again</button>');
|
|
4236
4292
|
if (canRevive(inst)) acts.push('<button type="button" class="btn small" data-revive="' + iid + '">Bring it back</button>');
|
|
4293
|
+
if (canRestart(inst)) acts.push('<button type="button" class="btn small" data-restart="' + iid + '">Restart it</button>');
|
|
4237
4294
|
if (canTeardown(inst)) acts.push('<button type="button" class="btn small danger" data-teardown="' + iid + '">Tear it down</button>');
|
|
4238
4295
|
if (acts.length) body += '<div class="actions" id="act-' + iid + '">' + acts.join('') + '</div>';
|
|
4239
4296
|
body += attachFold(inst, iid);
|
|
@@ -4270,7 +4327,7 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
4270
4327
|
if (!el || !el.getAttribute) return null;
|
|
4271
4328
|
if (el.tagName === 'SUMMARY') return 'details.attachFold summary';
|
|
4272
4329
|
if (el.hasAttribute('data-teardown-yes') || el.hasAttribute('data-teardown-no')) return '[data-teardown]';
|
|
4273
|
-
var keys = ['data-manage', 'data-attach', 'data-retry', 'data-revive', 'data-teardown'];
|
|
4330
|
+
var keys = ['data-manage', 'data-attach', 'data-retry', 'data-revive', 'data-restart', 'data-teardown'];
|
|
4274
4331
|
for (var i = 0; i < keys.length; i++) if (el.hasAttribute(keys[i])) return '[' + keys[i] + ']';
|
|
4275
4332
|
return null;
|
|
4276
4333
|
}
|
|
@@ -4340,7 +4397,11 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
4340
4397
|
var at = inst.last_verified_at ? new Date(inst.last_verified_at).getTime() : 0;
|
|
4341
4398
|
return JSON.stringify([inst.status, inst.domain, inst.error_note,
|
|
4342
4399
|
inst.open_intent && inst.open_intent.action, inst.open_intent && inst.open_intent.state,
|
|
4343
|
-
inst.last_verified_up, at ? Math.floor((Date.now() - at) / 60000) : -1
|
|
4400
|
+
inst.last_verified_up, at ? Math.floor((Date.now() - at) / 60000) : -1,
|
|
4401
|
+
/* the landing moment is time-boxed (task 1003243), so its own age is part of
|
|
4402
|
+
what the card renders from — otherwise a stalled heartbeat freezes
|
|
4403
|
+
last_verified_at's bucket and the moment never clears itself */
|
|
4404
|
+
justLive(inst)]);
|
|
4344
4405
|
}
|
|
4345
4406
|
function takeUpdate(id, inst) {
|
|
4346
4407
|
var prev = byId[id];
|
|
@@ -4506,6 +4567,7 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
4506
4567
|
if (inst.status === 'active' && inst.domain) acts.push('<a class="btn small ghost" href="https://' + esc(inst.domain) + '" target="_blank" rel="noopener">Open site →</a>');
|
|
4507
4568
|
if (canRetry(inst)) acts.push('<button type="button" class="btn small" data-m-retry>Try again</button>');
|
|
4508
4569
|
if (canRevive(inst)) acts.push('<button type="button" class="btn small" data-m-revive>Bring it back</button>');
|
|
4570
|
+
if (canRestart(inst)) acts.push('<button type="button" class="btn small" data-m-restart>Restart it</button>');
|
|
4509
4571
|
$('mActs').innerHTML = acts.join('');
|
|
4510
4572
|
$('mActs').hidden = !acts.length;
|
|
4511
4573
|
}
|
|
@@ -4542,11 +4604,33 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
4542
4604
|
box.textContent = 'Couldn’t reach the platform — check your connection and try again.';
|
|
4543
4605
|
}).then(function () { loadManage(true, true); });
|
|
4544
4606
|
}
|
|
4607
|
+
/* Restart from the project's own page — the same POST the list card sends */
|
|
4608
|
+
function mRestart() {
|
|
4609
|
+
var inst = manageInst;
|
|
4610
|
+
if (!inst) return;
|
|
4611
|
+
var box = $('mMsg');
|
|
4612
|
+
box.className = 'projMsg'; box.textContent = 'Asking the platform to restart it…';
|
|
4613
|
+
$('mActs').querySelectorAll('button').forEach(function (b) { b.disabled = true; });
|
|
4614
|
+
fetch(API + '/provisioning/instances/' + encodeURIComponent(inst.id) + '/restart', {
|
|
4615
|
+
method: 'POST', credentials: 'same-origin'
|
|
4616
|
+
}).then(function (res) {
|
|
4617
|
+
if (res.status === 401) { box.className = 'projMsg err'; box.innerHTML = signInAgain(); return; }
|
|
4618
|
+
return jsonOrEmpty(res).then(function (data) {
|
|
4619
|
+
if (res.ok) { box.className = 'projMsg ok'; box.textContent = data.message || 'Restart queued.'; return; }
|
|
4620
|
+
box.className = 'projMsg err';
|
|
4621
|
+
box.textContent = restartFailure(data, res.status);
|
|
4622
|
+
});
|
|
4623
|
+
}).catch(function () {
|
|
4624
|
+
box.className = 'projMsg err';
|
|
4625
|
+
box.textContent = 'Couldn’t reach the platform — check your connection and try again.';
|
|
4626
|
+
}).then(function () { loadManage(true, true); });
|
|
4627
|
+
}
|
|
4545
4628
|
$('mActs').addEventListener('click', function (ev) {
|
|
4546
4629
|
var t = ev.target.closest && ev.target.closest('button');
|
|
4547
4630
|
if (!t) return;
|
|
4548
4631
|
if (t.hasAttribute('data-m-retry')) return mReProvision('retry');
|
|
4549
4632
|
if (t.hasAttribute('data-m-revive')) return mReProvision('revive');
|
|
4633
|
+
if (t.hasAttribute('data-m-restart')) return mRestart();
|
|
4550
4634
|
});
|
|
4551
4635
|
|
|
4552
4636
|
/* an IN-FLIGHT project is watched here at the card's cadence — its own copy
|
|
@@ -5282,6 +5366,39 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
5282
5366
|
});
|
|
5283
5367
|
}
|
|
5284
5368
|
|
|
5369
|
+
/* ── restart an unreachable project (task 1003238). Its own route, because
|
|
5370
|
+
PROVISION_ENQUEUE_FROM excludes 'active' — re-POSTing the slug the way
|
|
5371
|
+
Try again / Bring it back do would be refused on every press. Every
|
|
5372
|
+
refusal is shown as the server worded it: the server owns the gate, and
|
|
5373
|
+
a card that invents its own reason drifts from it. ── */
|
|
5374
|
+
|
|
5375
|
+
function restartFailure(data, status) {
|
|
5376
|
+
var e = apiError(data, status);
|
|
5377
|
+
if (e.code === 'rate_limited') return e.message || 'Too many requests right now — give it a moment and try again.';
|
|
5378
|
+
return e.message || 'The restart could not be queued — try again in a moment.';
|
|
5379
|
+
}
|
|
5380
|
+
|
|
5381
|
+
function doRestart(id) {
|
|
5382
|
+
var inst = byId[id];
|
|
5383
|
+
if (!inst) return;
|
|
5384
|
+
quietActions(id);
|
|
5385
|
+
msg(id, 'Asking…');
|
|
5386
|
+
fetch(API + '/provisioning/instances/' + encodeURIComponent(id) + '/restart', {
|
|
5387
|
+
method: 'POST', credentials: 'same-origin'
|
|
5388
|
+
}).then(function (res) {
|
|
5389
|
+
if (res.status === 401) { msg(id, signInAgain(), 'err'); return null; }
|
|
5390
|
+
return jsonOrEmpty(res).then(function (data) {
|
|
5391
|
+
if (res.ok) { msg(id, esc(data.message || 'Restart queued.'), 'ok'); return; }
|
|
5392
|
+
msg(id, esc(restartFailure(data, res.status)), 'err');
|
|
5393
|
+
});
|
|
5394
|
+
}).catch(function () {
|
|
5395
|
+
msg(id, 'Couldn’t reach the platform — check your connection and try again.', 'err');
|
|
5396
|
+
}).then(function () {
|
|
5397
|
+
if (byId[id]) renderCard(byId[id]);
|
|
5398
|
+
watchOnce(id);
|
|
5399
|
+
});
|
|
5400
|
+
}
|
|
5401
|
+
|
|
5285
5402
|
/* ── domain attach: same truth-telling for every refusal (task 1002692) ── */
|
|
5286
5403
|
|
|
5287
5404
|
function doAttach(id) {
|
|
@@ -5340,6 +5457,7 @@ summary{min-height:24px;padding:3px 0;}
|
|
|
5340
5457
|
if (t.hasAttribute('data-attach')) return doAttach(t.getAttribute('data-attach'));
|
|
5341
5458
|
if (t.hasAttribute('data-retry')) return reProvision(t.getAttribute('data-retry'), 'retry');
|
|
5342
5459
|
if (t.hasAttribute('data-revive')) return reProvision(t.getAttribute('data-revive'), 'revive');
|
|
5460
|
+
if (t.hasAttribute('data-restart')) return doRestart(t.getAttribute('data-restart'));
|
|
5343
5461
|
});
|
|
5344
5462
|
|
|
5345
5463
|
/* ── boot: who am I → my instances ────────────────────────────────────── */
|
package/scripts/gds/provision.js
CHANGED
|
@@ -768,6 +768,107 @@ async function settingsApplyInstance(inst, deps) {
|
|
|
768
768
|
return { ok: true, healthy, consumed };
|
|
769
769
|
}
|
|
770
770
|
|
|
771
|
+
// ---------------------------------------------------------------------------
|
|
772
|
+
// Restart an ALREADY-ACTIVE instance the heartbeat cannot reach (task 1003238 —
|
|
773
|
+
// the honest owner action behind the card's "unreachable" line). This is
|
|
774
|
+
// settingsApplyInstance minus the env patch: nothing about the instance's
|
|
775
|
+
// configuration changes, so there is no file to read and no file to write — just
|
|
776
|
+
// the restart and the same PATIENT healthz verify, whose result is recorded on
|
|
777
|
+
// the liveness overlay so the card that sent the owner here updates itself.
|
|
778
|
+
//
|
|
779
|
+
// Why its own leg rather than a re-provision: PROVISION_ENQUEUE_FROM deliberately
|
|
780
|
+
// excludes 'active' (task 1002691), and provisionInstance is a no-op on an active
|
|
781
|
+
// instance by design — a full standup is the wrong tool for "turn it off and on
|
|
782
|
+
// again". Why not settings-apply: that leg only runs behind a value that actually
|
|
783
|
+
// CHANGED, which is exactly the case a stuck project does not have.
|
|
784
|
+
//
|
|
785
|
+
// Like the settings push, EVERY expected failure RETURNS { ok:false } instead of
|
|
786
|
+
// throwing: the drain loop's catch flips the instance to 'error', and an instance
|
|
787
|
+
// that is merely unreachable must not be relabelled broken by the very action
|
|
788
|
+
// meant to recover it — that would drop it off the liveness sweep and leave a
|
|
789
|
+
// re-provision as its only exit (ADR 0190 §2).
|
|
790
|
+
// ---------------------------------------------------------------------------
|
|
791
|
+
async function restartInstance(inst, deps) {
|
|
792
|
+
const { db, provisioning, exec, log, apply, privileged } = deps;
|
|
793
|
+
// Same box routing as settingsApplyInstance: a standalone runs control-plane-local.
|
|
794
|
+
const boxExec = inst.hosting_shape === 'standalone' ? (deps.controlExec || exec) : exec;
|
|
795
|
+
const sudoP = privileged ? 'sudo ' : '';
|
|
796
|
+
log(`restart ${inst.slug} (shape=${inst.hosting_shape}, status=${inst.status})`);
|
|
797
|
+
|
|
798
|
+
// A clean no-op (not an error) so a teardown that raced the restart never burns
|
|
799
|
+
// the intent's retry budget: every non-active state either has nothing running
|
|
800
|
+
// yet or is being reclaimed.
|
|
801
|
+
if (inst.status !== 'active') {
|
|
802
|
+
log(` instance is '${inst.status}' — nothing running to restart`);
|
|
803
|
+
return { ok: true, noop: true };
|
|
804
|
+
}
|
|
805
|
+
// A dedicated droplet has no runner exec path (the placeManifestCreds refusal).
|
|
806
|
+
// The route refuses to enqueue for this shape too; this is the second gate.
|
|
807
|
+
if (inst.hosting_shape === 'dedicated') {
|
|
808
|
+
return { ok: false, error: `restart is not automated for a dedicated droplet — run 'systemctl restart ${inst.slug}' on that machine` };
|
|
809
|
+
}
|
|
810
|
+
if (!apply) {
|
|
811
|
+
boxExec(`${sudoP}systemctl restart ${inst.slug}`);
|
|
812
|
+
boxExec(healthzCmd(inst), { allowFail: true });
|
|
813
|
+
log(` ✓ would restart ${inst.slug} and wait for it to answer (dry-run — nothing ran)`);
|
|
814
|
+
return { ok: true, healthy: null };
|
|
815
|
+
}
|
|
816
|
+
try {
|
|
817
|
+
log(' [service] restart');
|
|
818
|
+
boxExec(`${sudoP}systemctl restart ${inst.slug}`);
|
|
819
|
+
} catch (e) {
|
|
820
|
+
// ok:false, never a throw — the retry simply restarts again (idempotent).
|
|
821
|
+
return { ok: false, error: `could not restart ${inst.slug}: ${(e && e.message) || e}` };
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
// Verify it came back, with the manifest probe's patience (the F23 lesson). A
|
|
825
|
+
// false "down" is never cosmetic here: this instance is ALREADY flagged
|
|
826
|
+
// unreachable, and a hasty verdict would tell the owner their restart failed.
|
|
827
|
+
log(' [healthz] verify the instance answers after the restart');
|
|
828
|
+
const sleep = deps.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
829
|
+
let healthy = false;
|
|
830
|
+
for (let i = 0; i < MANIFEST_VERIFY_TRIES; i++) {
|
|
831
|
+
const res = boxExec(healthzCmd(inst), { allowFail: true });
|
|
832
|
+
if (res && !res.softFailed) { healthy = true; break; }
|
|
833
|
+
if (i < MANIFEST_VERIFY_TRIES - 1) await sleep(MANIFEST_VERIFY_INTERVAL_MS);
|
|
834
|
+
}
|
|
835
|
+
// Same narrowness as the settings push: a loopback PASS proves the process, not
|
|
836
|
+
// the address, so only the domain-scoped sweep may claim "up" for an instance
|
|
837
|
+
// that HAS a domain. Silence after the whole budget is a verdict on any shape.
|
|
838
|
+
const probeIsVerdict = healthy === false || (healthy === true && !inst.domain);
|
|
839
|
+
if (probeIsVerdict && provisioning.recordLivenessProbe) await provisioning.recordLivenessProbe(db, inst.id, healthy).catch(() => {});
|
|
840
|
+
// A restart that worked retires any "we stopped trying" note left by earlier
|
|
841
|
+
// work (the F19 latch) — the note outliving its own recovery is the bug.
|
|
842
|
+
// DELIBERATELY gated on `healthy`, NOT on probeIsVerdict: the two answer
|
|
843
|
+
// different questions. probeIsVerdict asks "may this run claim the instance is
|
|
844
|
+
// UP?" — a loopback pass may not, for an instance whose address the sweep owns.
|
|
845
|
+
// The note asks "did work stop partway?", and a process that restarted and
|
|
846
|
+
// answered is that work finishing, whatever the public address does next.
|
|
847
|
+
// Holding the note until the sweep agrees would leave a stale "nothing more
|
|
848
|
+
// will happen" on the card of a project that just visibly recovered.
|
|
849
|
+
if (healthy && provisioning.clearErrorNote) await provisioning.clearErrorNote(db, inst.id).catch(() => {});
|
|
850
|
+
// Name WHAT was observed, not a verdict this probe cannot reach. healthzCmd is
|
|
851
|
+
// loopback (127.0.0.1:<port>) while the liveness sweep probes the PUBLIC
|
|
852
|
+
// https://<domain>/healthz — so an instance can answer here while DNS, Caddy or
|
|
853
|
+
// TLS is the thing actually broken. "Answering again" on a domained instance
|
|
854
|
+
// would contradict the card still reading "unreachable"; the same asymmetry is
|
|
855
|
+
// why probeIsVerdict above declines to write an up-verdict for that case.
|
|
856
|
+
const outcome = !healthy ? 'restarted, but it is not answering on the box — something there needs a look'
|
|
857
|
+
: inst.domain ? 'restarted and answering on the box; whether its public address answers is for the next liveness sweep to say'
|
|
858
|
+
: 'restarted and answering again';
|
|
859
|
+
await provisioning.recordEvent(db, {
|
|
860
|
+
instanceId: inst.id, ownerBuilderId: inst.owner_builder_id, event: 'restarted',
|
|
861
|
+
detail: `restart requested — ${outcome}`,
|
|
862
|
+
actor: 'runner',
|
|
863
|
+
}).catch(() => {});
|
|
864
|
+
log(healthy ? ` ✓ ${inst.slug} ${outcome}` : ` ⚠ ${inst.slug} ${outcome}`);
|
|
865
|
+
// ok:true either way: the restart RAN. A project that stays down is a truthful
|
|
866
|
+
// observation for the card, not a failed intent to retry MAX_INTENT_ATTEMPTS
|
|
867
|
+
// times — restarting a broken instance four more times fixes nothing and would
|
|
868
|
+
// end by stamping an error_note over the owner's own deliberate action.
|
|
869
|
+
return { ok: true, healthy };
|
|
870
|
+
}
|
|
871
|
+
|
|
771
872
|
// ---------------------------------------------------------------------------
|
|
772
873
|
// Teardown one instance (reverse the runbook; idempotent).
|
|
773
874
|
// ---------------------------------------------------------------------------
|
|
@@ -1049,6 +1150,7 @@ async function cmdRunIntents(deps) {
|
|
|
1049
1150
|
else if (intent.action === 'teardown') result = await teardownInstance(inst, deps);
|
|
1050
1151
|
else if (intent.action === 'domain-attach') result = await domainAttachInstance(inst, deps);
|
|
1051
1152
|
else if (intent.action === 'settings-apply') result = await settingsApplyInstance(inst, deps);
|
|
1153
|
+
else if (intent.action === 'restart') result = await restartInstance(inst, deps);
|
|
1052
1154
|
else result = { ok: false, error: `unknown intent action '${intent.action}' — refusing to guess` };
|
|
1053
1155
|
if (result && result.ok === false) {
|
|
1054
1156
|
if (intent.attempts >= MAX_INTENT_ATTEMPTS) {
|
|
@@ -1238,6 +1340,8 @@ module.exports = {
|
|
|
1238
1340
|
domainAttachInstance, cmdRunIntents, cmdReconcile, cmdStatus,
|
|
1239
1341
|
// live settings push (task 1003140) + the in-place env patch + read-back it rides on
|
|
1240
1342
|
settingsApplyInstance, upsertEnvVars, settingsEnvVarsFor, settingsConsumed, instanceManifestCmd,
|
|
1343
|
+
// restart an unreachable ACTIVE instance (task 1003238) — the settings leg minus the env patch
|
|
1344
|
+
restartInstance,
|
|
1241
1345
|
// GitHub App Manifest flow (task 2080, ADR 0133)
|
|
1242
1346
|
convertManifestCode, placeManifestCreds, drainOAuthManifests, MANIFEST_UA,
|
|
1243
1347
|
// cross-site identity federation (ADR 0141 §4, task #1002109)
|
package/src/module-api.js
CHANGED
|
@@ -1236,8 +1236,12 @@ const { validateOrRespond, LIMITS, parseId, asyncHandler, corsPublicGet, parsePa
|
|
|
1236
1236
|
// landed since 1.19.396 with no explicit bump. run 33193365480. (task 1002620)
|
|
1237
1237
|
// 1.19.398 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
1238
1238
|
// landed since 1.19.397 with no explicit bump. run 33199474439. (task 1002620)
|
|
1239
|
+
// 1.19.399 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
1240
|
+
// landed since 1.19.398 with no explicit bump. run 33201243958. (task 1002620)
|
|
1241
|
+
// 1.19.400 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
1242
|
+
// landed since 1.19.399 with no explicit bump. run 33201979306. (task 1002620)
|
|
1239
1243
|
// ---------------------------------------------------------------------------
|
|
1240
|
-
const CORE_VERSION = '1.19.
|
|
1244
|
+
const CORE_VERSION = '1.19.400'; // CI auto-patch carrier (ADR 0161); see changelog above
|
|
1241
1245
|
|
|
1242
1246
|
// A namespaced logger so a module's log lines are attributable + consistent.
|
|
1243
1247
|
// Usage: const log = api.logger('dev-box'); log.info('mounted');
|