@7ots/cli 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/cli/7ots.mjs +28 -4
- package/cli/lib/account.mjs +152 -2
- package/cli/lib/assistant.mjs +106 -41
- package/cli/lib/brain.mjs +13 -2
- package/cli/lib/config.mjs +3 -0
- package/cli/lib/inferences.mjs +30 -6
- package/cli/lib/integrations.mjs +69 -5
- package/cli/lib/orquesta.mjs +44 -1
- package/cli/lib/pet-server.mjs +344 -11
- package/cli/pet/electron/director.cjs +37 -5
- package/cli/pet/electron/main.cjs +92 -10
- package/cli/pet/pet.html +149 -10
- package/cli/pet/settings.html +545 -42
- package/dist/7ots.esm.js +216 -137
- package/dist/7ots.esm.js.map +4 -4
- package/dist/7ots.iife.js +216 -137
- package/dist/7ots.iife.js.map +4 -4
- package/package.json +1 -1
- package/server/channels/agent.mjs +7 -1
- package/server/channels/index.mjs +3 -2
- package/server/platform/connect.mjs +4 -1
- package/server/platform/db.mjs +9 -1
- package/server/platform/device.mjs +101 -4
- package/server/platform/mods.mjs +222 -0
- package/server/platform/orquesta.mjs +78 -14
- package/server/platform/profile.mjs +161 -0
- package/server/platform/routes.mjs +38 -33
- package/server/platform/runtime.mjs +3 -1
- package/server/platform/store.mjs +1 -1
- package/server/platform/vm.mjs +370 -91
- package/server/server.mjs +12 -2
- package/src/avatar/three.js +3 -2
- package/src/character/AvatarEditor.js +95 -7
- package/src/character/Character.js +116 -17
- package/src/character/TradingCard.js +226 -0
- package/src/character/mods/index.js +4 -0
- package/src/character/mods/orc.js +64 -0
- package/src/character/mods.js +251 -0
- package/src/character/more/acc-head.js +213 -0
- package/src/character/more/acc-rest.js +342 -0
- package/src/character/more/brows.js +39 -0
- package/src/character/more/cheeks.js +78 -0
- package/src/character/more/eyes.js +288 -0
- package/src/character/more/mouths.js +259 -0
- package/src/character/more/patterns.js +120 -0
- package/src/character/parts.js +90 -0
- package/src/character/skin.js +166 -1
- package/src/character/styles.js +1285 -9
- package/src/character3d/Character3D.js +148 -35
- package/src/character3d/index.js +7 -1
- package/src/character3d/model.js +367 -0
- package/src/i18n/messages/character.js +34 -22
- package/src/i18n/messages/platform.js +15 -0
- package/src/identity/random.js +17 -3
- package/src/identity/schema.js +10 -0
- package/src/index.js +1 -0
package/server/platform/vm.mjs
CHANGED
|
@@ -4,18 +4,31 @@
|
|
|
4
4
|
* Provider 'byo': the owner brings their own Linux box: the dashboard hands out a one-time
|
|
5
5
|
* `7ve_` code and the exact command; `7ots vm enroll` trades it for a `7vm_` token bound to that one
|
|
6
6
|
* ot and `7ots vm worker` connects outbound only (no SSH keys ever reach 7ots).
|
|
7
|
-
* Provider 'orquesta'
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
7
|
+
* Provider 'orquesta' ("7ots server, operated by Orquesta"): gated by accounts.vm_enabled and
|
|
8
|
+
* ORQUESTA_VM_TOKEN (7ots' partner oak_ key: orgs:write + projects:write + vms:write, never shown to
|
|
9
|
+
* anyone). Each 7ots account gets its own Orquesta child org (POST /api/v1/partner/orgs, external_ref
|
|
10
|
+
* `<prefix>acct:<accountId>`, kept in accounts.orq_org_id) and every project/VM call for it carries
|
|
11
|
+
* `X-Orquesta-Org: <child>`. Orquesta bills the child through its own Stripe Checkout: until the child's
|
|
12
|
+
* billing is active, POST /vm answers { needsPayment, checkoutUrl } (back to /app/?vm_ots=<id>).
|
|
13
|
+
* Provider 'orquesta_user': the owner connected their own Orquesta account (orquesta.mjs) with an oak_
|
|
14
|
+
* key that has projects:write + vms:write; the VM is created in THEIR org, with their key (which never
|
|
15
|
+
* leaves orquesta.mjs) and billed to them by Orquesta. No child org, no 7ots checkout.
|
|
16
|
+
* Both create project `ot-<id>` + its VM with a bootstrap that enrolls it the same way (code valid
|
|
17
|
+
* VM_ORQ_ENROLL_MIN, default 60, to cover the boot). Quotas VM_PER_ACCOUNT (default 1, both providers)
|
|
18
|
+
* and VM_MAX_TOTAL (default 10, 'orquesta' only). Deleting the VM, the ot or the account tears down VM
|
|
19
|
+
* and project with the credential that made them; releasing the account also deletes its child org
|
|
20
|
+
* (which cancels its Orquesta subscription). A reconciliation every VM_RECONCILE_MIN (30) compares
|
|
21
|
+
* GET /api/v1/vms of each child org with ots_vms, refreshes 'orquesta_user' VMs one by one, retries
|
|
22
|
+
* failed teardowns and org deletions and logs orphans (VM_REAP_ORPHANS=1 deletes them).
|
|
13
23
|
* Env: ORQUESTA_URL, ORQUESTA_VM_SIZE (default: cheapest), ORQUESTA_VM_SIZES ('slug[:usd],…'), ORQUESTA_VM_REGION,
|
|
14
24
|
* ORQUESTA_VM_REF_PREFIX ('7ots:', distinct per 7ots install sharing one org), SEVENOTS_VM_CLI (pinned, 0.1.6).
|
|
15
25
|
*
|
|
16
26
|
* Dashboard (session + X-7ots-Admin, via routes.mjs):
|
|
17
|
-
* GET /api/platform/ots/:id/vm { vm|null, jobs, kinds, daily, paidVms, hosted: { available, canCreate, reason, sizes, … } }
|
|
18
|
-
* POST /api/platform/ots/:id/vm { provider: 'orquesta', size? } → 202 { vm } (provisioning in the background)
|
|
27
|
+
* GET /api/platform/ots/:id/vm { vm|null, jobs, kinds, daily, paidVms, hosted: { available, canCreate, reason, sizes, billing, user, defaultOption, … } }
|
|
28
|
+
* POST /api/platform/ots/:id/vm { provider: 'orquesta'|'orquesta_user', size? } → 202 { vm } (provisioning in the background)
|
|
29
|
+
* · 'orquesta' unpaid → 200 { needsPayment: true, checkoutUrl, monthlyPrice }
|
|
30
|
+
* GET /api/platform/ots/:id/vm/billing { status, active } of the account's child org (polled after the checkout)
|
|
31
|
+
* DELETE /api/platform/ots/:id/vm/subscription no 7ots server left → deletes the child org (ends the subscription)
|
|
19
32
|
* POST /api/platform/ots/:id/vm/enroll { provider?: 'byo' } → { code, expiresIn, command }
|
|
20
33
|
* DELETE /api/platform/ots/:id/vm revokes the worker; byo → terminated, orquesta → 202 terminating → terminated
|
|
21
34
|
*
|
|
@@ -46,6 +59,7 @@ import { errorText, reqT } from '../i18n.mjs';
|
|
|
46
59
|
import { getDb, tx } from './db.mjs';
|
|
47
60
|
import { newId, sha256, token } from './crypto.mjs';
|
|
48
61
|
import { getOts, getOwnOts, getPlatformState, setPlatformState } from './store.mjs';
|
|
62
|
+
import { orquestaInfra, orquestaVmAccess } from './orquesta.mjs';
|
|
49
63
|
import { otsStep } from './runtime.mjs';
|
|
50
64
|
|
|
51
65
|
const penv = process.env;
|
|
@@ -121,7 +135,7 @@ function vmView(vm) {
|
|
|
121
135
|
enrollExpiresAt: vm.enroll_hash ? vm.enroll_exp : null,
|
|
122
136
|
error: vm.error || null,
|
|
123
137
|
since: vm.created_at,
|
|
124
|
-
...(vm.provider
|
|
138
|
+
...(HOSTED.includes(vm.provider) ? { orqStatus: vm.orq_status || null, ip: vm.ip || null, monthlyPrice: vm.price_cents ? vm.price_cents / 100 : null, checkedAt: vm.orq_checked_at || null } : {}),
|
|
125
139
|
};
|
|
126
140
|
}
|
|
127
141
|
|
|
@@ -207,12 +221,14 @@ const MAX_TOTAL = () => Math.max(0, Number(penv.VM_MAX_TOTAL ?? 10));
|
|
|
207
221
|
const PER_ACCOUNT = () => Math.max(0, Number(penv.VM_PER_ACCOUNT ?? 1));
|
|
208
222
|
const RECONCILE_MS = () => Math.max(0.01, Number(penv.VM_RECONCILE_MIN || 30)) * 60_000;
|
|
209
223
|
const PROJECT_RETRY_MS = () => Number(penv.VM_ORQ_RETRY_MS || 10_000);
|
|
210
|
-
const
|
|
224
|
+
const HOSTED = ['orquesta', 'orquesta_user'];
|
|
225
|
+
const LIVE_ORQ = "provider IN ('orquesta', 'orquesta_user') AND status <> 'terminated'"; // per-account quota
|
|
226
|
+
const LIVE_PARTNER = "provider = 'orquesta' AND status <> 'terminated'"; // 7ots' own fleet (VM_MAX_TOTAL)
|
|
211
227
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
212
228
|
|
|
213
229
|
/** The CLI the VM installs, pinned (SEVENOTS_VM_CLI: '0.1.6' or '@7ots/cli@0.1.6'). */
|
|
214
230
|
function cliSpec() {
|
|
215
|
-
const v = String(penv.SEVENOTS_VM_CLI || '0.1.
|
|
231
|
+
const v = String(penv.SEVENOTS_VM_CLI || '0.1.7').trim();
|
|
216
232
|
const s = v.startsWith('@7ots/cli@') ? v : `@7ots/cli@${v}`;
|
|
217
233
|
if (!/^@7ots\/cli@\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/.test(s)) throw new Error('SEVENOTS_VM_CLI must be an exact version');
|
|
218
234
|
return s;
|
|
@@ -245,12 +261,39 @@ function priceCents(v) {
|
|
|
245
261
|
|
|
246
262
|
class OrqError extends Error {}
|
|
247
263
|
|
|
248
|
-
|
|
264
|
+
const orqFail = (method, path, status, data) => {
|
|
265
|
+
const msg = typeof data.error === 'string' ? data.error : data.error?.message || data.message || '';
|
|
266
|
+
return Object.assign(new OrqError(`Orquesta ${method} ${path.replace(/\/[^/]{20,}/g, '/…')}: HTTP ${status}${msg ? ` ${clean(msg, 200)}` : ''}`), { status, code: data.error?.code || data.code });
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Orquesta's /api/v1 with a credential:
|
|
271
|
+
* { kind: 'partner', org } → ORQUESTA_VM_TOKEN, plus X-Orquesta-Org when org is a child ('' = 7ots' own org);
|
|
272
|
+
* { kind: 'user', accountId } → the owner's own key, through orquesta.mjs (it never reaches this module).
|
|
273
|
+
* Without cred: the partner key on its own org (the /partner/orgs routes refuse X-Orquesta-Org).
|
|
274
|
+
*/
|
|
275
|
+
async function orq(method, path, body, timeoutMs = 20_000, cred = null) {
|
|
276
|
+
if (cred?.kind === 'user') {
|
|
277
|
+
let r;
|
|
278
|
+
try {
|
|
279
|
+
r = await orquestaInfra(cred.accountId, method, `/api/v1${path}`, body, timeoutMs);
|
|
280
|
+
} catch (e) {
|
|
281
|
+
// Not connected / reconnect / unreachable: never a 404 or 409 of the resource itself.
|
|
282
|
+
throw Object.assign(new OrqError(`Orquesta (owner's account): ${clean(e.message, 200)}`), { status: 0, code: 'credentials', i18n: e.i18n });
|
|
283
|
+
}
|
|
284
|
+
if (!r.ok) throw orqFail(method, path, r.status, r.data || {});
|
|
285
|
+
return r.status === 204 ? {} : r.data || {};
|
|
286
|
+
}
|
|
249
287
|
let res;
|
|
250
288
|
try {
|
|
251
289
|
res = await fetch(`${ORQ()}/api/v1${path}`, {
|
|
252
290
|
method,
|
|
253
|
-
headers: {
|
|
291
|
+
headers: {
|
|
292
|
+
Accept: 'application/json',
|
|
293
|
+
Authorization: `Bearer ${orqToken()}`,
|
|
294
|
+
...(cred?.org ? { 'X-Orquesta-Org': cred.org } : {}),
|
|
295
|
+
...(body ? { 'Content-Type': 'application/json' } : {}),
|
|
296
|
+
},
|
|
254
297
|
body: body ? JSON.stringify(body) : undefined,
|
|
255
298
|
redirect: 'manual',
|
|
256
299
|
signal: AbortSignal.timeout(timeoutMs),
|
|
@@ -259,13 +302,76 @@ async function orq(method, path, body, timeoutMs = 20_000) {
|
|
|
259
302
|
throw Object.assign(new OrqError(`Orquesta unreachable (${e.name === 'TimeoutError' ? 'timeout' : e.code || e.name})`), { status: 0 });
|
|
260
303
|
}
|
|
261
304
|
const data = res.status === 204 ? {} : await res.json().catch(() => ({}));
|
|
262
|
-
if (!res.ok)
|
|
263
|
-
const msg = typeof data.error === 'string' ? data.error : data.error?.message || data.message || '';
|
|
264
|
-
throw Object.assign(new OrqError(`Orquesta ${method} ${path.replace(/\/[^/]{20,}/g, '/…')}: HTTP ${res.status}${msg ? ` ${clean(msg, 200)}` : ''}`), { status: res.status, code: data.error?.code || data.code });
|
|
265
|
-
}
|
|
305
|
+
if (!res.ok) throw orqFail(method, path, res.status, data);
|
|
266
306
|
return data;
|
|
267
307
|
}
|
|
268
308
|
|
|
309
|
+
/** The credential that made (and unmakes) a row's VM. */
|
|
310
|
+
const credOf = (row) => (row.provider === 'orquesta_user' ? { kind: 'user', accountId: row.account_id } : { kind: 'partner', org: row.orq_org_id || '' });
|
|
311
|
+
const credOfItem = (x) => (x.user ? { kind: 'user', accountId: x.accountId } : { kind: 'partner', org: x.org || '' });
|
|
312
|
+
|
|
313
|
+
// ── the account's child org (provider 'orquesta') ──
|
|
314
|
+
|
|
315
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
316
|
+
const accountOrg = (accountId) => getDb().prepare('SELECT orq_org_id FROM accounts WHERE id = ?').get(accountId)?.orq_org_id || '';
|
|
317
|
+
const accountRef = (accountId) => `${REF_PREFIX()}acct:${accountId}`;
|
|
318
|
+
const ensuring = new Map(); // account id → Promise<org id>
|
|
319
|
+
|
|
320
|
+
/** The account's Orquesta child org, created the first time (idempotent on Orquesta's side by external_ref). */
|
|
321
|
+
function ensureChildOrg(accountId) {
|
|
322
|
+
const have = accountOrg(accountId);
|
|
323
|
+
if (have) return Promise.resolve(have);
|
|
324
|
+
if (!ensuring.has(accountId)) {
|
|
325
|
+
const p = (async () => {
|
|
326
|
+
const r = await orq('POST', '/partner/orgs', { external_ref: accountRef(accountId), name: `7ots account ${accountId}`.slice(0, 100) });
|
|
327
|
+
const id = String(r.org?.id || '');
|
|
328
|
+
if (!UUID_RE.test(id)) throw new OrqError('Orquesta returned no organization id');
|
|
329
|
+
getDb().prepare("UPDATE accounts SET orq_org_id = ? WHERE id = ? AND orq_org_id = ''").run(id, accountId);
|
|
330
|
+
return accountOrg(accountId) || id;
|
|
331
|
+
})().finally(() => ensuring.delete(accountId));
|
|
332
|
+
ensuring.set(accountId, p);
|
|
333
|
+
}
|
|
334
|
+
return ensuring.get(accountId);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const billingSeen = new Map(); // account id → { status, at } (last answer from Orquesta, for the card)
|
|
338
|
+
|
|
339
|
+
/** Billing status of the account's child org: 'active' | 'past_due' | 'canceled' | null (never paid). */
|
|
340
|
+
async function childBilling(accountId, org) {
|
|
341
|
+
const r = await orq('GET', `/partner/orgs/${encodeURIComponent(org)}/billing`, null, 10_000);
|
|
342
|
+
const status = r.status ? clean(r.status, 20) : null;
|
|
343
|
+
billingSeen.set(accountId, { status, at: now() });
|
|
344
|
+
return status;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
function orgDeletes() {
|
|
348
|
+
try {
|
|
349
|
+
const q = JSON.parse(getPlatformState('vm_org_deletes') || '[]');
|
|
350
|
+
return Array.isArray(q) ? q : [];
|
|
351
|
+
} catch {
|
|
352
|
+
return [];
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
const setOrgDeletes = (q) => setPlatformState('vm_org_deletes', JSON.stringify(q.slice(-500)));
|
|
356
|
+
|
|
357
|
+
/** Deletes a child org (Orquesta cancels its subscription first). 409 while a VM is still going away → queued. */
|
|
358
|
+
async function dropChildOrg(accountId, org) {
|
|
359
|
+
if (!org) return true;
|
|
360
|
+
try {
|
|
361
|
+
await orq('DELETE', `/partner/orgs/${encodeURIComponent(org)}`, null, 30_000);
|
|
362
|
+
} catch (e) {
|
|
363
|
+
if (e.status !== 404) {
|
|
364
|
+
console.warn(`[7ots] vm org delete ${org}:`, e.message);
|
|
365
|
+
setOrgDeletes([...orgDeletes().filter((x) => x.org !== org), { org, accountId, at: now() }]);
|
|
366
|
+
return false;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
setOrgDeletes(orgDeletes().filter((x) => x.org !== org));
|
|
370
|
+
getDb().prepare("UPDATE accounts SET orq_org_id = '' WHERE id = ? AND orq_org_id = ?").run(accountId, org);
|
|
371
|
+
billingSeen.delete(accountId);
|
|
372
|
+
return true;
|
|
373
|
+
}
|
|
374
|
+
|
|
269
375
|
/** Bash run as root at the end of the VM's user-data (≤ 8 KB). Every value is validated, never free text. */
|
|
270
376
|
export function bootstrapScript({ server, code, cli = cliSpec() }) {
|
|
271
377
|
if (!/^https?:\/\/[A-Za-z0-9.:[\]-]+(\/[A-Za-z0-9._~/-]*)?$/.test(server)) throw new Error('bad server url');
|
|
@@ -333,6 +439,7 @@ function teardownQueue() {
|
|
|
333
439
|
return [];
|
|
334
440
|
}
|
|
335
441
|
}
|
|
442
|
+
/** item: { pid, otsId, accountId, org, user } — enough to rebuild the credential that made the project. */
|
|
336
443
|
function queueTeardown(item) {
|
|
337
444
|
const q = teardownQueue().filter((x) => x.pid !== item.pid);
|
|
338
445
|
setPlatformState('vm_teardowns', JSON.stringify([...q, { ...item, at: now() }].slice(-500)));
|
|
@@ -340,19 +447,20 @@ function queueTeardown(item) {
|
|
|
340
447
|
function dequeueTeardown(pid) {
|
|
341
448
|
setPlatformState('vm_teardowns', JSON.stringify(teardownQueue().filter((x) => x.pid !== pid)));
|
|
342
449
|
}
|
|
450
|
+
const teardownItem = (row, pid = row.orq_project_id) => ({ pid, otsId: row.ots_id, accountId: row.account_id, org: row.orq_org_id || '', user: row.provider === 'orquesta_user' });
|
|
343
451
|
|
|
344
452
|
/** Terminates the project's VM, then the project. 404 = already gone. 409 on the project = VM still going away. */
|
|
345
|
-
async function destroy(pid, { retries = 6 } = {}) {
|
|
453
|
+
async function destroy(pid, { retries = 6, cred = null } = {}) {
|
|
346
454
|
if (!pid) return;
|
|
347
455
|
const p = `/projects/${encodeURIComponent(pid)}`;
|
|
348
456
|
try {
|
|
349
|
-
await orq('DELETE', `${p}/vm`, null, 60_000);
|
|
457
|
+
await orq('DELETE', `${p}/vm`, null, 60_000, cred);
|
|
350
458
|
} catch (e) {
|
|
351
459
|
if (e.status !== 404) throw e;
|
|
352
460
|
}
|
|
353
461
|
for (let i = 0; ; i++) {
|
|
354
462
|
try {
|
|
355
|
-
return void (await orq('DELETE', p));
|
|
463
|
+
return void (await orq('DELETE', p, null, 20_000, cred));
|
|
356
464
|
} catch (e) {
|
|
357
465
|
if (e.status === 404) return;
|
|
358
466
|
if (e.status !== 409 || i >= retries) throw e;
|
|
@@ -367,7 +475,7 @@ async function finishTeardown(otsId) {
|
|
|
367
475
|
const row = vmRow(otsId);
|
|
368
476
|
if (!row || row.status !== 'terminating') return;
|
|
369
477
|
try {
|
|
370
|
-
await destroy(row.orq_project_id);
|
|
478
|
+
await destroy(row.orq_project_id, { cred: credOf(row) });
|
|
371
479
|
setRow(otsId, { status: 'terminated', orq_status: 'terminated', ip: '', error: '' }, "status = 'terminating'");
|
|
372
480
|
dequeueTeardown(row.orq_project_id);
|
|
373
481
|
} catch (e) {
|
|
@@ -376,43 +484,55 @@ async function finishTeardown(otsId) {
|
|
|
376
484
|
}
|
|
377
485
|
}
|
|
378
486
|
|
|
487
|
+
/** What the owner reads when a create fails (in the language of the request that started it). */
|
|
488
|
+
function createError(e, cred, t) {
|
|
489
|
+
if (cred.kind === 'partner' && e.status === 402) return t('platform.vm.needsPayment');
|
|
490
|
+
if (cred.kind === 'user' && e.code === 'credentials') return e.i18n ? t(e.i18n.key, e.i18n.vars) : e.message;
|
|
491
|
+
if (cred.kind === 'user' && (e.status === 401 || e.status === 403)) return `${t('platform.vm.userScopes')} (${e.message})`;
|
|
492
|
+
return e.message;
|
|
493
|
+
}
|
|
494
|
+
|
|
379
495
|
/** Creates project + VM. The row may be deleted (ot gone) or set 'terminating' meanwhile: then it undoes its own work. */
|
|
380
|
-
async function provision(ots, { server, code, size, region }) {
|
|
496
|
+
async function provision(ots, { server, code, size, region, cred, t }) {
|
|
381
497
|
let pid = '';
|
|
382
498
|
const gone = () => !vmRow(ots.id);
|
|
499
|
+
const item = () => ({ pid, otsId: ots.id, accountId: ots.accountId, org: cred.org || '', user: cred.kind === 'user' });
|
|
500
|
+
const undo = (retries) => destroy(pid, { retries, cred }).catch(() => queueTeardown(item()));
|
|
383
501
|
try {
|
|
384
|
-
const p = await orq('POST', '/projects', { name: `ot-${ots.id}`, description: clean(`7ots cloud worker for "${ots.name}" (ot ${ots.id}, account ${ots.accountId})`, 300) });
|
|
502
|
+
const p = await orq('POST', '/projects', { name: `ot-${ots.id}`, description: clean(`7ots cloud worker for "${ots.name}" (ot ${ots.id}, account ${ots.accountId})`, 300) }, 20_000, cred);
|
|
385
503
|
pid = String(p.project?.id || '');
|
|
386
504
|
if (!pid) throw new OrqError('Orquesta returned no project id');
|
|
387
505
|
setRow(ots.id, { orq_project_id: pid });
|
|
388
|
-
if (gone()) return void (await
|
|
506
|
+
if (gone()) return void (await undo(0));
|
|
389
507
|
if (vmRow(ots.id).status !== 'creating') return; // DELETE came first: finishTeardown takes it from here
|
|
390
|
-
const r = await orq('POST', `/projects/${encodeURIComponent(pid)}/vm`, { size, ...(region ? { region } : {}), external_ref: `${REF_PREFIX()}${ots.id}`, bootstrap: bootstrapScript({ server, code }) }, 6 * 60_000);
|
|
508
|
+
const r = await orq('POST', `/projects/${encodeURIComponent(pid)}/vm`, { size, ...(region ? { region } : {}), external_ref: `${REF_PREFIX()}${ots.id}`, bootstrap: bootstrapScript({ server, code }) }, 6 * 60_000, cred);
|
|
391
509
|
const v = r.vm || {};
|
|
392
|
-
if (gone()) return void (await
|
|
510
|
+
if (gone()) return void (await undo());
|
|
393
511
|
setRow(ots.id, { orq_vm_id: String(v.id || ''), orq_status: clean(v.status, 40), ip: clean(v.ip, 60), price_cents: priceCents(v) ?? vmRow(ots.id).price_cents, orq_checked_at: now() }, "status <> 'terminated'");
|
|
394
512
|
setRow(ots.id, { status: 'pending' }, "status = 'creating'");
|
|
395
513
|
} catch (e) {
|
|
396
514
|
console.warn(`[7ots] vm create ${ots.id}:`, e.message);
|
|
397
|
-
if (
|
|
515
|
+
if (e.status === 402) billingSeen.delete(ots.accountId); // the card asks for the payment again
|
|
516
|
+
if (gone()) return void (pid && (await undo()));
|
|
398
517
|
if (vmRow(ots.id).status !== 'creating') return;
|
|
399
518
|
// Undo what exists (a timed-out create may still have made the VM): ids cleared only once it is gone.
|
|
400
519
|
let undone = !pid;
|
|
401
|
-
if (pid) undone = await destroy(pid).then(() => true, () => false);
|
|
402
|
-
setRow(ots.id, { status: 'error', error: clean(e
|
|
520
|
+
if (pid) undone = await destroy(pid, { cred }).then(() => true, () => false);
|
|
521
|
+
setRow(ots.id, { status: 'error', error: clean(createError(e, cred, t), 500), enroll_hash: null, enroll_exp: null, ...(undone ? { orq_project_id: '', orq_vm_id: '', orq_status: '' } : {}) }, "status = 'creating'");
|
|
403
522
|
}
|
|
404
523
|
}
|
|
405
524
|
|
|
406
525
|
/** Orquesta's view of the VM, at most every 15 s per ot (shared by concurrent readers). */
|
|
407
526
|
async function refreshHosted(otsId, { force = false } = {}) {
|
|
408
527
|
const row = vmRow(otsId);
|
|
409
|
-
if (!row || row.provider
|
|
528
|
+
if (!row || !HOSTED.includes(row.provider) || !row.orq_project_id) return row;
|
|
529
|
+
if (row.provider === 'orquesta' && !hostedConfigured()) return row;
|
|
410
530
|
if (!['pending', 'enrolled', 'error'].includes(row.status) || inflight.has(otsId)) return row;
|
|
411
531
|
if (!force && row.orq_checked_at && now() - row.orq_checked_at < ORQ_CACHE_MS) return row;
|
|
412
532
|
if (!checking.has(otsId)) {
|
|
413
533
|
const p = (async () => {
|
|
414
534
|
try {
|
|
415
|
-
const v = (await orq('GET', `/projects/${encodeURIComponent(row.orq_project_id)}/vm`, null, 8000)).vm;
|
|
535
|
+
const v = (await orq('GET', `/projects/${encodeURIComponent(row.orq_project_id)}/vm`, null, 8000, credOf(row))).vm;
|
|
416
536
|
if (!v) throw Object.assign(new OrqError('no vm'), { status: 404 });
|
|
417
537
|
const st = clean(v.status, 40);
|
|
418
538
|
const fields = { orq_status: st, ip: clean(v.ip, 60), orq_checked_at: now(), ...(v.id ? { orq_vm_id: String(v.id) } : {}) };
|
|
@@ -443,49 +563,97 @@ async function refreshHosted(otsId, { force = false } = {}) {
|
|
|
443
563
|
return vmRow(otsId);
|
|
444
564
|
}
|
|
445
565
|
|
|
446
|
-
/**
|
|
566
|
+
/**
|
|
567
|
+
* Compares Orquesta with ots_vms. 'orquesta' VMs: GET /api/v1/vms of 7ots' own org and of every child
|
|
568
|
+
* org; refreshes them, retries teardowns, adopts or flags lost creates and logs orphans.
|
|
569
|
+
* 'orquesta_user' VMs: refreshed one by one with each owner's own credential (never listed: their org
|
|
570
|
+
* is theirs). Then queued teardowns and child-org deletions are retried.
|
|
571
|
+
*/
|
|
447
572
|
export async function reconcileVms() {
|
|
448
|
-
if (!hostedConfigured()) return null;
|
|
449
|
-
const { vms = [] } = await orq('GET', '/vms', null, 30_000);
|
|
450
|
-
const prefix = REF_PREFIX();
|
|
451
|
-
const ours = vms.filter((v) => String(v.external_ref || '').startsWith(prefix));
|
|
452
|
-
const byRef = new Map(ours.map((v) => [String(v.external_ref).slice(prefix.length), v]));
|
|
453
573
|
const rows = getDb().prepare(`SELECT * FROM ots_vms WHERE ${LIVE_ORQ}`).all();
|
|
454
|
-
const report = { checked: rows.length, orquesta:
|
|
455
|
-
|
|
574
|
+
const report = { checked: rows.length, orquesta: 0, orgs: 0, user: 0, orphans: [], missing: [], retried: 0 };
|
|
575
|
+
const prefix = REF_PREFIX();
|
|
576
|
+
const orgOf = (r) => r.orq_org_id || '';
|
|
577
|
+
if (hostedConfigured()) {
|
|
578
|
+
const partnerRows = rows.filter((r) => r.provider === 'orquesta');
|
|
579
|
+
// '' = 7ots' own org (VMs made before child orgs existed).
|
|
580
|
+
const orgs = new Set(['', ...partnerRows.map(orgOf), ...getDb().prepare("SELECT orq_org_id FROM accounts WHERE orq_org_id <> ''").all().map((a) => a.orq_org_id)]);
|
|
581
|
+
const ours = [];
|
|
582
|
+
const failed = new Set();
|
|
583
|
+
for (const org of orgs) {
|
|
584
|
+
try {
|
|
585
|
+
const { vms = [] } = await orq('GET', '/vms', null, 30_000, { kind: 'partner', org });
|
|
586
|
+
for (const v of vms) if (String(v.external_ref || '').startsWith(prefix)) ours.push({ v, org, otsId: String(v.external_ref).slice(prefix.length) });
|
|
587
|
+
report.orgs++;
|
|
588
|
+
} catch (e) {
|
|
589
|
+
failed.add(org);
|
|
590
|
+
console.warn(`[7ots] vm reconcile: GET /vms${org ? ` (org ${org})` : ''}:`, e.message);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
if (failed.size === orgs.size) throw new OrqError('Orquesta did not list any VMs');
|
|
594
|
+
report.orquesta = ours.length;
|
|
595
|
+
for (const row of partnerRows) {
|
|
596
|
+
if (inflight.has(row.ots_id) || failed.has(orgOf(row))) continue;
|
|
597
|
+
const hit = ours.find((x) => x.org === orgOf(row) && (x.otsId === row.ots_id || (row.orq_project_id && String(x.v.project_id) === row.orq_project_id)));
|
|
598
|
+
const v = hit?.v;
|
|
599
|
+
if (row.status === 'terminating') {
|
|
600
|
+
report.retried++;
|
|
601
|
+
await finishTeardown(row.ots_id);
|
|
602
|
+
continue;
|
|
603
|
+
}
|
|
604
|
+
if (row.status === 'creating') {
|
|
605
|
+
// The server stopped mid-create: adopt what Orquesta made, or give up on it.
|
|
606
|
+
if (v) setRow(row.ots_id, { orq_project_id: String(v.project_id || row.orq_project_id), orq_vm_id: String(v.id || ''), orq_status: clean(v.status, 40), ip: clean(v.ip, 60), status: 'pending' }, "status = 'creating'");
|
|
607
|
+
else if (now() - row.updated_at > 15 * 60_000) setRow(row.ots_id, { status: 'error', error: 'Creation was interrupted' }, "status = 'creating'");
|
|
608
|
+
continue;
|
|
609
|
+
}
|
|
610
|
+
if (!v && row.orq_project_id) {
|
|
611
|
+
report.missing.push(row.ots_id);
|
|
612
|
+
setRow(row.ots_id, { orq_status: 'gone', orq_checked_at: now() });
|
|
613
|
+
if (!online(row)) setRow(row.ots_id, { status: 'error', error: 'The VM no longer exists in Orquesta' }, "status IN ('pending', 'enrolled')");
|
|
614
|
+
console.warn(`[7ots] vm reconcile: ${row.ots_id} has no VM in Orquesta`);
|
|
615
|
+
} else if (v) setRow(row.ots_id, { orq_status: clean(v.status, 40), ip: clean(v.ip, 60), orq_checked_at: now(), ...(v.id ? { orq_vm_id: String(v.id) } : {}) });
|
|
616
|
+
}
|
|
617
|
+
const live = new Set(partnerRows.map((r) => `${orgOf(r)}|${r.ots_id}`));
|
|
618
|
+
const queued = new Set(teardownQueue().map((x) => x.pid));
|
|
619
|
+
for (const { v, org, otsId } of ours) {
|
|
620
|
+
if (live.has(`${org}|${otsId}`) || queued.has(String(v.project_id))) continue;
|
|
621
|
+
if (/^(terminated|terminating|deleted|destroyed)$/i.test(String(v.status))) continue;
|
|
622
|
+
report.orphans.push({ otsId, vmId: v.id, projectId: v.project_id, status: v.status, ...(org ? { org } : {}) });
|
|
623
|
+
console.warn(`[7ots] vm reconcile: orphan VM ${v.id} (project ${v.project_id}, ref ${v.external_ref}${org ? `, org ${org}` : ''}) has no live ot`);
|
|
624
|
+
if (penv.VM_REAP_ORPHANS === '1') await destroy(String(v.project_id), { cred: { kind: 'partner', org } }).catch((e) => console.warn('[7ots] vm orphan teardown:', e.message));
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
// Owners' own VMs, each with its owner's credential.
|
|
628
|
+
for (const row of rows.filter((r) => r.provider === 'orquesta_user')) {
|
|
456
629
|
if (inflight.has(row.ots_id)) continue;
|
|
457
|
-
|
|
630
|
+
report.user++;
|
|
458
631
|
if (row.status === 'terminating') {
|
|
459
632
|
report.retried++;
|
|
460
633
|
await finishTeardown(row.ots_id);
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
// The server stopped mid-create: adopt what Orquesta made, or give up on it.
|
|
465
|
-
if (v) setRow(row.ots_id, { orq_project_id: String(v.project_id || row.orq_project_id), orq_vm_id: String(v.id || ''), orq_status: clean(v.status, 40), ip: clean(v.ip, 60), status: 'pending' }, "status = 'creating'");
|
|
466
|
-
else if (now() - row.updated_at > 15 * 60_000) setRow(row.ots_id, { status: 'error', error: 'Creation was interrupted' }, "status = 'creating'");
|
|
467
|
-
continue;
|
|
468
|
-
}
|
|
469
|
-
if (!v && row.orq_project_id) {
|
|
470
|
-
report.missing.push(row.ots_id);
|
|
471
|
-
setRow(row.ots_id, { orq_status: 'gone', orq_checked_at: now() });
|
|
472
|
-
if (!online(row)) setRow(row.ots_id, { status: 'error', error: 'The VM no longer exists in Orquesta' }, "status IN ('pending', 'enrolled')");
|
|
473
|
-
console.warn(`[7ots] vm reconcile: ${row.ots_id} has no VM in Orquesta`);
|
|
474
|
-
} else if (v) setRow(row.ots_id, { orq_status: clean(v.status, 40), ip: clean(v.ip, 60), orq_checked_at: now(), ...(v.id ? { orq_vm_id: String(v.id) } : {}) });
|
|
634
|
+
} else if (row.status === 'creating') {
|
|
635
|
+
if (now() - row.updated_at > 15 * 60_000) setRow(row.ots_id, { status: 'error', error: 'Creation was interrupted' }, "status = 'creating'");
|
|
636
|
+
} else await refreshHosted(row.ots_id, { force: true });
|
|
475
637
|
}
|
|
476
638
|
// Teardowns that failed when an ot was deleted.
|
|
477
639
|
for (const item of teardownQueue()) {
|
|
640
|
+
if (item.user && !getDb().prepare('SELECT 1 FROM accounts WHERE id = ?').get(item.accountId)) {
|
|
641
|
+
// The owner's account (and with it their credential) is gone: the project stays in THEIR org.
|
|
642
|
+
console.warn(`[7ots] vm teardown ${item.pid}: the owner's account is gone; it must be removed in Orquesta`);
|
|
643
|
+
dequeueTeardown(item.pid);
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
if (!item.user && !hostedConfigured()) continue;
|
|
478
647
|
report.retried++;
|
|
479
|
-
await destroy(item.pid).then(() => dequeueTeardown(item.pid), (e) => console.warn(`[7ots] vm teardown retry ${item.pid}:`, e.message));
|
|
648
|
+
await destroy(item.pid, { cred: credOfItem(item) }).then(() => dequeueTeardown(item.pid), (e) => console.warn(`[7ots] vm teardown retry ${item.pid}:`, e.message));
|
|
480
649
|
}
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
if (penv.VM_REAP_ORPHANS === '1') await destroy(String(v.project_id)).catch((e) => console.warn('[7ots] vm orphan teardown:', e.message));
|
|
650
|
+
// Child orgs let go of (account released) while a VM was still going away.
|
|
651
|
+
if (hostedConfigured()) {
|
|
652
|
+
for (const x of orgDeletes()) {
|
|
653
|
+
if (getDb().prepare(`SELECT 1 FROM ots_vms WHERE ${LIVE_PARTNER} AND orq_org_id = ?`).get(x.org)) continue;
|
|
654
|
+
report.retried++;
|
|
655
|
+
await dropChildOrg(x.accountId, x.org);
|
|
656
|
+
}
|
|
489
657
|
}
|
|
490
658
|
setPlatformState('vm_orphans', JSON.stringify({ at: now(), orphans: report.orphans, missing: report.missing }));
|
|
491
659
|
return report;
|
|
@@ -496,20 +664,28 @@ export async function releaseOtsVm(otsId) {
|
|
|
496
664
|
const row = vmRow(otsId);
|
|
497
665
|
if (!row) return;
|
|
498
666
|
getDb().prepare('UPDATE ots_vms SET agent_hash = NULL, enroll_hash = NULL, enroll_exp = NULL, updated_at = ? WHERE ots_id = ?').run(now(), otsId);
|
|
499
|
-
if (row.provider
|
|
667
|
+
if (!HOSTED.includes(row.provider) || row.status === 'terminated') return;
|
|
500
668
|
setRow(otsId, { status: 'terminating' });
|
|
501
669
|
if (inflight.has(otsId)) return; // provision() sees the row gone and undoes its own work
|
|
502
670
|
if (!row.orq_project_id) return;
|
|
503
671
|
try {
|
|
504
|
-
await destroy(row.orq_project_id, { retries: 0 });
|
|
672
|
+
await destroy(row.orq_project_id, { retries: 0, cred: credOf(row) });
|
|
505
673
|
} catch (e) {
|
|
506
674
|
console.warn(`[7ots] vm teardown on delete ${otsId}:`, e.message);
|
|
507
|
-
queueTeardown(
|
|
675
|
+
queueTeardown(teardownItem(row));
|
|
508
676
|
}
|
|
509
677
|
}
|
|
510
678
|
|
|
679
|
+
/**
|
|
680
|
+
* Before an account is deleted: every VM of it goes, then its Orquesta child org (Orquesta cancels the
|
|
681
|
+
* subscription first). While a VM is still going away Orquesta answers 409 and the deletion is queued
|
|
682
|
+
* for the reconciliation.
|
|
683
|
+
*/
|
|
511
684
|
export async function releaseAccountVms(accountId) {
|
|
512
685
|
for (const { ots_id } of getDb().prepare('SELECT ots_id FROM ots_vms WHERE account_id = ?').all(accountId)) await releaseOtsVm(ots_id);
|
|
686
|
+
await Promise.all(getDb().prepare('SELECT ots_id FROM ots_vms WHERE account_id = ?').all(accountId).map(({ ots_id }) => inflight.get(ots_id)?.catch(() => {})));
|
|
687
|
+
const org = accountOrg(accountId);
|
|
688
|
+
if (org && hostedConfigured()) await dropChildOrg(accountId, org);
|
|
513
689
|
}
|
|
514
690
|
|
|
515
691
|
/**
|
|
@@ -685,21 +861,78 @@ export function createVm({ send, readJson, rateLimit, base }) {
|
|
|
685
861
|
|
|
686
862
|
// ───────────── dashboard (/api/platform/ots/:id/vm…) ─────────────
|
|
687
863
|
|
|
688
|
-
/** What the dashboard needs to offer a
|
|
864
|
+
/** What the dashboard needs to offer a hosted server: 7ots' (via the account's child org) and/or one in the owner's own Orquesta org. */
|
|
689
865
|
function hostedInfo(account) {
|
|
690
866
|
const enabled = account.vm_enabled === 1;
|
|
691
867
|
const configured = hostedConfigured();
|
|
692
868
|
const used = getDb().prepare(`SELECT COUNT(*) AS n FROM ots_vms WHERE ${LIVE_ORQ} AND account_id = ?`).get(account.id).n;
|
|
693
|
-
const total = getDb().prepare(`SELECT COUNT(*) AS n FROM ots_vms WHERE ${
|
|
869
|
+
const total = getDb().prepare(`SELECT COUNT(*) AS n FROM ots_vms WHERE ${LIVE_PARTNER}`).get().n;
|
|
694
870
|
const reason = !enabled ? 'notEnabled' : !configured ? 'notConfigured' : used >= PER_ACCOUNT() ? 'accountLimit' : total >= MAX_TOTAL() ? 'full' : null;
|
|
695
|
-
|
|
871
|
+
const size = defaultSize();
|
|
872
|
+
const access = orquestaVmAccess(account.id);
|
|
873
|
+
const userReason = !access.usable ? access.reason : used >= PER_ACCOUNT() ? 'accountLimit' : null;
|
|
874
|
+
const org = accountOrg(account.id);
|
|
875
|
+
return {
|
|
876
|
+
enabled,
|
|
877
|
+
configured,
|
|
878
|
+
available: enabled && configured,
|
|
879
|
+
canCreate: !reason,
|
|
880
|
+
reason,
|
|
881
|
+
sizes: hostedSizes().filter((z) => z.id === size),
|
|
882
|
+
defaultSize: size,
|
|
883
|
+
monthlyPrice: hostedSizes().find((z) => z.id === size)?.monthlyPrice ?? null,
|
|
884
|
+
region: penv.ORQUESTA_VM_REGION || null,
|
|
885
|
+
used,
|
|
886
|
+
max: PER_ACCOUNT(),
|
|
887
|
+
operator: 'Orquesta',
|
|
888
|
+
// 7ots server: Orquesta bills the account's child org; null = never paid (or not known yet).
|
|
889
|
+
billing: org ? (billingSeen.get(account.id)?.status ?? null) : null,
|
|
890
|
+
// The owner's own Orquesta account (path A): needs an oak_ key with projects:write + vms:write.
|
|
891
|
+
user: { connected: access.connected, canCreate: !userReason, reason: userReason },
|
|
892
|
+
defaultOption: access.connected ? 'user' : '7ots',
|
|
893
|
+
};
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
/** Same quotas for both providers, checked inside the reserving transaction (and once before any Orquesta call). */
|
|
897
|
+
function quotaError(d, t, account, ots, partner) {
|
|
898
|
+
const cur = d.prepare('SELECT * FROM ots_vms WHERE ots_id = ?').get(ots.id);
|
|
899
|
+
if (cur && cur.status !== 'terminated' && (HOSTED.includes(cur.provider) || cur.status === 'enrolled')) return [409, t('platform.vm.exists')];
|
|
900
|
+
if (inflight.has(ots.id)) return [409, t('platform.vm.exists')];
|
|
901
|
+
if (d.prepare(`SELECT COUNT(*) AS n FROM ots_vms WHERE ${LIVE_ORQ} AND account_id = ?`).get(account.id).n >= PER_ACCOUNT()) return [403, t('platform.vm.accountLimit', { max: PER_ACCOUNT() })];
|
|
902
|
+
if (partner && d.prepare(`SELECT COUNT(*) AS n FROM ots_vms WHERE ${LIVE_PARTNER}`).get().n >= MAX_TOTAL()) return [503, t('platform.vm.full')];
|
|
903
|
+
return null;
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
/** Billing of the account's child org; an org deleted on Orquesta's side is forgotten and made again. */
|
|
907
|
+
async function billingOf(account) {
|
|
908
|
+
let org = await ensureChildOrg(account.id);
|
|
909
|
+
try {
|
|
910
|
+
return { org, status: await childBilling(account.id, org) };
|
|
911
|
+
} catch (e) {
|
|
912
|
+
if (e.status !== 404) throw e;
|
|
913
|
+
getDb().prepare("UPDATE accounts SET orq_org_id = '' WHERE id = ? AND orq_org_id = ?").run(account.id, org);
|
|
914
|
+
org = await ensureChildOrg(account.id);
|
|
915
|
+
return { org, status: await childBilling(account.id, org) };
|
|
916
|
+
}
|
|
696
917
|
}
|
|
697
918
|
|
|
698
|
-
/**
|
|
699
|
-
|
|
919
|
+
/**
|
|
920
|
+
* POST /vm {provider:'orquesta'|'orquesta_user'}: reserves the slot (quotas, atomically) and provisions in the background.
|
|
921
|
+
* 'orquesta' first makes sure the account's child org exists and is paid: if not, { needsPayment, checkoutUrl }.
|
|
922
|
+
*/
|
|
923
|
+
async function createHosted(req, res, account, ots, provider) {
|
|
700
924
|
const t = reqT(req);
|
|
701
|
-
|
|
702
|
-
if (
|
|
925
|
+
const user = provider === 'orquesta_user';
|
|
926
|
+
if (user) {
|
|
927
|
+
const a = orquestaVmAccess(account.id);
|
|
928
|
+
if (!a.usable) {
|
|
929
|
+
const key = { needsKey: 'platform.vm.userNeedsKey', reconnect: 'platform.orquesta.reconnect' }[a.reason] || 'platform.orquesta.notConnected';
|
|
930
|
+
return send(res, 409, { error: t(key), code: `orquesta_${a.reason}` });
|
|
931
|
+
}
|
|
932
|
+
} else {
|
|
933
|
+
if (account.vm_enabled !== 1) return send(res, 403, { error: t('platform.vm.paidOff') });
|
|
934
|
+
if (!hostedConfigured()) return send(res, 503, { error: t('platform.vm.paidUnavailable') });
|
|
935
|
+
}
|
|
703
936
|
const size = defaultSize();
|
|
704
937
|
let server;
|
|
705
938
|
try {
|
|
@@ -709,24 +942,46 @@ export function createVm({ send, readJson, rateLimit, base }) {
|
|
|
709
942
|
console.warn('[7ots] vm create:', e.message);
|
|
710
943
|
return send(res, 503, { error: t('platform.vm.paidUnavailable') });
|
|
711
944
|
}
|
|
945
|
+
const pre = quotaError(getDb(), t, account, ots, !user);
|
|
946
|
+
if (pre) return send(res, pre[0], { error: pre[1] });
|
|
947
|
+
const price = hostedSizes().find((s) => s.id === size).monthlyPrice;
|
|
948
|
+
let cred = { kind: 'user', accountId: account.id };
|
|
949
|
+
if (!user) {
|
|
950
|
+
try {
|
|
951
|
+
const { org, status } = await billingOf(account);
|
|
952
|
+
if (status !== 'active') {
|
|
953
|
+
let ck = null;
|
|
954
|
+
try {
|
|
955
|
+
// No #fragment: Orquesta appends ?orq_checkout=success|cancel; the dashboard opens the ot from vm_ots.
|
|
956
|
+
ck = await orq('POST', `/partner/orgs/${encodeURIComponent(org)}/checkout`, { return_url: `${server}/app/?vm_ots=${encodeURIComponent(ots.id)}`, size });
|
|
957
|
+
} catch (e) {
|
|
958
|
+
if (e.status !== 409) throw e; // ALREADY_ACTIVE (paid meanwhile) / PARTNER_BILLED: nothing to pay
|
|
959
|
+
}
|
|
960
|
+
if (ck) {
|
|
961
|
+
if (!/^https:\/\//.test(String(ck.url || ''))) throw new OrqError('Orquesta returned no checkout URL');
|
|
962
|
+
return send(res, 200, { needsPayment: true, checkoutUrl: ck.url, size, monthlyPrice: price, operator: 'Orquesta', hosted: hostedInfo(account) });
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
cred = { kind: 'partner', org };
|
|
966
|
+
} catch (e) {
|
|
967
|
+
console.warn('[7ots] vm billing:', e.message);
|
|
968
|
+
return send(res, 503, { error: t('platform.vm.paidUnavailable') });
|
|
969
|
+
}
|
|
970
|
+
}
|
|
712
971
|
const region = penv.ORQUESTA_VM_REGION || '';
|
|
713
972
|
const code = `7ve_${token()}`;
|
|
714
973
|
const tm = now();
|
|
715
|
-
const price = hostedSizes().find((s) => s.id === size).monthlyPrice;
|
|
716
974
|
const err = tx((d) => {
|
|
717
|
-
const
|
|
718
|
-
if (
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
if (d.prepare(`SELECT COUNT(*) AS n FROM ots_vms WHERE ${LIVE_ORQ}`).get().n >= MAX_TOTAL()) return [503, t('platform.vm.full')];
|
|
722
|
-
d.prepare(`INSERT INTO ots_vms (ots_id, account_id, provider, status, size, region, enroll_hash, enroll_exp, price_cents, created_at, updated_at) VALUES (?, ?, 'orquesta', 'creating', ?, ?, ?, ?, ?, ?, ?)
|
|
723
|
-
ON CONFLICT(ots_id) DO UPDATE SET account_id = excluded.account_id, provider = 'orquesta', status = 'creating', orq_project_id = '', orq_vm_id = '', orq_status = '', orq_checked_at = NULL, size = excluded.size, region = excluded.region,
|
|
975
|
+
const q = quotaError(d, t, account, ots, !user);
|
|
976
|
+
if (q) return q;
|
|
977
|
+
d.prepare(`INSERT INTO ots_vms (ots_id, account_id, provider, orq_org_id, status, size, region, enroll_hash, enroll_exp, price_cents, created_at, updated_at) VALUES (?, ?, ?, ?, 'creating', ?, ?, ?, ?, ?, ?, ?)
|
|
978
|
+
ON CONFLICT(ots_id) DO UPDATE SET account_id = excluded.account_id, provider = excluded.provider, orq_org_id = excluded.orq_org_id, status = 'creating', orq_project_id = '', orq_vm_id = '', orq_status = '', orq_checked_at = NULL, size = excluded.size, region = excluded.region,
|
|
724
979
|
ip = '', error = '', enroll_hash = excluded.enroll_hash, enroll_exp = excluded.enroll_exp, agent_hash = NULL, agent_seen_at = NULL, worker_version = '', worker_info = '{}', price_cents = excluded.price_cents, created_at = excluded.created_at, updated_at = excluded.updated_at`)
|
|
725
|
-
.run(ots.id, account.id, size, region, sha256(code), tm + ORQ_ENROLL_TTL(), price ? Math.round(price * 100) : null, tm, tm);
|
|
980
|
+
.run(ots.id, account.id, user ? 'orquesta_user' : 'orquesta', cred.org || '', size, region, sha256(code), tm + ORQ_ENROLL_TTL(), price ? Math.round(price * 100) : null, tm, tm);
|
|
726
981
|
return null;
|
|
727
982
|
});
|
|
728
983
|
if (err) return send(res, err[0], { error: err[1] });
|
|
729
|
-
const job = provision(ots, { server, code, size, region }).finally(() => inflight.delete(ots.id));
|
|
984
|
+
const job = provision(ots, { server, code, size, region, cred, t }).finally(() => inflight.delete(ots.id));
|
|
730
985
|
inflight.set(ots.id, job);
|
|
731
986
|
return send(res, 202, { vm: vmView(vmRow(ots.id)), hosted: hostedInfo(account) });
|
|
732
987
|
}
|
|
@@ -738,25 +993,49 @@ export function createVm({ send, readJson, rateLimit, base }) {
|
|
|
738
993
|
let vm = vmRow(ots.id);
|
|
739
994
|
if (sub === '/vm' && req.method === 'GET') {
|
|
740
995
|
reap(ots.id);
|
|
741
|
-
if (vm?.provider
|
|
996
|
+
if (HOSTED.includes(vm?.provider)) vm = await refreshHosted(ots.id);
|
|
997
|
+
// What the card says about payment: asked to Orquesta at most every 5 min per account.
|
|
998
|
+
const org = accountOrg(account.id);
|
|
999
|
+
const seenAt = billingSeen.get(account.id)?.at || 0;
|
|
1000
|
+
if (org && hostedConfigured() && now() - seenAt > 5 * 60_000) await childBilling(account.id, org).catch((e) => console.warn('[7ots] vm billing:', e.message));
|
|
742
1001
|
return send(res, 200, { vm: vmView(vm), jobs: recentJobs(ots.id), kinds: otsVmKinds(ots), allKinds: VM_KINDS, daily: { used: jobsLastDay(ots.id), max: DAILY() }, paidVms: account.vm_enabled === 1, hosted: hostedInfo(account) }), true;
|
|
743
1002
|
}
|
|
1003
|
+
if (sub === '/vm/billing' && req.method === 'GET') {
|
|
1004
|
+
// Polled by the dashboard after Orquesta's checkout, until the webhook marks the subscription active.
|
|
1005
|
+
const org = accountOrg(account.id);
|
|
1006
|
+
if (!org || !hostedConfigured()) return send(res, 200, { status: null, active: false }), true;
|
|
1007
|
+
try {
|
|
1008
|
+
const status = await childBilling(account.id, org);
|
|
1009
|
+
return send(res, 200, { status, active: status === 'active' }), true;
|
|
1010
|
+
} catch (e) {
|
|
1011
|
+
console.warn('[7ots] vm billing:', e.message);
|
|
1012
|
+
return send(res, 503, { error: t('platform.vm.paidUnavailable') }), true;
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
if (sub === '/vm/subscription' && req.method === 'DELETE') {
|
|
1016
|
+
// Ends what the account pays for 7ots servers: only with none left (deleting the org cancels the subscription).
|
|
1017
|
+
if (getDb().prepare(`SELECT 1 FROM ots_vms WHERE ${LIVE_PARTNER} AND account_id = ?`).get(account.id)) return send(res, 409, { error: t('platform.vm.subscriptionInUse') }), true;
|
|
1018
|
+
const org = accountOrg(account.id);
|
|
1019
|
+
if (!org) return send(res, 200, { ok: true }), true;
|
|
1020
|
+
const done = await dropChildOrg(account.id, org);
|
|
1021
|
+
return send(res, done ? 200 : 202, { ok: true, queued: !done, hosted: hostedInfo(account) }), true;
|
|
1022
|
+
}
|
|
744
1023
|
if ((sub === '/vm' || sub === '/vm/enroll') && req.method === 'POST') {
|
|
745
1024
|
const body = await readJson(req, 4096);
|
|
746
1025
|
const provider = body.provider || (sub === '/vm' ? 'orquesta' : 'byo');
|
|
747
|
-
if (provider === '
|
|
1026
|
+
if (HOSTED.includes(provider) && sub === '/vm') return await createHosted(req, res, account, ots, provider), true;
|
|
748
1027
|
if (provider !== 'byo' || sub !== '/vm/enroll') return send(res, 400, { error: t('server.http.badRequest') }), true;
|
|
749
|
-
// A
|
|
750
|
-
if (vm?.provider
|
|
1028
|
+
// A hosted server must be deleted first: re-enrolling would leave its VM running unattached.
|
|
1029
|
+
if (HOSTED.includes(vm?.provider) && vm.status !== 'terminated') return send(res, 409, { error: t('platform.vm.exists') }), true;
|
|
751
1030
|
const code = `7ve_${token()}`;
|
|
752
1031
|
const tm = now();
|
|
753
|
-
if (vm) getDb().prepare("UPDATE ots_vms SET enroll_hash = ?, enroll_exp = ?, provider = 'byo', status = CASE WHEN status = 'terminated' THEN 'pending' ELSE status END, updated_at = ? WHERE ots_id = ?").run(sha256(code), tm + ENROLL_TTL, tm, ots.id);
|
|
1032
|
+
if (vm) getDb().prepare("UPDATE ots_vms SET enroll_hash = ?, enroll_exp = ?, provider = 'byo', orq_org_id = '', status = CASE WHEN status = 'terminated' THEN 'pending' ELSE status END, updated_at = ? WHERE ots_id = ?").run(sha256(code), tm + ENROLL_TTL, tm, ots.id);
|
|
754
1033
|
else getDb().prepare("INSERT INTO ots_vms (ots_id, account_id, provider, status, enroll_hash, enroll_exp, created_at, updated_at) VALUES (?, ?, 'byo', 'pending', ?, ?, ?, ?)").run(ots.id, account.id, sha256(code), tm + ENROLL_TTL, tm, tm);
|
|
755
1034
|
return send(res, 200, { code, expiresIn: ENROLL_TTL / 1000, command: `npx -y @7ots/cli vm enroll --server ${base(req)} --code ${code} --systemd` }), true;
|
|
756
1035
|
}
|
|
757
1036
|
if (sub === '/vm' && req.method === 'DELETE') {
|
|
758
1037
|
if (!vm) return send(res, 200, { ok: true }), true;
|
|
759
|
-
const hosted = vm.provider
|
|
1038
|
+
const hosted = HOSTED.includes(vm.provider) && vm.status !== 'terminated';
|
|
760
1039
|
getDb().prepare(`UPDATE ots_vms SET agent_hash = NULL, enroll_hash = NULL, enroll_exp = NULL, status = '${hosted ? 'terminating' : 'terminated'}', updated_at = ? WHERE ots_id = ?`).run(now(), ots.id);
|
|
761
1040
|
for (const j of getDb().prepare("SELECT * FROM vm_jobs WHERE ots_id = ? AND status NOT IN ('completed', 'failed', 'cancelled')").all(ots.id)) setStatus(j, 'cancelled', { error: 'worker revoked' });
|
|
762
1041
|
if (!hosted) return send(res, 200, { ok: true }), true;
|
|
@@ -766,8 +1045,8 @@ export function createVm({ send, readJson, rateLimit, base }) {
|
|
|
766
1045
|
return send(res, 404, { error: t('server.http.unknownRoute') }), true;
|
|
767
1046
|
}
|
|
768
1047
|
|
|
769
|
-
// Reconciliation with Orquesta (every VM_RECONCILE_MIN, default 30)
|
|
770
|
-
if (
|
|
1048
|
+
// Reconciliation with Orquesta (every VM_RECONCILE_MIN, default 30): 7ots' VMs when ORQUESTA_VM_TOKEN is set, owners' own VMs always.
|
|
1049
|
+
if (!createVm.timer) {
|
|
771
1050
|
const run = () => reconcileVms().catch((e) => console.warn('[7ots] vm reconcile:', e.message));
|
|
772
1051
|
createVm.timer = setInterval(run, RECONCILE_MS());
|
|
773
1052
|
createVm.timer.unref?.();
|