@7ots/cli 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/cli/7ots.mjs +13 -1
- package/cli/lib/account.mjs +152 -2
- package/cli/lib/assistant.mjs +107 -38
- package/cli/lib/brain.mjs +13 -2
- package/cli/lib/config.mjs +5 -1
- package/cli/lib/inferences.mjs +30 -6
- package/cli/lib/integrations.mjs +69 -5
- package/cli/lib/orquesta.mjs +44 -1
- package/cli/lib/pet-server.mjs +352 -11
- package/cli/lib/prompts.mjs +31 -17
- package/cli/lib/prowl.mjs +110 -0
- package/cli/pet/electron/director.cjs +37 -5
- package/cli/pet/electron/main.cjs +92 -10
- package/cli/pet/pet.html +149 -10
- package/cli/pet/settings.html +549 -43
- package/dist/7ots.esm.js +216 -137
- package/dist/7ots.esm.js.map +4 -4
- package/dist/7ots.iife.js +216 -137
- package/dist/7ots.iife.js.map +4 -4
- package/llms.txt +1 -1
- package/package.json +17 -14
- package/server/channels/agent.mjs +7 -1
- package/server/channels/index.mjs +3 -2
- package/server/platform/connect.mjs +4 -1
- package/server/platform/db.mjs +14 -1
- package/server/platform/device.mjs +101 -4
- package/server/platform/mods.mjs +222 -0
- package/server/platform/orquesta.mjs +78 -14
- package/server/platform/profile.mjs +161 -0
- package/server/platform/routes.mjs +39 -33
- package/server/platform/runtime.mjs +5 -3
- package/server/platform/store.mjs +1 -1
- package/server/platform/vm.mjs +688 -17
- package/server/server.mjs +12 -2
- package/server/tts.mjs +17 -4
- package/src/avatar/three.js +3 -2
- package/src/character/AvatarEditor.js +95 -7
- package/src/character/Character.js +116 -17
- package/src/character/TradingCard.js +226 -0
- package/src/character/mods/index.js +4 -0
- package/src/character/mods/orc.js +64 -0
- package/src/character/mods.js +251 -0
- package/src/character/more/acc-head.js +213 -0
- package/src/character/more/acc-rest.js +342 -0
- package/src/character/more/brows.js +39 -0
- package/src/character/more/cheeks.js +78 -0
- package/src/character/more/eyes.js +288 -0
- package/src/character/more/mouths.js +259 -0
- package/src/character/more/patterns.js +120 -0
- package/src/character/parts.js +100 -0
- package/src/character/skin.js +166 -1
- package/src/character/styles.js +1285 -9
- package/src/character3d/Character3D.js +253 -37
- package/src/character3d/index.js +7 -1
- package/src/character3d/model.js +367 -0
- package/src/i18n/messages/character.js +37 -22
- package/src/i18n/messages/platform.js +33 -0
- package/src/identity/random.js +17 -3
- package/src/identity/schema.js +10 -0
- package/src/index.js +1 -0
package/server/platform/vm.mjs
CHANGED
|
@@ -1,15 +1,36 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The machine where an ot's assistant worker runs, and the desktop → VM offload channel (docs/VM-PLAN.md).
|
|
3
3
|
*
|
|
4
|
-
*
|
|
4
|
+
* Provider 'byo': the owner brings their own Linux box: the dashboard hands out a one-time
|
|
5
5
|
* `7ve_` code and the exact command; `7ots vm enroll` trades it for a `7vm_` token bound to that one
|
|
6
|
-
* ot and `7ots vm worker` connects outbound only (no SSH keys ever reach 7ots).
|
|
7
|
-
*
|
|
6
|
+
* ot and `7ots vm worker` connects outbound only (no SSH keys ever reach 7ots).
|
|
7
|
+
* Provider 'orquesta' ("7ots server, operated by Orquesta"): gated by accounts.vm_enabled and
|
|
8
|
+
* ORQUESTA_VM_TOKEN (7ots' partner oak_ key: orgs:write + projects:write + vms:write, never shown to
|
|
9
|
+
* anyone). Each 7ots account gets its own Orquesta child org (POST /api/v1/partner/orgs, external_ref
|
|
10
|
+
* `<prefix>acct:<accountId>`, kept in accounts.orq_org_id) and every project/VM call for it carries
|
|
11
|
+
* `X-Orquesta-Org: <child>`. Orquesta bills the child through its own Stripe Checkout: until the child's
|
|
12
|
+
* billing is active, POST /vm answers { needsPayment, checkoutUrl } (back to /app/?vm_ots=<id>).
|
|
13
|
+
* Provider 'orquesta_user': the owner connected their own Orquesta account (orquesta.mjs) with an oak_
|
|
14
|
+
* key that has projects:write + vms:write; the VM is created in THEIR org, with their key (which never
|
|
15
|
+
* leaves orquesta.mjs) and billed to them by Orquesta. No child org, no 7ots checkout.
|
|
16
|
+
* Both create project `ot-<id>` + its VM with a bootstrap that enrolls it the same way (code valid
|
|
17
|
+
* VM_ORQ_ENROLL_MIN, default 60, to cover the boot). Quotas VM_PER_ACCOUNT (default 1, both providers)
|
|
18
|
+
* and VM_MAX_TOTAL (default 10, 'orquesta' only). Deleting the VM, the ot or the account tears down VM
|
|
19
|
+
* and project with the credential that made them; releasing the account also deletes its child org
|
|
20
|
+
* (which cancels its Orquesta subscription). A reconciliation every VM_RECONCILE_MIN (30) compares
|
|
21
|
+
* GET /api/v1/vms of each child org with ots_vms, refreshes 'orquesta_user' VMs one by one, retries
|
|
22
|
+
* failed teardowns and org deletions and logs orphans (VM_REAP_ORPHANS=1 deletes them).
|
|
23
|
+
* Env: ORQUESTA_URL, ORQUESTA_VM_SIZE (default: cheapest), ORQUESTA_VM_SIZES ('slug[:usd],…'), ORQUESTA_VM_REGION,
|
|
24
|
+
* ORQUESTA_VM_REF_PREFIX ('7ots:', distinct per 7ots install sharing one org), SEVENOTS_VM_CLI (pinned, 0.1.6).
|
|
8
25
|
*
|
|
9
26
|
* Dashboard (session + X-7ots-Admin, via routes.mjs):
|
|
10
|
-
* GET /api/platform/ots/:id/vm { vm|null,
|
|
27
|
+
* GET /api/platform/ots/:id/vm { vm|null, jobs, kinds, daily, paidVms, hosted: { available, canCreate, reason, sizes, billing, user, defaultOption, … } }
|
|
28
|
+
* POST /api/platform/ots/:id/vm { provider: 'orquesta'|'orquesta_user', size? } → 202 { vm } (provisioning in the background)
|
|
29
|
+
* · 'orquesta' unpaid → 200 { needsPayment: true, checkoutUrl, monthlyPrice }
|
|
30
|
+
* GET /api/platform/ots/:id/vm/billing { status, active } of the account's child org (polled after the checkout)
|
|
31
|
+
* DELETE /api/platform/ots/:id/vm/subscription no 7ots server left → deletes the child org (ends the subscription)
|
|
11
32
|
* POST /api/platform/ots/:id/vm/enroll { provider?: 'byo' } → { code, expiresIn, command }
|
|
12
|
-
* DELETE /api/platform/ots/:id/vm revokes the worker,
|
|
33
|
+
* DELETE /api/platform/ots/:id/vm revokes the worker; byo → terminated, orquesta → 202 terminating → terminated
|
|
13
34
|
*
|
|
14
35
|
* Worker on the VM (Authorization: Bearer 7vm_…):
|
|
15
36
|
* POST /api/vm/enroll { code } → { token, ots } (no auth, rate-limited, code used once)
|
|
@@ -37,7 +58,8 @@ import { identityPrompt } from '../identity.mjs';
|
|
|
37
58
|
import { errorText, reqT } from '../i18n.mjs';
|
|
38
59
|
import { getDb, tx } from './db.mjs';
|
|
39
60
|
import { newId, sha256, token } from './crypto.mjs';
|
|
40
|
-
import { getOts, getOwnOts } from './store.mjs';
|
|
61
|
+
import { getOts, getOwnOts, getPlatformState, setPlatformState } from './store.mjs';
|
|
62
|
+
import { orquestaInfra, orquestaVmAccess } from './orquesta.mjs';
|
|
41
63
|
import { otsStep } from './runtime.mjs';
|
|
42
64
|
|
|
43
65
|
const penv = process.env;
|
|
@@ -113,6 +135,7 @@ function vmView(vm) {
|
|
|
113
135
|
enrollExpiresAt: vm.enroll_hash ? vm.enroll_exp : null,
|
|
114
136
|
error: vm.error || null,
|
|
115
137
|
since: vm.created_at,
|
|
138
|
+
...(HOSTED.includes(vm.provider) ? { orqStatus: vm.orq_status || null, ip: vm.ip || null, monthlyPrice: vm.price_cents ? vm.price_cents / 100 : null, checkedAt: vm.orq_checked_at || null } : {}),
|
|
116
139
|
};
|
|
117
140
|
}
|
|
118
141
|
|
|
@@ -180,6 +203,491 @@ function workerJob(j, ots) {
|
|
|
180
203
|
return { id: j.id, kind: j.kind, input: j.input, context: j.context, prompt, createdAt: j.created_at };
|
|
181
204
|
}
|
|
182
205
|
|
|
206
|
+
// ───────────── VMs paid by 7ots, created through Orquesta (provider 'orquesta') ─────────────
|
|
207
|
+
//
|
|
208
|
+
// One Orquesta project `ot-<otsId>` per VM (Orquesta allows one VM per project), created with the
|
|
209
|
+
// org-wide ORQUESTA_VM_TOKEN; the VM boots with a bootstrap that enrolls it with a one-time 7ve_
|
|
210
|
+
// code, exactly as an owner's own server would. ots_vms.status: creating (Orquesta calls in flight)
|
|
211
|
+
// → pending (VM booting, waiting for the worker) → enrolled · error · terminating → terminated.
|
|
212
|
+
// orq_status mirrors Orquesta's own VM status (checked at most every 15 s).
|
|
213
|
+
|
|
214
|
+
const ORQ = () => (penv.ORQUESTA_URL || 'https://getorquesta.com').replace(/\/+$/, '');
|
|
215
|
+
const orqToken = () => String(penv.ORQUESTA_VM_TOKEN || '').trim();
|
|
216
|
+
export const hostedConfigured = () => !!orqToken();
|
|
217
|
+
const REF_PREFIX = () => penv.ORQUESTA_VM_REF_PREFIX || '7ots:';
|
|
218
|
+
const ORQ_ENROLL_TTL = () => Math.max(10, Number(penv.VM_ORQ_ENROLL_MIN || 60)) * 60_000; // covers the boot
|
|
219
|
+
const ORQ_CACHE_MS = 15_000;
|
|
220
|
+
const MAX_TOTAL = () => Math.max(0, Number(penv.VM_MAX_TOTAL ?? 10));
|
|
221
|
+
const PER_ACCOUNT = () => Math.max(0, Number(penv.VM_PER_ACCOUNT ?? 1));
|
|
222
|
+
const RECONCILE_MS = () => Math.max(0.01, Number(penv.VM_RECONCILE_MIN || 30)) * 60_000;
|
|
223
|
+
const PROJECT_RETRY_MS = () => Number(penv.VM_ORQ_RETRY_MS || 10_000);
|
|
224
|
+
const HOSTED = ['orquesta', 'orquesta_user'];
|
|
225
|
+
const LIVE_ORQ = "provider IN ('orquesta', 'orquesta_user') AND status <> 'terminated'"; // per-account quota
|
|
226
|
+
const LIVE_PARTNER = "provider = 'orquesta' AND status <> 'terminated'"; // 7ots' own fleet (VM_MAX_TOTAL)
|
|
227
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
228
|
+
|
|
229
|
+
/** The CLI the VM installs, pinned (SEVENOTS_VM_CLI: '0.1.6' or '@7ots/cli@0.1.6'). */
|
|
230
|
+
function cliSpec() {
|
|
231
|
+
const v = String(penv.SEVENOTS_VM_CLI || '0.1.7').trim();
|
|
232
|
+
const s = v.startsWith('@7ots/cli@') ? v : `@7ots/cli@${v}`;
|
|
233
|
+
if (!/^@7ots\/cli@\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/.test(s)) throw new Error('SEVENOTS_VM_CLI must be an exact version');
|
|
234
|
+
return s;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** Sizes known (ORQUESTA_VM_SIZES 'slug[:usd/month],…'; prices from Orquesta's AWS catalogue). */
|
|
238
|
+
function hostedSizes() {
|
|
239
|
+
const list = String(penv.ORQUESTA_VM_SIZES || 't3.micro:17,t3.small:28,t3.medium:49')
|
|
240
|
+
.split(/[\s,]+/)
|
|
241
|
+
.map((x) => x.split(':'))
|
|
242
|
+
.filter(([id]) => /^[a-z0-9][a-z0-9.-]{1,40}$/.test(id))
|
|
243
|
+
.map(([id, p]) => ({ id, monthlyPrice: Number(p) > 0 ? Number(p) : null }));
|
|
244
|
+
return list.length ? list : [{ id: 't3.small', monthlyPrice: null }];
|
|
245
|
+
}
|
|
246
|
+
/** Always the cheapest size (ORQUESTA_VM_SIZE overrides only if listed). */
|
|
247
|
+
function defaultSize() {
|
|
248
|
+
const all = hostedSizes();
|
|
249
|
+
const forced = all.find((s) => s.id === penv.ORQUESTA_VM_SIZE);
|
|
250
|
+
if (forced) return forced.id;
|
|
251
|
+
return all.reduce((a, b) => ((b.monthlyPrice ?? Infinity) < (a.monthlyPrice ?? Infinity) ? b : a)).id;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** Monthly price in cents from whatever Orquesta returns (its catalogue counts cents). */
|
|
255
|
+
function priceCents(v) {
|
|
256
|
+
const c = v?.priceMonthly ?? v?.price_monthly ?? v?.size?.priceMonthly ?? v?.size?.price_monthly;
|
|
257
|
+
if (Number(c) > 0) return Math.round(Number(c));
|
|
258
|
+
const usd = v?.monthly_price_usd ?? v?.monthlyPriceUsd ?? v?.monthly_price ?? v?.monthlyPrice;
|
|
259
|
+
return Number(usd) > 0 ? Math.round(Number(usd) * 100) : null;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
class OrqError extends Error {}
|
|
263
|
+
|
|
264
|
+
const orqFail = (method, path, status, data) => {
|
|
265
|
+
const msg = typeof data.error === 'string' ? data.error : data.error?.message || data.message || '';
|
|
266
|
+
return Object.assign(new OrqError(`Orquesta ${method} ${path.replace(/\/[^/]{20,}/g, '/…')}: HTTP ${status}${msg ? ` ${clean(msg, 200)}` : ''}`), { status, code: data.error?.code || data.code });
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Orquesta's /api/v1 with a credential:
|
|
271
|
+
* { kind: 'partner', org } → ORQUESTA_VM_TOKEN, plus X-Orquesta-Org when org is a child ('' = 7ots' own org);
|
|
272
|
+
* { kind: 'user', accountId } → the owner's own key, through orquesta.mjs (it never reaches this module).
|
|
273
|
+
* Without cred: the partner key on its own org (the /partner/orgs routes refuse X-Orquesta-Org).
|
|
274
|
+
*/
|
|
275
|
+
async function orq(method, path, body, timeoutMs = 20_000, cred = null) {
|
|
276
|
+
if (cred?.kind === 'user') {
|
|
277
|
+
let r;
|
|
278
|
+
try {
|
|
279
|
+
r = await orquestaInfra(cred.accountId, method, `/api/v1${path}`, body, timeoutMs);
|
|
280
|
+
} catch (e) {
|
|
281
|
+
// Not connected / reconnect / unreachable: never a 404 or 409 of the resource itself.
|
|
282
|
+
throw Object.assign(new OrqError(`Orquesta (owner's account): ${clean(e.message, 200)}`), { status: 0, code: 'credentials', i18n: e.i18n });
|
|
283
|
+
}
|
|
284
|
+
if (!r.ok) throw orqFail(method, path, r.status, r.data || {});
|
|
285
|
+
return r.status === 204 ? {} : r.data || {};
|
|
286
|
+
}
|
|
287
|
+
let res;
|
|
288
|
+
try {
|
|
289
|
+
res = await fetch(`${ORQ()}/api/v1${path}`, {
|
|
290
|
+
method,
|
|
291
|
+
headers: {
|
|
292
|
+
Accept: 'application/json',
|
|
293
|
+
Authorization: `Bearer ${orqToken()}`,
|
|
294
|
+
...(cred?.org ? { 'X-Orquesta-Org': cred.org } : {}),
|
|
295
|
+
...(body ? { 'Content-Type': 'application/json' } : {}),
|
|
296
|
+
},
|
|
297
|
+
body: body ? JSON.stringify(body) : undefined,
|
|
298
|
+
redirect: 'manual',
|
|
299
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
300
|
+
});
|
|
301
|
+
} catch (e) {
|
|
302
|
+
throw Object.assign(new OrqError(`Orquesta unreachable (${e.name === 'TimeoutError' ? 'timeout' : e.code || e.name})`), { status: 0 });
|
|
303
|
+
}
|
|
304
|
+
const data = res.status === 204 ? {} : await res.json().catch(() => ({}));
|
|
305
|
+
if (!res.ok) throw orqFail(method, path, res.status, data);
|
|
306
|
+
return data;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/** The credential that made (and unmakes) a row's VM. */
|
|
310
|
+
const credOf = (row) => (row.provider === 'orquesta_user' ? { kind: 'user', accountId: row.account_id } : { kind: 'partner', org: row.orq_org_id || '' });
|
|
311
|
+
const credOfItem = (x) => (x.user ? { kind: 'user', accountId: x.accountId } : { kind: 'partner', org: x.org || '' });
|
|
312
|
+
|
|
313
|
+
// ── the account's child org (provider 'orquesta') ──
|
|
314
|
+
|
|
315
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
316
|
+
const accountOrg = (accountId) => getDb().prepare('SELECT orq_org_id FROM accounts WHERE id = ?').get(accountId)?.orq_org_id || '';
|
|
317
|
+
const accountRef = (accountId) => `${REF_PREFIX()}acct:${accountId}`;
|
|
318
|
+
const ensuring = new Map(); // account id → Promise<org id>
|
|
319
|
+
|
|
320
|
+
/** The account's Orquesta child org, created the first time (idempotent on Orquesta's side by external_ref). */
|
|
321
|
+
function ensureChildOrg(accountId) {
|
|
322
|
+
const have = accountOrg(accountId);
|
|
323
|
+
if (have) return Promise.resolve(have);
|
|
324
|
+
if (!ensuring.has(accountId)) {
|
|
325
|
+
const p = (async () => {
|
|
326
|
+
const r = await orq('POST', '/partner/orgs', { external_ref: accountRef(accountId), name: `7ots account ${accountId}`.slice(0, 100) });
|
|
327
|
+
const id = String(r.org?.id || '');
|
|
328
|
+
if (!UUID_RE.test(id)) throw new OrqError('Orquesta returned no organization id');
|
|
329
|
+
getDb().prepare("UPDATE accounts SET orq_org_id = ? WHERE id = ? AND orq_org_id = ''").run(id, accountId);
|
|
330
|
+
return accountOrg(accountId) || id;
|
|
331
|
+
})().finally(() => ensuring.delete(accountId));
|
|
332
|
+
ensuring.set(accountId, p);
|
|
333
|
+
}
|
|
334
|
+
return ensuring.get(accountId);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const billingSeen = new Map(); // account id → { status, at } (last answer from Orquesta, for the card)
|
|
338
|
+
|
|
339
|
+
/** Billing status of the account's child org: 'active' | 'past_due' | 'canceled' | null (never paid). */
|
|
340
|
+
async function childBilling(accountId, org) {
|
|
341
|
+
const r = await orq('GET', `/partner/orgs/${encodeURIComponent(org)}/billing`, null, 10_000);
|
|
342
|
+
const status = r.status ? clean(r.status, 20) : null;
|
|
343
|
+
billingSeen.set(accountId, { status, at: now() });
|
|
344
|
+
return status;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
function orgDeletes() {
|
|
348
|
+
try {
|
|
349
|
+
const q = JSON.parse(getPlatformState('vm_org_deletes') || '[]');
|
|
350
|
+
return Array.isArray(q) ? q : [];
|
|
351
|
+
} catch {
|
|
352
|
+
return [];
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
const setOrgDeletes = (q) => setPlatformState('vm_org_deletes', JSON.stringify(q.slice(-500)));
|
|
356
|
+
|
|
357
|
+
/** Deletes a child org (Orquesta cancels its subscription first). 409 while a VM is still going away → queued. */
|
|
358
|
+
async function dropChildOrg(accountId, org) {
|
|
359
|
+
if (!org) return true;
|
|
360
|
+
try {
|
|
361
|
+
await orq('DELETE', `/partner/orgs/${encodeURIComponent(org)}`, null, 30_000);
|
|
362
|
+
} catch (e) {
|
|
363
|
+
if (e.status !== 404) {
|
|
364
|
+
console.warn(`[7ots] vm org delete ${org}:`, e.message);
|
|
365
|
+
setOrgDeletes([...orgDeletes().filter((x) => x.org !== org), { org, accountId, at: now() }]);
|
|
366
|
+
return false;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
setOrgDeletes(orgDeletes().filter((x) => x.org !== org));
|
|
370
|
+
getDb().prepare("UPDATE accounts SET orq_org_id = '' WHERE id = ? AND orq_org_id = ?").run(accountId, org);
|
|
371
|
+
billingSeen.delete(accountId);
|
|
372
|
+
return true;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/** Bash run as root at the end of the VM's user-data (≤ 8 KB). Every value is validated, never free text. */
|
|
376
|
+
export function bootstrapScript({ server, code, cli = cliSpec() }) {
|
|
377
|
+
if (!/^https?:\/\/[A-Za-z0-9.:[\]-]+(\/[A-Za-z0-9._~/-]*)?$/.test(server)) throw new Error('bad server url');
|
|
378
|
+
if (!/^7ve_[A-Za-z0-9_-]{20,100}$/.test(code)) throw new Error('bad code');
|
|
379
|
+
return `#!/bin/bash
|
|
380
|
+
# 7ots cloud worker: user "ot" runs \`7ots vm worker\` (outbound HTTPS only).
|
|
381
|
+
set -euo pipefail
|
|
382
|
+
exec >>/var/log/7ots-bootstrap.log 2>&1
|
|
383
|
+
echo "[7ots] bootstrap $(date -u +%FT%TZ)"
|
|
384
|
+
CLI='${cli}'
|
|
385
|
+
SERVER='${server}'
|
|
386
|
+
CODE='${code}'
|
|
387
|
+
if ! command -v node >/dev/null 2>&1 || ! node -e 'process.exit(+process.versions.node.split(".")[0] < 20 ? 1 : 0)'; then
|
|
388
|
+
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
|
|
389
|
+
DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs
|
|
390
|
+
fi
|
|
391
|
+
id ot >/dev/null 2>&1 || useradd --create-home --shell /bin/bash ot
|
|
392
|
+
# Pinned and installed once: the service never fetches code at restart.
|
|
393
|
+
npm install -g --no-fund --no-audit "$CLI"
|
|
394
|
+
BIN="$(command -v 7ots)"
|
|
395
|
+
sudo -u ot -H "$BIN" vm enroll --server "$SERVER" --code "$CODE"
|
|
396
|
+
# System unit (a --user unit would need a login session/linger for "ot").
|
|
397
|
+
cat >/etc/systemd/system/7ots-vm.service <<UNIT
|
|
398
|
+
[Unit]
|
|
399
|
+
Description=7ots cloud worker (7ots vm worker)
|
|
400
|
+
Wants=network-online.target
|
|
401
|
+
After=network-online.target
|
|
402
|
+
|
|
403
|
+
[Service]
|
|
404
|
+
User=ot
|
|
405
|
+
Group=ot
|
|
406
|
+
WorkingDirectory=/home/ot
|
|
407
|
+
Environment=HOME=/home/ot
|
|
408
|
+
ExecStart=$BIN vm worker
|
|
409
|
+
Restart=always
|
|
410
|
+
RestartSec=10
|
|
411
|
+
NoNewPrivileges=true
|
|
412
|
+
PrivateTmp=true
|
|
413
|
+
ProtectSystem=full
|
|
414
|
+
|
|
415
|
+
[Install]
|
|
416
|
+
WantedBy=multi-user.target
|
|
417
|
+
UNIT
|
|
418
|
+
systemctl daemon-reload
|
|
419
|
+
systemctl enable --now 7ots-vm
|
|
420
|
+
echo "[7ots] worker started"
|
|
421
|
+
`;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
function setRow(otsId, fields, where = '') {
|
|
425
|
+
const keys = Object.keys(fields);
|
|
426
|
+
return getDb()
|
|
427
|
+
.prepare(`UPDATE ots_vms SET ${keys.map((k) => `${k} = ?`).join(', ')}, updated_at = ? WHERE ots_id = ?${where ? ` AND ${where}` : ''}`)
|
|
428
|
+
.run(...keys.map((k) => fields[k]), now(), otsId).changes;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
const inflight = new Map(); // ots id → provisioning promise
|
|
432
|
+
const checking = new Map(); // ots id → status refresh promise
|
|
433
|
+
|
|
434
|
+
function teardownQueue() {
|
|
435
|
+
try {
|
|
436
|
+
const q = JSON.parse(getPlatformState('vm_teardowns') || '[]');
|
|
437
|
+
return Array.isArray(q) ? q : [];
|
|
438
|
+
} catch {
|
|
439
|
+
return [];
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
/** item: { pid, otsId, accountId, org, user } — enough to rebuild the credential that made the project. */
|
|
443
|
+
function queueTeardown(item) {
|
|
444
|
+
const q = teardownQueue().filter((x) => x.pid !== item.pid);
|
|
445
|
+
setPlatformState('vm_teardowns', JSON.stringify([...q, { ...item, at: now() }].slice(-500)));
|
|
446
|
+
}
|
|
447
|
+
function dequeueTeardown(pid) {
|
|
448
|
+
setPlatformState('vm_teardowns', JSON.stringify(teardownQueue().filter((x) => x.pid !== pid)));
|
|
449
|
+
}
|
|
450
|
+
const teardownItem = (row, pid = row.orq_project_id) => ({ pid, otsId: row.ots_id, accountId: row.account_id, org: row.orq_org_id || '', user: row.provider === 'orquesta_user' });
|
|
451
|
+
|
|
452
|
+
/** Terminates the project's VM, then the project. 404 = already gone. 409 on the project = VM still going away. */
|
|
453
|
+
async function destroy(pid, { retries = 6, cred = null } = {}) {
|
|
454
|
+
if (!pid) return;
|
|
455
|
+
const p = `/projects/${encodeURIComponent(pid)}`;
|
|
456
|
+
try {
|
|
457
|
+
await orq('DELETE', `${p}/vm`, null, 60_000, cred);
|
|
458
|
+
} catch (e) {
|
|
459
|
+
if (e.status !== 404) throw e;
|
|
460
|
+
}
|
|
461
|
+
for (let i = 0; ; i++) {
|
|
462
|
+
try {
|
|
463
|
+
return void (await orq('DELETE', p, null, 20_000, cred));
|
|
464
|
+
} catch (e) {
|
|
465
|
+
if (e.status === 404) return;
|
|
466
|
+
if (e.status !== 409 || i >= retries) throw e;
|
|
467
|
+
await sleep(PROJECT_RETRY_MS());
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/** Runs after DELETE (and after a provisioning in flight ends): row 'terminating' → 'terminated'. */
|
|
473
|
+
async function finishTeardown(otsId) {
|
|
474
|
+
await inflight.get(otsId)?.catch(() => {});
|
|
475
|
+
const row = vmRow(otsId);
|
|
476
|
+
if (!row || row.status !== 'terminating') return;
|
|
477
|
+
try {
|
|
478
|
+
await destroy(row.orq_project_id, { cred: credOf(row) });
|
|
479
|
+
setRow(otsId, { status: 'terminated', orq_status: 'terminated', ip: '', error: '' }, "status = 'terminating'");
|
|
480
|
+
dequeueTeardown(row.orq_project_id);
|
|
481
|
+
} catch (e) {
|
|
482
|
+
console.warn(`[7ots] vm teardown ${otsId}:`, e.message);
|
|
483
|
+
setRow(otsId, { error: clean(e.message, 500) }, "status = 'terminating'"); // reconciliation retries
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/** What the owner reads when a create fails (in the language of the request that started it). */
|
|
488
|
+
function createError(e, cred, t) {
|
|
489
|
+
if (cred.kind === 'partner' && e.status === 402) return t('platform.vm.needsPayment');
|
|
490
|
+
if (cred.kind === 'user' && e.code === 'credentials') return e.i18n ? t(e.i18n.key, e.i18n.vars) : e.message;
|
|
491
|
+
if (cred.kind === 'user' && (e.status === 401 || e.status === 403)) return `${t('platform.vm.userScopes')} (${e.message})`;
|
|
492
|
+
return e.message;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/** Creates project + VM. The row may be deleted (ot gone) or set 'terminating' meanwhile: then it undoes its own work. */
|
|
496
|
+
async function provision(ots, { server, code, size, region, cred, t }) {
|
|
497
|
+
let pid = '';
|
|
498
|
+
const gone = () => !vmRow(ots.id);
|
|
499
|
+
const item = () => ({ pid, otsId: ots.id, accountId: ots.accountId, org: cred.org || '', user: cred.kind === 'user' });
|
|
500
|
+
const undo = (retries) => destroy(pid, { retries, cred }).catch(() => queueTeardown(item()));
|
|
501
|
+
try {
|
|
502
|
+
const p = await orq('POST', '/projects', { name: `ot-${ots.id}`, description: clean(`7ots cloud worker for "${ots.name}" (ot ${ots.id}, account ${ots.accountId})`, 300) }, 20_000, cred);
|
|
503
|
+
pid = String(p.project?.id || '');
|
|
504
|
+
if (!pid) throw new OrqError('Orquesta returned no project id');
|
|
505
|
+
setRow(ots.id, { orq_project_id: pid });
|
|
506
|
+
if (gone()) return void (await undo(0));
|
|
507
|
+
if (vmRow(ots.id).status !== 'creating') return; // DELETE came first: finishTeardown takes it from here
|
|
508
|
+
const r = await orq('POST', `/projects/${encodeURIComponent(pid)}/vm`, { size, ...(region ? { region } : {}), external_ref: `${REF_PREFIX()}${ots.id}`, bootstrap: bootstrapScript({ server, code }) }, 6 * 60_000, cred);
|
|
509
|
+
const v = r.vm || {};
|
|
510
|
+
if (gone()) return void (await undo());
|
|
511
|
+
setRow(ots.id, { orq_vm_id: String(v.id || ''), orq_status: clean(v.status, 40), ip: clean(v.ip, 60), price_cents: priceCents(v) ?? vmRow(ots.id).price_cents, orq_checked_at: now() }, "status <> 'terminated'");
|
|
512
|
+
setRow(ots.id, { status: 'pending' }, "status = 'creating'");
|
|
513
|
+
} catch (e) {
|
|
514
|
+
console.warn(`[7ots] vm create ${ots.id}:`, e.message);
|
|
515
|
+
if (e.status === 402) billingSeen.delete(ots.accountId); // the card asks for the payment again
|
|
516
|
+
if (gone()) return void (pid && (await undo()));
|
|
517
|
+
if (vmRow(ots.id).status !== 'creating') return;
|
|
518
|
+
// Undo what exists (a timed-out create may still have made the VM): ids cleared only once it is gone.
|
|
519
|
+
let undone = !pid;
|
|
520
|
+
if (pid) undone = await destroy(pid, { cred }).then(() => true, () => false);
|
|
521
|
+
setRow(ots.id, { status: 'error', error: clean(createError(e, cred, t), 500), enroll_hash: null, enroll_exp: null, ...(undone ? { orq_project_id: '', orq_vm_id: '', orq_status: '' } : {}) }, "status = 'creating'");
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/** Orquesta's view of the VM, at most every 15 s per ot (shared by concurrent readers). */
|
|
526
|
+
async function refreshHosted(otsId, { force = false } = {}) {
|
|
527
|
+
const row = vmRow(otsId);
|
|
528
|
+
if (!row || !HOSTED.includes(row.provider) || !row.orq_project_id) return row;
|
|
529
|
+
if (row.provider === 'orquesta' && !hostedConfigured()) return row;
|
|
530
|
+
if (!['pending', 'enrolled', 'error'].includes(row.status) || inflight.has(otsId)) return row;
|
|
531
|
+
if (!force && row.orq_checked_at && now() - row.orq_checked_at < ORQ_CACHE_MS) return row;
|
|
532
|
+
if (!checking.has(otsId)) {
|
|
533
|
+
const p = (async () => {
|
|
534
|
+
try {
|
|
535
|
+
const v = (await orq('GET', `/projects/${encodeURIComponent(row.orq_project_id)}/vm`, null, 8000, credOf(row))).vm;
|
|
536
|
+
if (!v) throw Object.assign(new OrqError('no vm'), { status: 404 });
|
|
537
|
+
const st = clean(v.status, 40);
|
|
538
|
+
const fields = { orq_status: st, ip: clean(v.ip, 60), orq_checked_at: now(), ...(v.id ? { orq_vm_id: String(v.id) } : {}) };
|
|
539
|
+
const pc = priceCents(v);
|
|
540
|
+
if (pc) fields.price_cents = pc;
|
|
541
|
+
setRow(otsId, fields);
|
|
542
|
+
if (/^(error|failed)$/i.test(st)) setRow(otsId, { status: 'error', error: clean(v.error || `Orquesta reports the VM as ${st}`, 500) }, "status = 'pending'");
|
|
543
|
+
else if (/^(terminated|deleted|destroyed)$/i.test(st)) setRow(otsId, { status: 'error', error: 'The VM was terminated outside 7ots' }, "status IN ('pending', 'enrolled')");
|
|
544
|
+
else if (/^(running|active)$/i.test(st)) {
|
|
545
|
+
const cur = vmRow(otsId);
|
|
546
|
+
if (cur.status === 'pending' && cur.enroll_exp && cur.enroll_exp < now()) setRow(otsId, { status: 'error', error: 'The server did not connect before its enrollment code expired; delete it and create it again', enroll_hash: null, enroll_exp: null }, "status = 'pending'");
|
|
547
|
+
}
|
|
548
|
+
} catch (e) {
|
|
549
|
+
if (e.status === 404) {
|
|
550
|
+
setRow(otsId, { orq_status: 'gone', orq_checked_at: now() });
|
|
551
|
+
setRow(otsId, { status: 'error', error: 'The VM no longer exists in Orquesta' }, "status IN ('pending', 'enrolled')");
|
|
552
|
+
} else {
|
|
553
|
+
setRow(otsId, { orq_checked_at: now() }); // keep the cache window on errors too
|
|
554
|
+
console.warn(`[7ots] vm status ${otsId}:`, e.message);
|
|
555
|
+
}
|
|
556
|
+
} finally {
|
|
557
|
+
checking.delete(otsId);
|
|
558
|
+
}
|
|
559
|
+
})();
|
|
560
|
+
checking.set(otsId, p);
|
|
561
|
+
}
|
|
562
|
+
await checking.get(otsId);
|
|
563
|
+
return vmRow(otsId);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
/**
|
|
567
|
+
* Compares Orquesta with ots_vms. 'orquesta' VMs: GET /api/v1/vms of 7ots' own org and of every child
|
|
568
|
+
* org; refreshes them, retries teardowns, adopts or flags lost creates and logs orphans.
|
|
569
|
+
* 'orquesta_user' VMs: refreshed one by one with each owner's own credential (never listed: their org
|
|
570
|
+
* is theirs). Then queued teardowns and child-org deletions are retried.
|
|
571
|
+
*/
|
|
572
|
+
export async function reconcileVms() {
|
|
573
|
+
const rows = getDb().prepare(`SELECT * FROM ots_vms WHERE ${LIVE_ORQ}`).all();
|
|
574
|
+
const report = { checked: rows.length, orquesta: 0, orgs: 0, user: 0, orphans: [], missing: [], retried: 0 };
|
|
575
|
+
const prefix = REF_PREFIX();
|
|
576
|
+
const orgOf = (r) => r.orq_org_id || '';
|
|
577
|
+
if (hostedConfigured()) {
|
|
578
|
+
const partnerRows = rows.filter((r) => r.provider === 'orquesta');
|
|
579
|
+
// '' = 7ots' own org (VMs made before child orgs existed).
|
|
580
|
+
const orgs = new Set(['', ...partnerRows.map(orgOf), ...getDb().prepare("SELECT orq_org_id FROM accounts WHERE orq_org_id <> ''").all().map((a) => a.orq_org_id)]);
|
|
581
|
+
const ours = [];
|
|
582
|
+
const failed = new Set();
|
|
583
|
+
for (const org of orgs) {
|
|
584
|
+
try {
|
|
585
|
+
const { vms = [] } = await orq('GET', '/vms', null, 30_000, { kind: 'partner', org });
|
|
586
|
+
for (const v of vms) if (String(v.external_ref || '').startsWith(prefix)) ours.push({ v, org, otsId: String(v.external_ref).slice(prefix.length) });
|
|
587
|
+
report.orgs++;
|
|
588
|
+
} catch (e) {
|
|
589
|
+
failed.add(org);
|
|
590
|
+
console.warn(`[7ots] vm reconcile: GET /vms${org ? ` (org ${org})` : ''}:`, e.message);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
if (failed.size === orgs.size) throw new OrqError('Orquesta did not list any VMs');
|
|
594
|
+
report.orquesta = ours.length;
|
|
595
|
+
for (const row of partnerRows) {
|
|
596
|
+
if (inflight.has(row.ots_id) || failed.has(orgOf(row))) continue;
|
|
597
|
+
const hit = ours.find((x) => x.org === orgOf(row) && (x.otsId === row.ots_id || (row.orq_project_id && String(x.v.project_id) === row.orq_project_id)));
|
|
598
|
+
const v = hit?.v;
|
|
599
|
+
if (row.status === 'terminating') {
|
|
600
|
+
report.retried++;
|
|
601
|
+
await finishTeardown(row.ots_id);
|
|
602
|
+
continue;
|
|
603
|
+
}
|
|
604
|
+
if (row.status === 'creating') {
|
|
605
|
+
// The server stopped mid-create: adopt what Orquesta made, or give up on it.
|
|
606
|
+
if (v) setRow(row.ots_id, { orq_project_id: String(v.project_id || row.orq_project_id), orq_vm_id: String(v.id || ''), orq_status: clean(v.status, 40), ip: clean(v.ip, 60), status: 'pending' }, "status = 'creating'");
|
|
607
|
+
else if (now() - row.updated_at > 15 * 60_000) setRow(row.ots_id, { status: 'error', error: 'Creation was interrupted' }, "status = 'creating'");
|
|
608
|
+
continue;
|
|
609
|
+
}
|
|
610
|
+
if (!v && row.orq_project_id) {
|
|
611
|
+
report.missing.push(row.ots_id);
|
|
612
|
+
setRow(row.ots_id, { orq_status: 'gone', orq_checked_at: now() });
|
|
613
|
+
if (!online(row)) setRow(row.ots_id, { status: 'error', error: 'The VM no longer exists in Orquesta' }, "status IN ('pending', 'enrolled')");
|
|
614
|
+
console.warn(`[7ots] vm reconcile: ${row.ots_id} has no VM in Orquesta`);
|
|
615
|
+
} else if (v) setRow(row.ots_id, { orq_status: clean(v.status, 40), ip: clean(v.ip, 60), orq_checked_at: now(), ...(v.id ? { orq_vm_id: String(v.id) } : {}) });
|
|
616
|
+
}
|
|
617
|
+
const live = new Set(partnerRows.map((r) => `${orgOf(r)}|${r.ots_id}`));
|
|
618
|
+
const queued = new Set(teardownQueue().map((x) => x.pid));
|
|
619
|
+
for (const { v, org, otsId } of ours) {
|
|
620
|
+
if (live.has(`${org}|${otsId}`) || queued.has(String(v.project_id))) continue;
|
|
621
|
+
if (/^(terminated|terminating|deleted|destroyed)$/i.test(String(v.status))) continue;
|
|
622
|
+
report.orphans.push({ otsId, vmId: v.id, projectId: v.project_id, status: v.status, ...(org ? { org } : {}) });
|
|
623
|
+
console.warn(`[7ots] vm reconcile: orphan VM ${v.id} (project ${v.project_id}, ref ${v.external_ref}${org ? `, org ${org}` : ''}) has no live ot`);
|
|
624
|
+
if (penv.VM_REAP_ORPHANS === '1') await destroy(String(v.project_id), { cred: { kind: 'partner', org } }).catch((e) => console.warn('[7ots] vm orphan teardown:', e.message));
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
// Owners' own VMs, each with its owner's credential.
|
|
628
|
+
for (const row of rows.filter((r) => r.provider === 'orquesta_user')) {
|
|
629
|
+
if (inflight.has(row.ots_id)) continue;
|
|
630
|
+
report.user++;
|
|
631
|
+
if (row.status === 'terminating') {
|
|
632
|
+
report.retried++;
|
|
633
|
+
await finishTeardown(row.ots_id);
|
|
634
|
+
} else if (row.status === 'creating') {
|
|
635
|
+
if (now() - row.updated_at > 15 * 60_000) setRow(row.ots_id, { status: 'error', error: 'Creation was interrupted' }, "status = 'creating'");
|
|
636
|
+
} else await refreshHosted(row.ots_id, { force: true });
|
|
637
|
+
}
|
|
638
|
+
// Teardowns that failed when an ot was deleted.
|
|
639
|
+
for (const item of teardownQueue()) {
|
|
640
|
+
if (item.user && !getDb().prepare('SELECT 1 FROM accounts WHERE id = ?').get(item.accountId)) {
|
|
641
|
+
// The owner's account (and with it their credential) is gone: the project stays in THEIR org.
|
|
642
|
+
console.warn(`[7ots] vm teardown ${item.pid}: the owner's account is gone; it must be removed in Orquesta`);
|
|
643
|
+
dequeueTeardown(item.pid);
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
if (!item.user && !hostedConfigured()) continue;
|
|
647
|
+
report.retried++;
|
|
648
|
+
await destroy(item.pid, { cred: credOfItem(item) }).then(() => dequeueTeardown(item.pid), (e) => console.warn(`[7ots] vm teardown retry ${item.pid}:`, e.message));
|
|
649
|
+
}
|
|
650
|
+
// Child orgs let go of (account released) while a VM was still going away.
|
|
651
|
+
if (hostedConfigured()) {
|
|
652
|
+
for (const x of orgDeletes()) {
|
|
653
|
+
if (getDb().prepare(`SELECT 1 FROM ots_vms WHERE ${LIVE_PARTNER} AND orq_org_id = ?`).get(x.org)) continue;
|
|
654
|
+
report.retried++;
|
|
655
|
+
await dropChildOrg(x.accountId, x.org);
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
setPlatformState('vm_orphans', JSON.stringify({ at: now(), orphans: report.orphans, missing: report.missing }));
|
|
659
|
+
return report;
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/** Before an ot (or its account) is deleted: revoke its worker and tear its Orquesta VM down (queued if Orquesta fails). */
|
|
663
|
+
export async function releaseOtsVm(otsId) {
|
|
664
|
+
const row = vmRow(otsId);
|
|
665
|
+
if (!row) return;
|
|
666
|
+
getDb().prepare('UPDATE ots_vms SET agent_hash = NULL, enroll_hash = NULL, enroll_exp = NULL, updated_at = ? WHERE ots_id = ?').run(now(), otsId);
|
|
667
|
+
if (!HOSTED.includes(row.provider) || row.status === 'terminated') return;
|
|
668
|
+
setRow(otsId, { status: 'terminating' });
|
|
669
|
+
if (inflight.has(otsId)) return; // provision() sees the row gone and undoes its own work
|
|
670
|
+
if (!row.orq_project_id) return;
|
|
671
|
+
try {
|
|
672
|
+
await destroy(row.orq_project_id, { retries: 0, cred: credOf(row) });
|
|
673
|
+
} catch (e) {
|
|
674
|
+
console.warn(`[7ots] vm teardown on delete ${otsId}:`, e.message);
|
|
675
|
+
queueTeardown(teardownItem(row));
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/**
|
|
680
|
+
* Before an account is deleted: every VM of it goes, then its Orquesta child org (Orquesta cancels the
|
|
681
|
+
* subscription first). While a VM is still going away Orquesta answers 409 and the deletion is queued
|
|
682
|
+
* for the reconciliation.
|
|
683
|
+
*/
|
|
684
|
+
export async function releaseAccountVms(accountId) {
|
|
685
|
+
for (const { ots_id } of getDb().prepare('SELECT ots_id FROM ots_vms WHERE account_id = ?').all(accountId)) await releaseOtsVm(ots_id);
|
|
686
|
+
await Promise.all(getDb().prepare('SELECT ots_id FROM ots_vms WHERE account_id = ?').all(accountId).map(({ ots_id }) => inflight.get(ots_id)?.catch(() => {})));
|
|
687
|
+
const org = accountOrg(accountId);
|
|
688
|
+
if (org && hostedConfigured()) await dropChildOrg(accountId, org);
|
|
689
|
+
}
|
|
690
|
+
|
|
183
691
|
/**
|
|
184
692
|
* @param {{ send: Function, readJson: Function, rateLimit: (req) => boolean, base: (req) => string }} deps
|
|
185
693
|
*/
|
|
@@ -353,34 +861,197 @@ export function createVm({ send, readJson, rateLimit, base }) {
|
|
|
353
861
|
|
|
354
862
|
// ───────────── dashboard (/api/platform/ots/:id/vm…) ─────────────
|
|
355
863
|
|
|
864
|
+
/** What the dashboard needs to offer a hosted server: 7ots' (via the account's child org) and/or one in the owner's own Orquesta org. */
|
|
865
|
+
function hostedInfo(account) {
|
|
866
|
+
const enabled = account.vm_enabled === 1;
|
|
867
|
+
const configured = hostedConfigured();
|
|
868
|
+
const used = getDb().prepare(`SELECT COUNT(*) AS n FROM ots_vms WHERE ${LIVE_ORQ} AND account_id = ?`).get(account.id).n;
|
|
869
|
+
const total = getDb().prepare(`SELECT COUNT(*) AS n FROM ots_vms WHERE ${LIVE_PARTNER}`).get().n;
|
|
870
|
+
const reason = !enabled ? 'notEnabled' : !configured ? 'notConfigured' : used >= PER_ACCOUNT() ? 'accountLimit' : total >= MAX_TOTAL() ? 'full' : null;
|
|
871
|
+
const size = defaultSize();
|
|
872
|
+
const access = orquestaVmAccess(account.id);
|
|
873
|
+
const userReason = !access.usable ? access.reason : used >= PER_ACCOUNT() ? 'accountLimit' : null;
|
|
874
|
+
const org = accountOrg(account.id);
|
|
875
|
+
return {
|
|
876
|
+
enabled,
|
|
877
|
+
configured,
|
|
878
|
+
available: enabled && configured,
|
|
879
|
+
canCreate: !reason,
|
|
880
|
+
reason,
|
|
881
|
+
sizes: hostedSizes().filter((z) => z.id === size),
|
|
882
|
+
defaultSize: size,
|
|
883
|
+
monthlyPrice: hostedSizes().find((z) => z.id === size)?.monthlyPrice ?? null,
|
|
884
|
+
region: penv.ORQUESTA_VM_REGION || null,
|
|
885
|
+
used,
|
|
886
|
+
max: PER_ACCOUNT(),
|
|
887
|
+
operator: 'Orquesta',
|
|
888
|
+
// 7ots server: Orquesta bills the account's child org; null = never paid (or not known yet).
|
|
889
|
+
billing: org ? (billingSeen.get(account.id)?.status ?? null) : null,
|
|
890
|
+
// The owner's own Orquesta account (path A): needs an oak_ key with projects:write + vms:write.
|
|
891
|
+
user: { connected: access.connected, canCreate: !userReason, reason: userReason },
|
|
892
|
+
defaultOption: access.connected ? 'user' : '7ots',
|
|
893
|
+
};
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
/** Same quotas for both providers, checked inside the reserving transaction (and once before any Orquesta call). */
|
|
897
|
+
function quotaError(d, t, account, ots, partner) {
|
|
898
|
+
const cur = d.prepare('SELECT * FROM ots_vms WHERE ots_id = ?').get(ots.id);
|
|
899
|
+
if (cur && cur.status !== 'terminated' && (HOSTED.includes(cur.provider) || cur.status === 'enrolled')) return [409, t('platform.vm.exists')];
|
|
900
|
+
if (inflight.has(ots.id)) return [409, t('platform.vm.exists')];
|
|
901
|
+
if (d.prepare(`SELECT COUNT(*) AS n FROM ots_vms WHERE ${LIVE_ORQ} AND account_id = ?`).get(account.id).n >= PER_ACCOUNT()) return [403, t('platform.vm.accountLimit', { max: PER_ACCOUNT() })];
|
|
902
|
+
if (partner && d.prepare(`SELECT COUNT(*) AS n FROM ots_vms WHERE ${LIVE_PARTNER}`).get().n >= MAX_TOTAL()) return [503, t('platform.vm.full')];
|
|
903
|
+
return null;
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
/** Billing of the account's child org; an org deleted on Orquesta's side is forgotten and made again. */
|
|
907
|
+
async function billingOf(account) {
|
|
908
|
+
let org = await ensureChildOrg(account.id);
|
|
909
|
+
try {
|
|
910
|
+
return { org, status: await childBilling(account.id, org) };
|
|
911
|
+
} catch (e) {
|
|
912
|
+
if (e.status !== 404) throw e;
|
|
913
|
+
getDb().prepare("UPDATE accounts SET orq_org_id = '' WHERE id = ? AND orq_org_id = ?").run(account.id, org);
|
|
914
|
+
org = await ensureChildOrg(account.id);
|
|
915
|
+
return { org, status: await childBilling(account.id, org) };
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
/**
|
|
920
|
+
* POST /vm {provider:'orquesta'|'orquesta_user'}: reserves the slot (quotas, atomically) and provisions in the background.
|
|
921
|
+
* 'orquesta' first makes sure the account's child org exists and is paid: if not, { needsPayment, checkoutUrl }.
|
|
922
|
+
*/
|
|
923
|
+
async function createHosted(req, res, account, ots, provider) {
|
|
924
|
+
const t = reqT(req);
|
|
925
|
+
const user = provider === 'orquesta_user';
|
|
926
|
+
if (user) {
|
|
927
|
+
const a = orquestaVmAccess(account.id);
|
|
928
|
+
if (!a.usable) {
|
|
929
|
+
const key = { needsKey: 'platform.vm.userNeedsKey', reconnect: 'platform.orquesta.reconnect' }[a.reason] || 'platform.orquesta.notConnected';
|
|
930
|
+
return send(res, 409, { error: t(key), code: `orquesta_${a.reason}` });
|
|
931
|
+
}
|
|
932
|
+
} else {
|
|
933
|
+
if (account.vm_enabled !== 1) return send(res, 403, { error: t('platform.vm.paidOff') });
|
|
934
|
+
if (!hostedConfigured()) return send(res, 503, { error: t('platform.vm.paidUnavailable') });
|
|
935
|
+
}
|
|
936
|
+
const size = defaultSize();
|
|
937
|
+
let server;
|
|
938
|
+
try {
|
|
939
|
+
server = base(req);
|
|
940
|
+
bootstrapScript({ server, code: `7ve_${'x'.repeat(30)}` }); // validates the URL and SEVENOTS_VM_CLI before spending anything
|
|
941
|
+
} catch (e) {
|
|
942
|
+
console.warn('[7ots] vm create:', e.message);
|
|
943
|
+
return send(res, 503, { error: t('platform.vm.paidUnavailable') });
|
|
944
|
+
}
|
|
945
|
+
const pre = quotaError(getDb(), t, account, ots, !user);
|
|
946
|
+
if (pre) return send(res, pre[0], { error: pre[1] });
|
|
947
|
+
const price = hostedSizes().find((s) => s.id === size).monthlyPrice;
|
|
948
|
+
let cred = { kind: 'user', accountId: account.id };
|
|
949
|
+
if (!user) {
|
|
950
|
+
try {
|
|
951
|
+
const { org, status } = await billingOf(account);
|
|
952
|
+
if (status !== 'active') {
|
|
953
|
+
let ck = null;
|
|
954
|
+
try {
|
|
955
|
+
// No #fragment: Orquesta appends ?orq_checkout=success|cancel; the dashboard opens the ot from vm_ots.
|
|
956
|
+
ck = await orq('POST', `/partner/orgs/${encodeURIComponent(org)}/checkout`, { return_url: `${server}/app/?vm_ots=${encodeURIComponent(ots.id)}`, size });
|
|
957
|
+
} catch (e) {
|
|
958
|
+
if (e.status !== 409) throw e; // ALREADY_ACTIVE (paid meanwhile) / PARTNER_BILLED: nothing to pay
|
|
959
|
+
}
|
|
960
|
+
if (ck) {
|
|
961
|
+
if (!/^https:\/\//.test(String(ck.url || ''))) throw new OrqError('Orquesta returned no checkout URL');
|
|
962
|
+
return send(res, 200, { needsPayment: true, checkoutUrl: ck.url, size, monthlyPrice: price, operator: 'Orquesta', hosted: hostedInfo(account) });
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
cred = { kind: 'partner', org };
|
|
966
|
+
} catch (e) {
|
|
967
|
+
console.warn('[7ots] vm billing:', e.message);
|
|
968
|
+
return send(res, 503, { error: t('platform.vm.paidUnavailable') });
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
const region = penv.ORQUESTA_VM_REGION || '';
|
|
972
|
+
const code = `7ve_${token()}`;
|
|
973
|
+
const tm = now();
|
|
974
|
+
const err = tx((d) => {
|
|
975
|
+
const q = quotaError(d, t, account, ots, !user);
|
|
976
|
+
if (q) return q;
|
|
977
|
+
d.prepare(`INSERT INTO ots_vms (ots_id, account_id, provider, orq_org_id, status, size, region, enroll_hash, enroll_exp, price_cents, created_at, updated_at) VALUES (?, ?, ?, ?, 'creating', ?, ?, ?, ?, ?, ?, ?)
|
|
978
|
+
ON CONFLICT(ots_id) DO UPDATE SET account_id = excluded.account_id, provider = excluded.provider, orq_org_id = excluded.orq_org_id, status = 'creating', orq_project_id = '', orq_vm_id = '', orq_status = '', orq_checked_at = NULL, size = excluded.size, region = excluded.region,
|
|
979
|
+
ip = '', error = '', enroll_hash = excluded.enroll_hash, enroll_exp = excluded.enroll_exp, agent_hash = NULL, agent_seen_at = NULL, worker_version = '', worker_info = '{}', price_cents = excluded.price_cents, created_at = excluded.created_at, updated_at = excluded.updated_at`)
|
|
980
|
+
.run(ots.id, account.id, user ? 'orquesta_user' : 'orquesta', cred.org || '', size, region, sha256(code), tm + ORQ_ENROLL_TTL(), price ? Math.round(price * 100) : null, tm, tm);
|
|
981
|
+
return null;
|
|
982
|
+
});
|
|
983
|
+
if (err) return send(res, err[0], { error: err[1] });
|
|
984
|
+
const job = provision(ots, { server, code, size, region, cred, t }).finally(() => inflight.delete(ots.id));
|
|
985
|
+
inflight.set(ots.id, job);
|
|
986
|
+
return send(res, 202, { vm: vmView(vmRow(ots.id)), hosted: hostedInfo(account) });
|
|
987
|
+
}
|
|
988
|
+
|
|
356
989
|
/** @returns {Promise<boolean>} true if it served the route */
|
|
357
990
|
async function dashboard(req, res, account, ots, sub) {
|
|
358
991
|
if (sub !== '/vm' && !sub.startsWith('/vm/')) return false;
|
|
359
992
|
const t = reqT(req);
|
|
360
|
-
|
|
993
|
+
let vm = vmRow(ots.id);
|
|
361
994
|
if (sub === '/vm' && req.method === 'GET') {
|
|
362
995
|
reap(ots.id);
|
|
363
|
-
|
|
996
|
+
if (HOSTED.includes(vm?.provider)) vm = await refreshHosted(ots.id);
|
|
997
|
+
// What the card says about payment: asked to Orquesta at most every 5 min per account.
|
|
998
|
+
const org = accountOrg(account.id);
|
|
999
|
+
const seenAt = billingSeen.get(account.id)?.at || 0;
|
|
1000
|
+
if (org && hostedConfigured() && now() - seenAt > 5 * 60_000) await childBilling(account.id, org).catch((e) => console.warn('[7ots] vm billing:', e.message));
|
|
1001
|
+
return send(res, 200, { vm: vmView(vm), jobs: recentJobs(ots.id), kinds: otsVmKinds(ots), allKinds: VM_KINDS, daily: { used: jobsLastDay(ots.id), max: DAILY() }, paidVms: account.vm_enabled === 1, hosted: hostedInfo(account) }), true;
|
|
1002
|
+
}
|
|
1003
|
+
if (sub === '/vm/billing' && req.method === 'GET') {
|
|
1004
|
+
// Polled by the dashboard after Orquesta's checkout, until the webhook marks the subscription active.
|
|
1005
|
+
const org = accountOrg(account.id);
|
|
1006
|
+
if (!org || !hostedConfigured()) return send(res, 200, { status: null, active: false }), true;
|
|
1007
|
+
try {
|
|
1008
|
+
const status = await childBilling(account.id, org);
|
|
1009
|
+
return send(res, 200, { status, active: status === 'active' }), true;
|
|
1010
|
+
} catch (e) {
|
|
1011
|
+
console.warn('[7ots] vm billing:', e.message);
|
|
1012
|
+
return send(res, 503, { error: t('platform.vm.paidUnavailable') }), true;
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
if (sub === '/vm/subscription' && req.method === 'DELETE') {
|
|
1016
|
+
// Ends what the account pays for 7ots servers: only with none left (deleting the org cancels the subscription).
|
|
1017
|
+
if (getDb().prepare(`SELECT 1 FROM ots_vms WHERE ${LIVE_PARTNER} AND account_id = ?`).get(account.id)) return send(res, 409, { error: t('platform.vm.subscriptionInUse') }), true;
|
|
1018
|
+
const org = accountOrg(account.id);
|
|
1019
|
+
if (!org) return send(res, 200, { ok: true }), true;
|
|
1020
|
+
const done = await dropChildOrg(account.id, org);
|
|
1021
|
+
return send(res, done ? 200 : 202, { ok: true, queued: !done, hosted: hostedInfo(account) }), true;
|
|
364
1022
|
}
|
|
365
|
-
if (sub === '/vm/enroll' && req.method === 'POST') {
|
|
366
|
-
const
|
|
367
|
-
|
|
368
|
-
if (provider === '
|
|
369
|
-
if (provider !== 'byo') return send(res, 400, { error: t('server.http.badRequest') }), true;
|
|
1023
|
+
if ((sub === '/vm' || sub === '/vm/enroll') && req.method === 'POST') {
|
|
1024
|
+
const body = await readJson(req, 4096);
|
|
1025
|
+
const provider = body.provider || (sub === '/vm' ? 'orquesta' : 'byo');
|
|
1026
|
+
if (HOSTED.includes(provider) && sub === '/vm') return await createHosted(req, res, account, ots, provider), true;
|
|
1027
|
+
if (provider !== 'byo' || sub !== '/vm/enroll') return send(res, 400, { error: t('server.http.badRequest') }), true;
|
|
1028
|
+
// A hosted server must be deleted first: re-enrolling would leave its VM running unattached.
|
|
1029
|
+
if (HOSTED.includes(vm?.provider) && vm.status !== 'terminated') return send(res, 409, { error: t('platform.vm.exists') }), true;
|
|
370
1030
|
const code = `7ve_${token()}`;
|
|
371
1031
|
const tm = now();
|
|
372
|
-
if (vm) getDb().prepare("UPDATE ots_vms SET enroll_hash = ?, enroll_exp = ?, provider = 'byo', status = CASE WHEN status = 'terminated' THEN 'pending' ELSE status END, updated_at = ? WHERE ots_id = ?").run(sha256(code), tm + ENROLL_TTL, tm, ots.id);
|
|
1032
|
+
if (vm) getDb().prepare("UPDATE ots_vms SET enroll_hash = ?, enroll_exp = ?, provider = 'byo', orq_org_id = '', status = CASE WHEN status = 'terminated' THEN 'pending' ELSE status END, updated_at = ? WHERE ots_id = ?").run(sha256(code), tm + ENROLL_TTL, tm, ots.id);
|
|
373
1033
|
else getDb().prepare("INSERT INTO ots_vms (ots_id, account_id, provider, status, enroll_hash, enroll_exp, created_at, updated_at) VALUES (?, ?, 'byo', 'pending', ?, ?, ?, ?)").run(ots.id, account.id, sha256(code), tm + ENROLL_TTL, tm, tm);
|
|
374
1034
|
return send(res, 200, { code, expiresIn: ENROLL_TTL / 1000, command: `npx -y @7ots/cli vm enroll --server ${base(req)} --code ${code} --systemd` }), true;
|
|
375
1035
|
}
|
|
376
1036
|
if (sub === '/vm' && req.method === 'DELETE') {
|
|
377
1037
|
if (!vm) return send(res, 200, { ok: true }), true;
|
|
378
|
-
|
|
1038
|
+
const hosted = HOSTED.includes(vm.provider) && vm.status !== 'terminated';
|
|
1039
|
+
getDb().prepare(`UPDATE ots_vms SET agent_hash = NULL, enroll_hash = NULL, enroll_exp = NULL, status = '${hosted ? 'terminating' : 'terminated'}', updated_at = ? WHERE ots_id = ?`).run(now(), ots.id);
|
|
379
1040
|
for (const j of getDb().prepare("SELECT * FROM vm_jobs WHERE ots_id = ? AND status NOT IN ('completed', 'failed', 'cancelled')").all(ots.id)) setStatus(j, 'cancelled', { error: 'worker revoked' });
|
|
380
|
-
return send(res, 200, { ok: true }), true;
|
|
1041
|
+
if (!hosted) return send(res, 200, { ok: true }), true;
|
|
1042
|
+
finishTeardown(ots.id); // background: VM, then project; reconciliation retries if Orquesta fails
|
|
1043
|
+
return send(res, 202, { ok: true, vm: vmView(vmRow(ots.id)) }), true;
|
|
381
1044
|
}
|
|
382
1045
|
return send(res, 404, { error: t('server.http.unknownRoute') }), true;
|
|
383
1046
|
}
|
|
384
1047
|
|
|
385
|
-
|
|
1048
|
+
// Reconciliation with Orquesta (every VM_RECONCILE_MIN, default 30): 7ots' VMs when ORQUESTA_VM_TOKEN is set, owners' own VMs always.
|
|
1049
|
+
if (!createVm.timer) {
|
|
1050
|
+
const run = () => reconcileVms().catch((e) => console.warn('[7ots] vm reconcile:', e.message));
|
|
1051
|
+
createVm.timer = setInterval(run, RECONCILE_MS());
|
|
1052
|
+
createVm.timer.unref?.();
|
|
1053
|
+
setTimeout(run, Math.min(60_000, RECONCILE_MS())).unref?.();
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
return { worker, device, dashboard, releaseOts: releaseOtsVm, releaseAccount: releaseAccountVms, reconcile: reconcileVms };
|
|
386
1057
|
}
|