@moldesk/core 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cloud/accounts.d.ts +69 -0
- package/dist/cloud/accounts.js +113 -0
- package/dist/cloud/accounts.test.d.ts +1 -0
- package/dist/cloud/accounts.test.js +188 -0
- package/dist/cloud/backend.test.d.ts +1 -0
- package/dist/cloud/backend.test.js +339 -0
- package/dist/cloud/deps.d.ts +15 -0
- package/dist/cloud/deps.js +30 -0
- package/dist/cloud/manage.d.ts +57 -0
- package/dist/cloud/manage.js +210 -0
- package/dist/cloud/manage.test.d.ts +1 -0
- package/dist/cloud/manage.test.js +105 -0
- package/dist/cloud/quote.d.ts +131 -0
- package/dist/cloud/quote.js +225 -0
- package/dist/cloud/quote.test.d.ts +1 -0
- package/dist/cloud/quote.test.js +264 -0
- package/dist/cloud/run.d.ts +114 -0
- package/dist/cloud/run.js +498 -0
- package/dist/cloud/run.test.d.ts +1 -0
- package/dist/cloud/run.test.js +597 -0
- package/dist/cloud/state.d.ts +159 -0
- package/dist/cloud/state.js +395 -0
- package/dist/cloud/state.test.d.ts +1 -0
- package/dist/cloud/state.test.js +199 -0
- package/dist/cloud/worker/cache.d.ts +75 -0
- package/dist/cloud/worker/cache.js +152 -0
- package/dist/cloud/worker/cache.test.d.ts +1 -0
- package/dist/cloud/worker/cache.test.js +123 -0
- package/dist/cloud/worker/main.d.ts +2 -0
- package/dist/cloud/worker/main.js +34 -0
- package/dist/cloud/worker/pipeline.d.ts +26 -0
- package/dist/cloud/worker/pipeline.js +160 -0
- package/dist/cloud/worker/pipeline.test.d.ts +1 -0
- package/dist/cloud/worker/pipeline.test.js +298 -0
- package/dist/cloud/worker/server.d.ts +38 -0
- package/dist/cloud/worker/server.js +287 -0
- package/dist/cloud/worker/server.test.d.ts +1 -0
- package/dist/cloud/worker/server.test.js +391 -0
- package/dist/cloud/worker/watchdog.d.ts +27 -0
- package/dist/cloud/worker/watchdog.js +46 -0
- package/dist/cloud/worker-release.d.ts +12 -0
- package/dist/cloud/worker-release.js +25 -0
- package/dist/cloud/worker-release.test.d.ts +1 -0
- package/dist/cloud/worker-release.test.js +21 -0
- package/dist/index.d.ts +17 -3
- package/dist/index.js +9 -2
- package/dist/run-lifecycle.d.ts +74 -0
- package/dist/run-lifecycle.js +178 -0
- package/dist/run.d.ts +61 -1
- package/dist/run.js +54 -125
- package/dist/run.test.js +47 -1
- package/package.json +10 -6
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import type { MoldeskPaths } from "@moldesk/runtime";
|
|
2
|
+
/**
|
|
3
|
+
* Cloud journal: the durable record of paid-resource intent and ownership, kept in
|
|
4
|
+
* `<state>/cloud.sqlite`. Scientific history stays in each run's `run.json`; this file only exists
|
|
5
|
+
* so a restarted client can find, reconcile and clean up what it created. It never stores secrets.
|
|
6
|
+
*/
|
|
7
|
+
export type CloudJobStage = "quoted" | "approved" | "provisioning" | "preparing_assets" | "installing_runtime" | "running" | "transferring" | "cleaning_up" | "done";
|
|
8
|
+
export type CloudJobStatus = "active" | "succeeded" | "failed" | "cancelled" | "cleanup_pending";
|
|
9
|
+
export type CloudResourceRole = "cpu" | "gpu" | "volume";
|
|
10
|
+
export type CloudResourceState = "create_pending" | "active" | "delete_pending" | "deleted";
|
|
11
|
+
export type CloudCacheState = "PREPARING" | "VERIFYING" | "READY" | "FAILED";
|
|
12
|
+
export type CloudCachePolicy = "ephemeral" | "persistent";
|
|
13
|
+
export interface CloudJobRecord {
|
|
14
|
+
runId: string;
|
|
15
|
+
accountScope: string;
|
|
16
|
+
quoteDigest: string;
|
|
17
|
+
cachePolicy: CloudCachePolicy;
|
|
18
|
+
stage: CloudJobStage;
|
|
19
|
+
status: CloudJobStatus;
|
|
20
|
+
gpuId: string | null;
|
|
21
|
+
datacenterId: string | null;
|
|
22
|
+
volumeId: string | null;
|
|
23
|
+
/** ISO time after which all of this job's resources must be gone. */
|
|
24
|
+
deadlineAt: string;
|
|
25
|
+
manifestDigest: string;
|
|
26
|
+
runtimeDigest: string | null;
|
|
27
|
+
transferState: string | null;
|
|
28
|
+
error: {
|
|
29
|
+
code: string;
|
|
30
|
+
message: string;
|
|
31
|
+
} | null;
|
|
32
|
+
createdAt: string;
|
|
33
|
+
updatedAt: string;
|
|
34
|
+
}
|
|
35
|
+
export interface CloudResourceRecord {
|
|
36
|
+
intentId: string;
|
|
37
|
+
runId: string | null;
|
|
38
|
+
clientId: string;
|
|
39
|
+
role: CloudResourceRole;
|
|
40
|
+
name: string;
|
|
41
|
+
providerId: string | null;
|
|
42
|
+
state: CloudResourceState;
|
|
43
|
+
deadlineAt: string | null;
|
|
44
|
+
createdAt: string;
|
|
45
|
+
updatedAt: string;
|
|
46
|
+
}
|
|
47
|
+
export interface CloudCacheRecord {
|
|
48
|
+
accountScope: string;
|
|
49
|
+
datacenterId: string;
|
|
50
|
+
volumeId: string;
|
|
51
|
+
model: string;
|
|
52
|
+
modelVersion: string;
|
|
53
|
+
cacheKey: string;
|
|
54
|
+
layoutRevision: string;
|
|
55
|
+
manifestDigest: string;
|
|
56
|
+
cachePath: string;
|
|
57
|
+
sizeBytes: number | null;
|
|
58
|
+
checksumManifestDigest: string | null;
|
|
59
|
+
state: CloudCacheState;
|
|
60
|
+
leaseOwner: string | null;
|
|
61
|
+
leaseExpiresAt: string | null;
|
|
62
|
+
createdAt: string;
|
|
63
|
+
lastUsedAt: string;
|
|
64
|
+
}
|
|
65
|
+
export interface NewCloudJob {
|
|
66
|
+
runId: string;
|
|
67
|
+
accountScope: string;
|
|
68
|
+
quoteDigest: string;
|
|
69
|
+
cachePolicy: CloudCachePolicy;
|
|
70
|
+
gpuId?: string;
|
|
71
|
+
datacenterId?: string;
|
|
72
|
+
deadlineAt: string;
|
|
73
|
+
manifestDigest: string;
|
|
74
|
+
}
|
|
75
|
+
/** What a provider listing says exists, reduced to what reconciliation needs. */
|
|
76
|
+
export interface ProviderResourceView {
|
|
77
|
+
providerId: string;
|
|
78
|
+
name: string;
|
|
79
|
+
role: CloudResourceRole;
|
|
80
|
+
}
|
|
81
|
+
export interface ReconciliationReport {
|
|
82
|
+
/** Provider resources whose name claims this client but that no journal row owns. Never auto-deleted. */
|
|
83
|
+
orphaned: ProviderResourceView[];
|
|
84
|
+
/** Journal rows the provider no longer has; they were marked deleted. */
|
|
85
|
+
vanished: CloudResourceRecord[];
|
|
86
|
+
/** `create_pending` rows that matched a provider resource by name; their provider ID was recorded. */
|
|
87
|
+
adopted: CloudResourceRecord[];
|
|
88
|
+
/** `create_pending` rows with no provider match: the create never took effect. Marked deleted. */
|
|
89
|
+
abandoned: CloudResourceRecord[];
|
|
90
|
+
/** Rows whose deletion is still owed (delete_pending, or past deadline and still active). */
|
|
91
|
+
cleanupOwed: CloudResourceRecord[];
|
|
92
|
+
}
|
|
93
|
+
/** `moldesk-<client8>-<run8>-<role>`: discovery hint only; ownership is decided by journal match. */
|
|
94
|
+
export declare function cloudResourceName(clientId: string, runId: string, role: CloudResourceRole): string;
|
|
95
|
+
export declare class CloudJournal {
|
|
96
|
+
private readonly db;
|
|
97
|
+
/** Stable per-installation identity used in resource names. */
|
|
98
|
+
readonly clientId: string;
|
|
99
|
+
private readonly now;
|
|
100
|
+
private constructor();
|
|
101
|
+
static open(paths: MoldeskPaths, options?: {
|
|
102
|
+
now?: () => string;
|
|
103
|
+
}): CloudJournal;
|
|
104
|
+
static openFile(file: string, options?: {
|
|
105
|
+
now?: () => string;
|
|
106
|
+
}): CloudJournal;
|
|
107
|
+
private static migrate;
|
|
108
|
+
close(): void;
|
|
109
|
+
createJob(job: NewCloudJob): CloudJobRecord;
|
|
110
|
+
getJob(runId: string): CloudJobRecord | undefined;
|
|
111
|
+
updateJob(runId: string, patch: Partial<Pick<CloudJobRecord, "stage" | "status" | "gpuId" | "datacenterId" | "volumeId" | "runtimeDigest" | "transferState" | "error">>): CloudJobRecord;
|
|
112
|
+
/** Jobs that have not reached a terminal status, or whose cleanup is still owed. */
|
|
113
|
+
listOpenJobs(): CloudJobRecord[];
|
|
114
|
+
/** Persist intent BEFORE calling the provider. The returned `name` is what must be sent to it. */
|
|
115
|
+
recordResourceIntent(input: {
|
|
116
|
+
runId?: string;
|
|
117
|
+
role: CloudResourceRole;
|
|
118
|
+
deadlineAt?: string;
|
|
119
|
+
intentId?: string;
|
|
120
|
+
}): CloudResourceRecord;
|
|
121
|
+
/** Record the provider ID immediately after the create response. */
|
|
122
|
+
recordResourceCreated(intentId: string, providerId: string): CloudResourceRecord;
|
|
123
|
+
/** Persist deletion intent BEFORE calling the provider's delete. */
|
|
124
|
+
markResourceDeletePending(intentId: string): CloudResourceRecord;
|
|
125
|
+
/** Call only after the provider confirmed deletion or confirmed absence. */
|
|
126
|
+
markResourceDeleted(intentId: string): CloudResourceRecord;
|
|
127
|
+
private transitionResource;
|
|
128
|
+
getResource(intentId: string): CloudResourceRecord | undefined;
|
|
129
|
+
/** Authorization check before any delete: only a journal row owned by this client may be deleted. */
|
|
130
|
+
findOwnedByProviderId(providerId: string): CloudResourceRecord | undefined;
|
|
131
|
+
listResources(filter?: {
|
|
132
|
+
runId?: string;
|
|
133
|
+
states?: CloudResourceState[];
|
|
134
|
+
}): CloudResourceRecord[];
|
|
135
|
+
/**
|
|
136
|
+
* Compare the journal with a provider listing. Pure bookkeeping plus journal updates; it never
|
|
137
|
+
* calls the provider and never deletes anything. Names are discovery hints only: a provider
|
|
138
|
+
* resource that merely looks ours is reported as orphaned, not adopted by guesswork, unless it
|
|
139
|
+
* matches a `create_pending` row's exact name (an ambiguous create that did take effect).
|
|
140
|
+
*/
|
|
141
|
+
reconcile(provider: ProviderResourceView[]): ReconciliationReport;
|
|
142
|
+
upsertCache(cache: Omit<CloudCacheRecord, "createdAt" | "lastUsedAt" | "leaseOwner" | "leaseExpiresAt" | "state"> & {
|
|
143
|
+
state?: CloudCacheState;
|
|
144
|
+
}): CloudCacheRecord;
|
|
145
|
+
getCache(accountScope: string, datacenterId: string, volumeId: string, cacheKey: string): CloudCacheRecord | undefined;
|
|
146
|
+
listCaches(): CloudCacheRecord[];
|
|
147
|
+
/** Allowed: PREPARING → VERIFYING → READY, and any → FAILED. READY is never set without verification. */
|
|
148
|
+
setCacheState(key: Pick<CloudCacheRecord, "accountScope" | "datacenterId" | "volumeId" | "cacheKey">, next: CloudCacheState, details?: {
|
|
149
|
+
sizeBytes?: number;
|
|
150
|
+
checksumManifestDigest?: string;
|
|
151
|
+
}): CloudCacheRecord;
|
|
152
|
+
/**
|
|
153
|
+
* Lease for same-client concurrency. Succeeds if unheld, expired, or already ours (renewal).
|
|
154
|
+
* Cross-machine safety additionally needs the on-volume lock; this alone does not provide it.
|
|
155
|
+
*/
|
|
156
|
+
acquireCacheLease(key: Pick<CloudCacheRecord, "accountScope" | "datacenterId" | "volumeId" | "cacheKey">, owner: string, ttlSeconds: number): boolean;
|
|
157
|
+
releaseCacheLease(key: Pick<CloudCacheRecord, "accountScope" | "datacenterId" | "volumeId" | "cacheKey">, owner: string): void;
|
|
158
|
+
deleteCache(key: Pick<CloudCacheRecord, "accountScope" | "datacenterId" | "volumeId" | "cacheKey">): void;
|
|
159
|
+
}
|
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import { createRequire } from "node:module";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
const SCHEMA_VERSION = 1;
|
|
6
|
+
/** Loaded lazily so a platform without a better-sqlite3 binary only breaks cloud commands, not the whole CLI. */
|
|
7
|
+
function loadDatabase() {
|
|
8
|
+
return createRequire(import.meta.url)("better-sqlite3");
|
|
9
|
+
}
|
|
10
|
+
const RESOURCE_NAME_PREFIX = "moldesk";
|
|
11
|
+
function nowIso() {
|
|
12
|
+
return new Date().toISOString();
|
|
13
|
+
}
|
|
14
|
+
/** `moldesk-<client8>-<run8>-<role>`: discovery hint only; ownership is decided by journal match. */
|
|
15
|
+
export function cloudResourceName(clientId, runId, role) {
|
|
16
|
+
const client = clientId.replace(/-/g, "").slice(0, 8);
|
|
17
|
+
// Run IDs end in a UUID; use its first block so the name stays short and stable.
|
|
18
|
+
const run = (runId.split("-").slice(-5)[0] ?? runId).slice(0, 8);
|
|
19
|
+
return `${RESOURCE_NAME_PREFIX}-${client}-${run}-${role}`;
|
|
20
|
+
}
|
|
21
|
+
function parseResourceName(name) {
|
|
22
|
+
const match = /^moldesk-([0-9a-f]{8})-([0-9a-f]{8})-(cpu|gpu|volume)$/.exec(name);
|
|
23
|
+
return match ? { client: match[1], run: match[2], role: match[3] } : undefined;
|
|
24
|
+
}
|
|
25
|
+
function toJob(row) {
|
|
26
|
+
return {
|
|
27
|
+
runId: row.run_id,
|
|
28
|
+
accountScope: row.account_scope,
|
|
29
|
+
quoteDigest: row.quote_digest,
|
|
30
|
+
cachePolicy: row.cache_policy,
|
|
31
|
+
stage: row.stage,
|
|
32
|
+
status: row.status,
|
|
33
|
+
gpuId: row.gpu_id ?? null,
|
|
34
|
+
datacenterId: row.datacenter_id ?? null,
|
|
35
|
+
volumeId: row.volume_id ?? null,
|
|
36
|
+
deadlineAt: row.deadline_at,
|
|
37
|
+
manifestDigest: row.manifest_digest,
|
|
38
|
+
runtimeDigest: row.runtime_digest ?? null,
|
|
39
|
+
transferState: row.transfer_state ?? null,
|
|
40
|
+
error: row.error_code ? { code: row.error_code, message: row.error_message } : null,
|
|
41
|
+
createdAt: row.created_at,
|
|
42
|
+
updatedAt: row.updated_at,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function toResource(row) {
|
|
46
|
+
return {
|
|
47
|
+
intentId: row.intent_id,
|
|
48
|
+
runId: row.run_id ?? null,
|
|
49
|
+
clientId: row.client_id,
|
|
50
|
+
role: row.role,
|
|
51
|
+
name: row.name,
|
|
52
|
+
providerId: row.provider_id ?? null,
|
|
53
|
+
state: row.state,
|
|
54
|
+
deadlineAt: row.deadline_at ?? null,
|
|
55
|
+
createdAt: row.created_at,
|
|
56
|
+
updatedAt: row.updated_at,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
function toCache(row) {
|
|
60
|
+
return {
|
|
61
|
+
accountScope: row.account_scope,
|
|
62
|
+
datacenterId: row.datacenter_id,
|
|
63
|
+
volumeId: row.volume_id,
|
|
64
|
+
model: row.model,
|
|
65
|
+
modelVersion: row.model_version,
|
|
66
|
+
cacheKey: row.cache_key,
|
|
67
|
+
layoutRevision: row.layout_revision,
|
|
68
|
+
manifestDigest: row.manifest_digest,
|
|
69
|
+
cachePath: row.cache_path,
|
|
70
|
+
sizeBytes: row.size_bytes ?? null,
|
|
71
|
+
checksumManifestDigest: row.checksum_manifest_digest ?? null,
|
|
72
|
+
state: row.state,
|
|
73
|
+
leaseOwner: row.lease_owner ?? null,
|
|
74
|
+
leaseExpiresAt: row.lease_expires_at ?? null,
|
|
75
|
+
createdAt: row.created_at,
|
|
76
|
+
lastUsedAt: row.last_used_at,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
export class CloudJournal {
|
|
80
|
+
db;
|
|
81
|
+
clientId;
|
|
82
|
+
now;
|
|
83
|
+
constructor(db,
|
|
84
|
+
/** Stable per-installation identity used in resource names. */
|
|
85
|
+
clientId, now) {
|
|
86
|
+
this.db = db;
|
|
87
|
+
this.clientId = clientId;
|
|
88
|
+
this.now = now;
|
|
89
|
+
}
|
|
90
|
+
static open(paths, options = {}) {
|
|
91
|
+
fs.mkdirSync(paths.state, { recursive: true });
|
|
92
|
+
return CloudJournal.openFile(path.join(paths.state, "cloud.sqlite"), options);
|
|
93
|
+
}
|
|
94
|
+
static openFile(file, options = {}) {
|
|
95
|
+
const db = new (loadDatabase())(file);
|
|
96
|
+
db.pragma("journal_mode = WAL");
|
|
97
|
+
db.pragma("busy_timeout = 5000");
|
|
98
|
+
db.pragma("foreign_keys = ON");
|
|
99
|
+
CloudJournal.migrate(db);
|
|
100
|
+
const existing = db.prepare("SELECT value FROM meta WHERE key = 'client_id'").get();
|
|
101
|
+
let clientId = existing?.value;
|
|
102
|
+
if (!clientId) {
|
|
103
|
+
clientId = randomUUID();
|
|
104
|
+
db.prepare("INSERT OR IGNORE INTO meta (key, value) VALUES ('client_id', ?)").run(clientId);
|
|
105
|
+
clientId = db.prepare("SELECT value FROM meta WHERE key = 'client_id'").get().value;
|
|
106
|
+
}
|
|
107
|
+
return new CloudJournal(db, clientId, options.now ?? nowIso);
|
|
108
|
+
}
|
|
109
|
+
static migrate(db) {
|
|
110
|
+
const version = db.pragma("user_version", { simple: true });
|
|
111
|
+
if (version > SCHEMA_VERSION) {
|
|
112
|
+
throw new Error(`cloud.sqlite schema version ${version} is newer than this MoleculeDesk supports (${SCHEMA_VERSION}).`);
|
|
113
|
+
}
|
|
114
|
+
if (version === SCHEMA_VERSION)
|
|
115
|
+
return;
|
|
116
|
+
db.transaction(() => {
|
|
117
|
+
db.exec(`
|
|
118
|
+
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
119
|
+
CREATE TABLE IF NOT EXISTS cloud_jobs (
|
|
120
|
+
run_id TEXT PRIMARY KEY,
|
|
121
|
+
account_scope TEXT NOT NULL,
|
|
122
|
+
quote_digest TEXT NOT NULL,
|
|
123
|
+
cache_policy TEXT NOT NULL CHECK (cache_policy IN ('ephemeral','persistent')),
|
|
124
|
+
stage TEXT NOT NULL,
|
|
125
|
+
status TEXT NOT NULL,
|
|
126
|
+
gpu_id TEXT,
|
|
127
|
+
datacenter_id TEXT,
|
|
128
|
+
volume_id TEXT,
|
|
129
|
+
deadline_at TEXT NOT NULL,
|
|
130
|
+
manifest_digest TEXT NOT NULL,
|
|
131
|
+
runtime_digest TEXT,
|
|
132
|
+
transfer_state TEXT,
|
|
133
|
+
error_code TEXT,
|
|
134
|
+
error_message TEXT,
|
|
135
|
+
created_at TEXT NOT NULL,
|
|
136
|
+
updated_at TEXT NOT NULL
|
|
137
|
+
);
|
|
138
|
+
CREATE TABLE IF NOT EXISTS cloud_resources (
|
|
139
|
+
intent_id TEXT PRIMARY KEY,
|
|
140
|
+
run_id TEXT,
|
|
141
|
+
client_id TEXT NOT NULL,
|
|
142
|
+
role TEXT NOT NULL CHECK (role IN ('cpu','gpu','volume')),
|
|
143
|
+
name TEXT NOT NULL,
|
|
144
|
+
provider_id TEXT,
|
|
145
|
+
state TEXT NOT NULL CHECK (state IN ('create_pending','active','delete_pending','deleted')),
|
|
146
|
+
deadline_at TEXT,
|
|
147
|
+
created_at TEXT NOT NULL,
|
|
148
|
+
updated_at TEXT NOT NULL
|
|
149
|
+
);
|
|
150
|
+
CREATE INDEX IF NOT EXISTS cloud_resources_state ON cloud_resources (state);
|
|
151
|
+
CREATE INDEX IF NOT EXISTS cloud_resources_provider ON cloud_resources (provider_id);
|
|
152
|
+
CREATE TABLE IF NOT EXISTS cloud_caches (
|
|
153
|
+
account_scope TEXT NOT NULL,
|
|
154
|
+
datacenter_id TEXT NOT NULL,
|
|
155
|
+
volume_id TEXT NOT NULL,
|
|
156
|
+
model TEXT NOT NULL,
|
|
157
|
+
model_version TEXT NOT NULL,
|
|
158
|
+
cache_key TEXT NOT NULL,
|
|
159
|
+
layout_revision TEXT NOT NULL,
|
|
160
|
+
manifest_digest TEXT NOT NULL,
|
|
161
|
+
cache_path TEXT NOT NULL,
|
|
162
|
+
size_bytes INTEGER,
|
|
163
|
+
checksum_manifest_digest TEXT,
|
|
164
|
+
state TEXT NOT NULL CHECK (state IN ('PREPARING','VERIFYING','READY','FAILED')),
|
|
165
|
+
lease_owner TEXT,
|
|
166
|
+
lease_expires_at TEXT,
|
|
167
|
+
created_at TEXT NOT NULL,
|
|
168
|
+
last_used_at TEXT NOT NULL,
|
|
169
|
+
PRIMARY KEY (account_scope, datacenter_id, volume_id, cache_key)
|
|
170
|
+
);
|
|
171
|
+
`);
|
|
172
|
+
db.pragma(`user_version = ${SCHEMA_VERSION}`);
|
|
173
|
+
})();
|
|
174
|
+
}
|
|
175
|
+
close() {
|
|
176
|
+
this.db.close();
|
|
177
|
+
}
|
|
178
|
+
// --- jobs ---------------------------------------------------------------------------------
|
|
179
|
+
createJob(job) {
|
|
180
|
+
const at = this.now();
|
|
181
|
+
this.db
|
|
182
|
+
.prepare(`INSERT INTO cloud_jobs (run_id, account_scope, quote_digest, cache_policy, stage, status, gpu_id, datacenter_id,
|
|
183
|
+
deadline_at, manifest_digest, created_at, updated_at)
|
|
184
|
+
VALUES (@runId, @accountScope, @quoteDigest, @cachePolicy, 'approved', 'active', @gpuId, @datacenterId,
|
|
185
|
+
@deadlineAt, @manifestDigest, @at, @at)`)
|
|
186
|
+
.run({ ...job, gpuId: job.gpuId ?? null, datacenterId: job.datacenterId ?? null, at });
|
|
187
|
+
return this.getJob(job.runId);
|
|
188
|
+
}
|
|
189
|
+
getJob(runId) {
|
|
190
|
+
const row = this.db.prepare("SELECT * FROM cloud_jobs WHERE run_id = ?").get(runId);
|
|
191
|
+
return row ? toJob(row) : undefined;
|
|
192
|
+
}
|
|
193
|
+
updateJob(runId, patch) {
|
|
194
|
+
const columns = {
|
|
195
|
+
stage: patch.stage,
|
|
196
|
+
status: patch.status,
|
|
197
|
+
gpu_id: patch.gpuId,
|
|
198
|
+
datacenter_id: patch.datacenterId,
|
|
199
|
+
volume_id: patch.volumeId,
|
|
200
|
+
runtime_digest: patch.runtimeDigest,
|
|
201
|
+
transfer_state: patch.transferState,
|
|
202
|
+
};
|
|
203
|
+
if (patch.error !== undefined) {
|
|
204
|
+
columns.error_code = patch.error?.code ?? null;
|
|
205
|
+
columns.error_message = patch.error?.message ?? null;
|
|
206
|
+
}
|
|
207
|
+
const entries = Object.entries(columns).filter(([, value]) => value !== undefined);
|
|
208
|
+
const assignments = [...entries.map(([key]) => `${key} = @${key}`), "updated_at = @updated_at"].join(", ");
|
|
209
|
+
const result = this.db
|
|
210
|
+
.prepare(`UPDATE cloud_jobs SET ${assignments} WHERE run_id = @run_id`)
|
|
211
|
+
.run({ ...Object.fromEntries(entries), updated_at: this.now(), run_id: runId });
|
|
212
|
+
if (result.changes === 0)
|
|
213
|
+
throw new Error(`No cloud job for run ${runId}.`);
|
|
214
|
+
return this.getJob(runId);
|
|
215
|
+
}
|
|
216
|
+
/** Jobs that have not reached a terminal status, or whose cleanup is still owed. */
|
|
217
|
+
listOpenJobs() {
|
|
218
|
+
const rows = this.db.prepare("SELECT * FROM cloud_jobs WHERE status IN ('active','cleanup_pending') ORDER BY created_at").all();
|
|
219
|
+
return rows.map(toJob);
|
|
220
|
+
}
|
|
221
|
+
// --- resources ----------------------------------------------------------------------------
|
|
222
|
+
/** Persist intent BEFORE calling the provider. The returned `name` is what must be sent to it. */
|
|
223
|
+
recordResourceIntent(input) {
|
|
224
|
+
const intentId = input.intentId ?? randomUUID();
|
|
225
|
+
const at = this.now();
|
|
226
|
+
const name = cloudResourceName(this.clientId, input.runId ?? intentId, input.role);
|
|
227
|
+
this.db
|
|
228
|
+
.prepare(`INSERT INTO cloud_resources (intent_id, run_id, client_id, role, name, state, deadline_at, created_at, updated_at)
|
|
229
|
+
VALUES (?, ?, ?, ?, ?, 'create_pending', ?, ?, ?)`)
|
|
230
|
+
.run(intentId, input.runId ?? null, this.clientId, input.role, name, input.deadlineAt ?? null, at, at);
|
|
231
|
+
return this.getResource(intentId);
|
|
232
|
+
}
|
|
233
|
+
/** Record the provider ID immediately after the create response. */
|
|
234
|
+
recordResourceCreated(intentId, providerId) {
|
|
235
|
+
return this.transitionResource(intentId, ["create_pending"], "active", providerId);
|
|
236
|
+
}
|
|
237
|
+
/** Persist deletion intent BEFORE calling the provider's delete. */
|
|
238
|
+
markResourceDeletePending(intentId) {
|
|
239
|
+
return this.transitionResource(intentId, ["create_pending", "active", "delete_pending"], "delete_pending");
|
|
240
|
+
}
|
|
241
|
+
/** Call only after the provider confirmed deletion or confirmed absence. */
|
|
242
|
+
markResourceDeleted(intentId) {
|
|
243
|
+
return this.transitionResource(intentId, ["create_pending", "active", "delete_pending", "deleted"], "deleted");
|
|
244
|
+
}
|
|
245
|
+
transitionResource(intentId, from, to, providerId) {
|
|
246
|
+
const current = this.getResource(intentId);
|
|
247
|
+
if (!current)
|
|
248
|
+
throw new Error(`No cloud resource intent ${intentId}.`);
|
|
249
|
+
if (!from.includes(current.state)) {
|
|
250
|
+
throw new Error(`Cannot move cloud resource ${intentId} from ${current.state} to ${to}.`);
|
|
251
|
+
}
|
|
252
|
+
this.db
|
|
253
|
+
.prepare("UPDATE cloud_resources SET state = ?, provider_id = COALESCE(?, provider_id), updated_at = ? WHERE intent_id = ?")
|
|
254
|
+
.run(to, providerId ?? null, this.now(), intentId);
|
|
255
|
+
return this.getResource(intentId);
|
|
256
|
+
}
|
|
257
|
+
getResource(intentId) {
|
|
258
|
+
const row = this.db.prepare("SELECT * FROM cloud_resources WHERE intent_id = ?").get(intentId);
|
|
259
|
+
return row ? toResource(row) : undefined;
|
|
260
|
+
}
|
|
261
|
+
/** Authorization check before any delete: only a journal row owned by this client may be deleted. */
|
|
262
|
+
findOwnedByProviderId(providerId) {
|
|
263
|
+
const row = this.db
|
|
264
|
+
.prepare("SELECT * FROM cloud_resources WHERE provider_id = ? AND client_id = ? AND state != 'deleted'")
|
|
265
|
+
.get(providerId, this.clientId);
|
|
266
|
+
return row ? toResource(row) : undefined;
|
|
267
|
+
}
|
|
268
|
+
listResources(filter = {}) {
|
|
269
|
+
const rows = this.db.prepare("SELECT * FROM cloud_resources ORDER BY created_at").all();
|
|
270
|
+
return rows
|
|
271
|
+
.map(toResource)
|
|
272
|
+
.filter((row) => (filter.runId === undefined || row.runId === filter.runId) && (filter.states === undefined || filter.states.includes(row.state)));
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Compare the journal with a provider listing. Pure bookkeeping plus journal updates; it never
|
|
276
|
+
* calls the provider and never deletes anything. Names are discovery hints only: a provider
|
|
277
|
+
* resource that merely looks ours is reported as orphaned, not adopted by guesswork, unless it
|
|
278
|
+
* matches a `create_pending` row's exact name (an ambiguous create that did take effect).
|
|
279
|
+
*/
|
|
280
|
+
reconcile(provider) {
|
|
281
|
+
const report = { orphaned: [], vanished: [], adopted: [], abandoned: [], cleanupOwed: [] };
|
|
282
|
+
const byProviderId = new Map(provider.map((resource) => [resource.providerId, resource]));
|
|
283
|
+
const knownProviderIds = new Set();
|
|
284
|
+
const clientPrefix = this.clientId.replace(/-/g, "").slice(0, 8);
|
|
285
|
+
this.db.transaction(() => {
|
|
286
|
+
for (const row of this.listResources({ states: ["create_pending", "active", "delete_pending"] })) {
|
|
287
|
+
if (row.state === "create_pending") {
|
|
288
|
+
const matches = provider.filter((candidate) => candidate.name === row.name && candidate.role === row.role && !this.findOwnedByProviderId(candidate.providerId));
|
|
289
|
+
if (matches.length === 1) {
|
|
290
|
+
knownProviderIds.add(matches[0].providerId);
|
|
291
|
+
report.adopted.push(this.recordResourceCreated(row.intentId, matches[0].providerId));
|
|
292
|
+
}
|
|
293
|
+
else if (matches.length === 0) {
|
|
294
|
+
report.abandoned.push(this.markResourceDeleted(row.intentId));
|
|
295
|
+
}
|
|
296
|
+
else {
|
|
297
|
+
// Several candidates share the name: cannot decide ownership safely; leave pending and flag.
|
|
298
|
+
for (const match of matches)
|
|
299
|
+
report.orphaned.push(match);
|
|
300
|
+
}
|
|
301
|
+
continue;
|
|
302
|
+
}
|
|
303
|
+
if (row.providerId)
|
|
304
|
+
knownProviderIds.add(row.providerId);
|
|
305
|
+
if (row.providerId && !byProviderId.has(row.providerId)) {
|
|
306
|
+
report.vanished.push(this.markResourceDeleted(row.intentId));
|
|
307
|
+
continue;
|
|
308
|
+
}
|
|
309
|
+
const overdue = row.state === "active" && row.deadlineAt !== null && row.deadlineAt <= this.now();
|
|
310
|
+
if (row.state === "delete_pending" || overdue)
|
|
311
|
+
report.cleanupOwed.push(row);
|
|
312
|
+
}
|
|
313
|
+
})();
|
|
314
|
+
for (const resource of provider) {
|
|
315
|
+
if (knownProviderIds.has(resource.providerId) || this.findOwnedByProviderId(resource.providerId))
|
|
316
|
+
continue;
|
|
317
|
+
if (report.orphaned.some((entry) => entry.providerId === resource.providerId))
|
|
318
|
+
continue;
|
|
319
|
+
const parsed = parseResourceName(resource.name);
|
|
320
|
+
if (parsed && parsed.client === clientPrefix)
|
|
321
|
+
report.orphaned.push(resource);
|
|
322
|
+
}
|
|
323
|
+
return report;
|
|
324
|
+
}
|
|
325
|
+
// --- caches -------------------------------------------------------------------------------
|
|
326
|
+
upsertCache(cache) {
|
|
327
|
+
const at = this.now();
|
|
328
|
+
this.db
|
|
329
|
+
.prepare(`INSERT INTO cloud_caches (account_scope, datacenter_id, volume_id, model, model_version, cache_key, layout_revision,
|
|
330
|
+
manifest_digest, cache_path, size_bytes, checksum_manifest_digest, state, created_at, last_used_at)
|
|
331
|
+
VALUES (@accountScope, @datacenterId, @volumeId, @model, @modelVersion, @cacheKey, @layoutRevision,
|
|
332
|
+
@manifestDigest, @cachePath, @sizeBytes, @checksumManifestDigest, @state, @at, @at)
|
|
333
|
+
ON CONFLICT (account_scope, datacenter_id, volume_id, cache_key) DO UPDATE SET
|
|
334
|
+
manifest_digest = excluded.manifest_digest, cache_path = excluded.cache_path, last_used_at = excluded.last_used_at`)
|
|
335
|
+
.run({ ...cache, state: cache.state ?? "PREPARING", sizeBytes: cache.sizeBytes ?? null, checksumManifestDigest: cache.checksumManifestDigest ?? null, at });
|
|
336
|
+
return this.getCache(cache.accountScope, cache.datacenterId, cache.volumeId, cache.cacheKey);
|
|
337
|
+
}
|
|
338
|
+
getCache(accountScope, datacenterId, volumeId, cacheKey) {
|
|
339
|
+
const row = this.db
|
|
340
|
+
.prepare("SELECT * FROM cloud_caches WHERE account_scope = ? AND datacenter_id = ? AND volume_id = ? AND cache_key = ?")
|
|
341
|
+
.get(accountScope, datacenterId, volumeId, cacheKey);
|
|
342
|
+
return row ? toCache(row) : undefined;
|
|
343
|
+
}
|
|
344
|
+
listCaches() {
|
|
345
|
+
return this.db.prepare("SELECT * FROM cloud_caches ORDER BY created_at").all().map(toCache);
|
|
346
|
+
}
|
|
347
|
+
/** Allowed: PREPARING → VERIFYING → READY, and any → FAILED. READY is never set without verification. */
|
|
348
|
+
setCacheState(key, next, details = {}) {
|
|
349
|
+
const current = this.getCache(key.accountScope, key.datacenterId, key.volumeId, key.cacheKey);
|
|
350
|
+
if (!current)
|
|
351
|
+
throw new Error("No such cloud cache.");
|
|
352
|
+
const allowed = {
|
|
353
|
+
PREPARING: ["VERIFYING", "FAILED"],
|
|
354
|
+
VERIFYING: ["READY", "FAILED"],
|
|
355
|
+
READY: ["FAILED"],
|
|
356
|
+
FAILED: ["PREPARING"],
|
|
357
|
+
};
|
|
358
|
+
if (!allowed[current.state].includes(next))
|
|
359
|
+
throw new Error(`Cannot move cloud cache from ${current.state} to ${next}.`);
|
|
360
|
+
if (next === "READY" && !(details.checksumManifestDigest ?? current.checksumManifestDigest)) {
|
|
361
|
+
throw new Error("A cache cannot become READY without a checksum manifest digest.");
|
|
362
|
+
}
|
|
363
|
+
this.db
|
|
364
|
+
.prepare(`UPDATE cloud_caches SET state = ?, size_bytes = COALESCE(?, size_bytes),
|
|
365
|
+
checksum_manifest_digest = COALESCE(?, checksum_manifest_digest), last_used_at = ?
|
|
366
|
+
WHERE account_scope = ? AND datacenter_id = ? AND volume_id = ? AND cache_key = ?`)
|
|
367
|
+
.run(next, details.sizeBytes ?? null, details.checksumManifestDigest ?? null, this.now(), key.accountScope, key.datacenterId, key.volumeId, key.cacheKey);
|
|
368
|
+
return this.getCache(key.accountScope, key.datacenterId, key.volumeId, key.cacheKey);
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* Lease for same-client concurrency. Succeeds if unheld, expired, or already ours (renewal).
|
|
372
|
+
* Cross-machine safety additionally needs the on-volume lock; this alone does not provide it.
|
|
373
|
+
*/
|
|
374
|
+
acquireCacheLease(key, owner, ttlSeconds) {
|
|
375
|
+
const at = this.now();
|
|
376
|
+
const expires = new Date(Date.parse(at) + ttlSeconds * 1000).toISOString();
|
|
377
|
+
const result = this.db
|
|
378
|
+
.prepare(`UPDATE cloud_caches SET lease_owner = ?, lease_expires_at = ?
|
|
379
|
+
WHERE account_scope = ? AND datacenter_id = ? AND volume_id = ? AND cache_key = ?
|
|
380
|
+
AND (lease_owner IS NULL OR lease_owner = ? OR lease_expires_at <= ?)`)
|
|
381
|
+
.run(owner, expires, key.accountScope, key.datacenterId, key.volumeId, key.cacheKey, owner, at);
|
|
382
|
+
return result.changes === 1;
|
|
383
|
+
}
|
|
384
|
+
releaseCacheLease(key, owner) {
|
|
385
|
+
this.db
|
|
386
|
+
.prepare(`UPDATE cloud_caches SET lease_owner = NULL, lease_expires_at = NULL
|
|
387
|
+
WHERE account_scope = ? AND datacenter_id = ? AND volume_id = ? AND cache_key = ? AND lease_owner = ?`)
|
|
388
|
+
.run(key.accountScope, key.datacenterId, key.volumeId, key.cacheKey, owner);
|
|
389
|
+
}
|
|
390
|
+
deleteCache(key) {
|
|
391
|
+
this.db
|
|
392
|
+
.prepare("DELETE FROM cloud_caches WHERE account_scope = ? AND datacenter_id = ? AND volume_id = ? AND cache_key = ?")
|
|
393
|
+
.run(key.accountScope, key.datacenterId, key.volumeId, key.cacheKey);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|