@fnndsc/salsa 3.5.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/dist/connect/index.d.ts +11 -0
- package/dist/connect/index.js +11 -0
- package/dist/connect/index.js.map +1 -1
- package/dist/dag/feedJoins.js +4 -1
- package/dist/dag/feedJoins.js.map +1 -1
- package/dist/feeds/index.d.ts +15 -2
- package/dist/feeds/index.js.map +1 -1
- package/dist/files/index.d.ts +7 -0
- package/dist/files/index.js +12 -11
- package/dist/files/index.js.map +1 -1
- package/dist/files/regular_content.js +11 -3
- package/dist/files/regular_content.js.map +1 -1
- package/dist/groups/index.d.ts +59 -0
- package/dist/groups/index.js +89 -0
- package/dist/groups/index.js.map +1 -1
- package/dist/groups/membershipRevision.d.ts +19 -0
- package/dist/groups/membershipRevision.js +29 -0
- package/dist/groups/membershipRevision.js.map +1 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/jobs/jobOps.js +45 -2
- package/dist/jobs/jobOps.js.map +1 -1
- package/dist/pipelines/index.d.ts +28 -5
- package/dist/pipelines/index.js +110 -48
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pipelines/invocation.d.ts +80 -0
- package/dist/pipelines/invocation.js +290 -0
- package/dist/pipelines/invocation.js.map +1 -0
- package/dist/pipelines/manifest.d.ts +102 -0
- package/dist/pipelines/manifest.js +204 -0
- package/dist/pipelines/manifest.js.map +1 -0
- package/dist/plugins/index.d.ts +29 -0
- package/dist/plugins/index.js +51 -10
- package/dist/plugins/index.js.map +1 -1
- package/dist/plugins/internal/plugin_executeNewFeed.d.ts +0 -14
- package/dist/plugins/internal/plugin_executeNewFeed.js +3 -3
- package/dist/plugins/internal/plugin_executeNewFeed.js.map +1 -1
- package/dist/plugins/plugin_register.js +5 -5
- package/dist/plugins/plugin_register.js.map +1 -1
- package/dist/plugins/store_import.js +1 -1
- package/dist/plugins/store_import.js.map +1 -1
- package/dist/retrieve/watch.d.ts +114 -0
- package/dist/retrieve/watch.js +360 -0
- package/dist/retrieve/watch.js.map +1 -0
- package/dist/store/index.d.ts +17 -4
- package/dist/store/index.js +9 -4
- package/dist/store/index.js.map +1 -1
- package/dist/users/index.d.ts +36 -0
- package/dist/users/index.js +50 -0
- package/dist/users/index.js.map +1 -0
- package/dist/vfs/dispatcher.d.ts +11 -0
- package/dist/vfs/dispatcher.js +37 -6
- package/dist/vfs/dispatcher.js.map +1 -1
- package/dist/vfs/index.d.ts +1 -0
- package/dist/vfs/index.js +3 -0
- package/dist/vfs/index.js.map +1 -1
- package/dist/vfs/provider.d.ts +13 -0
- package/dist/vfs/providers/etc.d.ts +2 -0
- package/dist/vfs/providers/etc.js +41 -2
- package/dist/vfs/providers/etc.js.map +1 -1
- package/dist/vfs/providers/native.js +41 -7
- package/dist/vfs/providers/native.js.map +1 -1
- package/dist/vfs/providers/pacs.d.ts +13 -1
- package/dist/vfs/providers/pacs.js +86 -107
- package/dist/vfs/providers/pacs.js.map +1 -1
- package/dist/vfs/providers/pacsHelpers.js +9 -5
- package/dist/vfs/providers/pacsHelpers.js.map +1 -1
- package/dist/vfs/providers/proc.d.ts +61 -6
- package/dist/vfs/providers/proc.js +445 -100
- package/dist/vfs/providers/proc.js.map +1 -1
- package/package.json +11 -8
|
@@ -21,50 +21,83 @@ exports.procFeed_ensureLoaded = procFeed_ensureLoaded;
|
|
|
21
21
|
exports.feedMeta_ensure = feedMeta_ensure;
|
|
22
22
|
exports.feedStatus_refresh = feedStatus_refresh;
|
|
23
23
|
exports.procTopology_warmup = procTopology_warmup;
|
|
24
|
+
exports.procTopology_reconcileFeeds = procTopology_reconcileFeeds;
|
|
25
|
+
exports.procTopology_retry = procTopology_retry;
|
|
26
|
+
exports.procTopology_status = procTopology_status;
|
|
27
|
+
exports.procTopology_await = procTopology_await;
|
|
24
28
|
exports.procCache_refresh = procCache_refresh;
|
|
25
29
|
const cumin_1 = require("@fnndsc/cumin");
|
|
26
30
|
const index_js_1 = require("../../jobs/index.js");
|
|
27
31
|
/** Virtual filenames inside each instance directory. */
|
|
28
|
-
const INSTANCE_FILES = new Set(['status', 'params', 'log']);
|
|
32
|
+
const INSTANCE_FILES = new Set(['status', 'params', 'log', 'data']);
|
|
29
33
|
/** Virtual filenames inside each feed directory. */
|
|
30
34
|
const FEED_FILES = new Set(['status', 'title']);
|
|
35
|
+
const PROC_JOBS_PREFIX = '/proc/jobs';
|
|
31
36
|
const PAGE = 100;
|
|
37
|
+
let procTopologyPromise = null;
|
|
38
|
+
let procTopologyFailure;
|
|
39
|
+
let procTopologyResumeState = null;
|
|
40
|
+
let procTopologyScopedResumeFeedIDs = null;
|
|
41
|
+
/** Converts a CUBE feed row into the cache's feed model. */
|
|
42
|
+
function procFeed_create(feed) {
|
|
43
|
+
return {
|
|
44
|
+
id: Number(feed.id),
|
|
45
|
+
title: String(feed.name),
|
|
46
|
+
ownerUsername: String(feed.owner_username ?? ''),
|
|
47
|
+
public: Boolean(feed.public ?? false),
|
|
48
|
+
creationDate: String(feed.creation_date ?? ''),
|
|
49
|
+
finishedJobs: Number(feed.finished_jobs ?? 0),
|
|
50
|
+
erroredJobs: Number(feed.errored_jobs ?? 0),
|
|
51
|
+
startedJobs: Number(feed.started_jobs ?? 0),
|
|
52
|
+
scheduledJobs: Number(feed.scheduled_jobs ?? 0),
|
|
53
|
+
cancelledJobs: Number(feed.cancelled_jobs ?? 0),
|
|
54
|
+
createdJobs: Number(feed.created_jobs ?? 0),
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
/** Indexes one complete paginated CUBE feed collection into the shared cache. */
|
|
58
|
+
async function procFeeds_index(page_fetch, indexed) {
|
|
59
|
+
let offset = 0;
|
|
60
|
+
let total = 0;
|
|
61
|
+
const seenFeedIDs = new Set();
|
|
62
|
+
while (true) {
|
|
63
|
+
const page = await page_fetch({ limit: PAGE, offset });
|
|
64
|
+
const chunk = page.data ?? [];
|
|
65
|
+
if (offset === 0 && typeof page.totalCount === 'number' && page.totalCount >= 0) {
|
|
66
|
+
total = page.totalCount;
|
|
67
|
+
}
|
|
68
|
+
for (const feed of chunk) {
|
|
69
|
+
const procFeed = procFeed_create(feed);
|
|
70
|
+
indexed.set(procFeed.id, procFeed);
|
|
71
|
+
seenFeedIDs.add(Number(feed.id));
|
|
72
|
+
}
|
|
73
|
+
if (chunk.length === 0 || (total > 0 && seenFeedIDs.size >= total))
|
|
74
|
+
break;
|
|
75
|
+
if (total === 0 && chunk.length < PAGE)
|
|
76
|
+
break;
|
|
77
|
+
offset += chunk.length;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
32
80
|
// ── Cache build ────────────────────────────────────────────────────────────
|
|
33
81
|
/**
|
|
34
|
-
* Builds the feed index (fast). Fetches
|
|
82
|
+
* Builds the feed index (fast). Fetches owned/shared and public feeds with job counters.
|
|
35
83
|
* Instance topology is loaded separately via procTopology_warmup().
|
|
36
84
|
*/
|
|
37
85
|
async function procCache_build() {
|
|
38
86
|
const cache = (0, cumin_1.procCache_get)();
|
|
39
|
-
cache.cache_clear();
|
|
40
87
|
const client = await cumin_1.chrisConnection.client_get();
|
|
41
88
|
if (!client) {
|
|
42
89
|
cumin_1.errorStack.stack_push('error', 'procCache_build: not connected');
|
|
43
|
-
|
|
90
|
+
throw new Error('not connected');
|
|
44
91
|
}
|
|
45
92
|
const typedClient = client;
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
for (const f of chunk) {
|
|
51
|
-
cache.feed_add({
|
|
52
|
-
id: Number(f.id),
|
|
53
|
-
title: String(f.name),
|
|
54
|
-
creationDate: String(f.creation_date ?? ''),
|
|
55
|
-
finishedJobs: Number(f.finished_jobs ?? 0),
|
|
56
|
-
erroredJobs: Number(f.errored_jobs ?? 0),
|
|
57
|
-
startedJobs: Number(f.started_jobs ?? 0),
|
|
58
|
-
scheduledJobs: Number(f.scheduled_jobs ?? 0),
|
|
59
|
-
cancelledJobs: Number(f.cancelled_jobs ?? 0),
|
|
60
|
-
createdJobs: Number(f.created_jobs ?? 0),
|
|
61
|
-
});
|
|
62
|
-
}
|
|
63
|
-
if (chunk.length < PAGE)
|
|
64
|
-
break;
|
|
65
|
-
feedOffset += PAGE;
|
|
93
|
+
const indexed = new Map();
|
|
94
|
+
await procFeeds_index(typedClient.getFeeds.bind(typedClient), indexed);
|
|
95
|
+
if (typedClient.getPublicFeeds) {
|
|
96
|
+
await procFeeds_index(typedClient.getPublicFeeds.bind(typedClient), indexed);
|
|
66
97
|
}
|
|
98
|
+
const reconciliationTargets = cache.feeds_reconcile(Array.from(indexed.values()));
|
|
67
99
|
cache.built_set();
|
|
100
|
+
return reconciliationTargets;
|
|
68
101
|
}
|
|
69
102
|
/** Ensures the feed index is built, building it on first access. */
|
|
70
103
|
async function cache_ensure() {
|
|
@@ -80,21 +113,28 @@ async function feedInstances_load(feedID) {
|
|
|
80
113
|
const cache = (0, cumin_1.procCache_get)();
|
|
81
114
|
const client = await cumin_1.chrisConnection.client_get();
|
|
82
115
|
if (!client)
|
|
83
|
-
|
|
116
|
+
throw new Error('not connected');
|
|
84
117
|
const typedClient = client;
|
|
85
|
-
const instances =
|
|
118
|
+
const instances = new Map();
|
|
86
119
|
let offset = 0;
|
|
120
|
+
let total = 0;
|
|
87
121
|
while (true) {
|
|
88
122
|
const page = await typedClient.getPluginInstances({
|
|
89
123
|
feed_id: feedID, limit: PAGE, offset,
|
|
90
124
|
});
|
|
91
125
|
const chunk = page.data ?? [];
|
|
92
|
-
|
|
93
|
-
|
|
126
|
+
if (offset === 0 && typeof page.totalCount === 'number' && page.totalCount >= 0) {
|
|
127
|
+
total = page.totalCount;
|
|
128
|
+
}
|
|
129
|
+
for (const instance of chunk)
|
|
130
|
+
instances.set(Number(instance.id), instance);
|
|
131
|
+
if (chunk.length === 0 || (total > 0 && instances.size >= total))
|
|
94
132
|
break;
|
|
95
|
-
|
|
133
|
+
if (total === 0 && chunk.length < PAGE)
|
|
134
|
+
break;
|
|
135
|
+
offset += chunk.length;
|
|
96
136
|
}
|
|
97
|
-
for (const inst of instances) {
|
|
137
|
+
for (const inst of instances.values()) {
|
|
98
138
|
const prevID = (inst.previous_id !== null && inst.previous_id !== undefined)
|
|
99
139
|
? Number(inst.previous_id)
|
|
100
140
|
: null;
|
|
@@ -105,6 +145,7 @@ async function feedInstances_load(feedID) {
|
|
|
105
145
|
pluginName: String(inst.plugin_name),
|
|
106
146
|
pluginType: inst.plugin_type !== undefined ? String(inst.plugin_type) : undefined,
|
|
107
147
|
params: null,
|
|
148
|
+
outputPath: outputPath_normalize(inst.output_path) ?? undefined,
|
|
108
149
|
status: String(inst.status ?? 'unknown'),
|
|
109
150
|
});
|
|
110
151
|
}
|
|
@@ -124,8 +165,12 @@ async function feedInstances_ensureLoaded(feedID) {
|
|
|
124
165
|
return inflight;
|
|
125
166
|
const promise = feedInstances_load(feedID);
|
|
126
167
|
cache.loading_set(feedID, promise);
|
|
127
|
-
|
|
128
|
-
|
|
168
|
+
try {
|
|
169
|
+
await promise;
|
|
170
|
+
}
|
|
171
|
+
finally {
|
|
172
|
+
cache.loading_clear(feedID);
|
|
173
|
+
}
|
|
129
174
|
}
|
|
130
175
|
// ── Aggregate status ───────────────────────────────────────────────────────
|
|
131
176
|
/** Derives aggregate feed status from stored job counters — no API call. */
|
|
@@ -143,7 +188,14 @@ function feedStatus_derive(feed) {
|
|
|
143
188
|
}
|
|
144
189
|
// ── Path parsing ───────────────────────────────────────────────────────────
|
|
145
190
|
function procPath_parse(pathStr) {
|
|
146
|
-
|
|
191
|
+
let relativePath = pathStr;
|
|
192
|
+
if (pathStr === PROC_JOBS_PREFIX) {
|
|
193
|
+
relativePath = '';
|
|
194
|
+
}
|
|
195
|
+
else if (pathStr.startsWith(`${PROC_JOBS_PREFIX}/`)) {
|
|
196
|
+
relativePath = pathStr.slice(PROC_JOBS_PREFIX.length + 1);
|
|
197
|
+
}
|
|
198
|
+
const parts = relativePath.split('/').filter(Boolean);
|
|
147
199
|
let feedID = null;
|
|
148
200
|
let instanceID = null;
|
|
149
201
|
let virtualFile = null;
|
|
@@ -178,6 +230,79 @@ function params_render(inst) {
|
|
|
178
230
|
.map(([k, v]) => `${k}=${String(v)}`)
|
|
179
231
|
.join('\n');
|
|
180
232
|
}
|
|
233
|
+
/**
|
|
234
|
+
* Normalizes CUBE's output-path spelling into an absolute CFS path.
|
|
235
|
+
*
|
|
236
|
+
* @param value - Raw `output_path` value from a plugin-instance resource.
|
|
237
|
+
* @returns Absolute CFS path, or null when no usable path was supplied.
|
|
238
|
+
*/
|
|
239
|
+
function outputPath_normalize(value) {
|
|
240
|
+
if (typeof value !== 'string' || value.trim() === '')
|
|
241
|
+
return null;
|
|
242
|
+
const clean = value.trim();
|
|
243
|
+
return clean.startsWith('/') ? clean : `/${clean}`;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Resolves and memoizes the authoritative CFS output directory for one job.
|
|
247
|
+
*
|
|
248
|
+
* @param instanceID - Plugin-instance ID whose output path is needed.
|
|
249
|
+
* @returns Absolute output path, or null when CUBE has not recorded one.
|
|
250
|
+
*/
|
|
251
|
+
async function instanceOutputPath_ensure(instanceID) {
|
|
252
|
+
const cache = (0, cumin_1.procCache_get)();
|
|
253
|
+
const inst = cache.instance_get(instanceID);
|
|
254
|
+
if (!inst)
|
|
255
|
+
return null;
|
|
256
|
+
if (inst.outputPath !== undefined)
|
|
257
|
+
return inst.outputPath;
|
|
258
|
+
const client = await cumin_1.chrisConnection.client_get();
|
|
259
|
+
if (!client)
|
|
260
|
+
return null;
|
|
261
|
+
try {
|
|
262
|
+
const resource = await client
|
|
263
|
+
.getPluginInstance(instanceID);
|
|
264
|
+
const outputPath = outputPath_normalize(resource?.data?.['output_path']);
|
|
265
|
+
cache.outputPath_update(instanceID, outputPath);
|
|
266
|
+
return outputPath;
|
|
267
|
+
}
|
|
268
|
+
catch (error) {
|
|
269
|
+
// "No CFS output path" downstream must not mask a failed fetch.
|
|
270
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
271
|
+
cumin_1.errorStack.stack_push("warning", `Could not fetch output path for instance ${instanceID}: ${msg}`);
|
|
272
|
+
return null;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* Reads every page of effective parameter values from one plugin instance.
|
|
277
|
+
*
|
|
278
|
+
* @param resource - Detail resource for the instance being inspected.
|
|
279
|
+
* @returns Parameter names mapped to their recorded invocation values.
|
|
280
|
+
*/
|
|
281
|
+
async function instanceParams_fetch(resource) {
|
|
282
|
+
const params = {};
|
|
283
|
+
let offset = 0;
|
|
284
|
+
let total = 0;
|
|
285
|
+
let fetched = 0;
|
|
286
|
+
while (true) {
|
|
287
|
+
const page = await resource.getParameters({ limit: PAGE, offset });
|
|
288
|
+
const items = page.data
|
|
289
|
+
?? (page.getItems ? page.getItems().map((item) => item.data) : []);
|
|
290
|
+
if (offset === 0 && typeof page.totalCount === 'number' && page.totalCount >= 0) {
|
|
291
|
+
total = page.totalCount;
|
|
292
|
+
}
|
|
293
|
+
for (const item of items) {
|
|
294
|
+
if (item.param_name)
|
|
295
|
+
params[item.param_name] = item.value;
|
|
296
|
+
}
|
|
297
|
+
fetched += items.length;
|
|
298
|
+
if (items.length === 0 || (total > 0 && fetched >= total))
|
|
299
|
+
break;
|
|
300
|
+
if (total === 0 && items.length < PAGE)
|
|
301
|
+
break;
|
|
302
|
+
offset += items.length;
|
|
303
|
+
}
|
|
304
|
+
return params;
|
|
305
|
+
}
|
|
181
306
|
/** Collects all instance IDs for a feed recursively. */
|
|
182
307
|
function getAllInstanceIDs_forFeed(feedID, cache) {
|
|
183
308
|
const result = [];
|
|
@@ -194,13 +319,35 @@ function getAllInstanceIDs_forFeed(feedID, cache) {
|
|
|
194
319
|
* VFS provider exposing running jobs and feeds under the /proc namespace.
|
|
195
320
|
*/
|
|
196
321
|
class ProcVfsProvider {
|
|
197
|
-
prefix =
|
|
322
|
+
prefix = PROC_JOBS_PREFIX;
|
|
323
|
+
/**
|
|
324
|
+
* Resolves the synthetic `data` link for one job only when navigation asks
|
|
325
|
+
* to follow it. This keeps `/proc` topology listings and recursive trees
|
|
326
|
+
* entirely cache-backed.
|
|
327
|
+
*
|
|
328
|
+
* @param pathStr - Absolute `/proc/jobs/.../data` path to resolve.
|
|
329
|
+
* @returns The authoritative CFS output path, or an error when absent.
|
|
330
|
+
*/
|
|
331
|
+
async linkTarget_resolve(pathStr) {
|
|
332
|
+
await cache_ensure();
|
|
333
|
+
const clean = pathStr.replace(/\/$/, '');
|
|
334
|
+
const { feedID, instanceID, virtualFile } = procPath_parse(clean);
|
|
335
|
+
if (feedID === null || instanceID === null || virtualFile !== 'data') {
|
|
336
|
+
cumin_1.errorStack.stack_push('error', `Not a /proc job data link: ${pathStr}`);
|
|
337
|
+
return (0, cumin_1.Err)();
|
|
338
|
+
}
|
|
339
|
+
const outputPath = await instanceOutputPath_ensure(instanceID);
|
|
340
|
+
if (outputPath)
|
|
341
|
+
return (0, cumin_1.Ok)(outputPath);
|
|
342
|
+
cumin_1.errorStack.stack_push('error', `No CFS output path is available for job ${instanceID}`);
|
|
343
|
+
return (0, cumin_1.Err)();
|
|
344
|
+
}
|
|
198
345
|
async list(pathStr, _options) {
|
|
199
346
|
await cache_ensure();
|
|
200
347
|
const cache = (0, cumin_1.procCache_get)();
|
|
201
348
|
const clean = pathStr.replace(/\/$/, '');
|
|
202
349
|
// /proc/jobs — list all feeds with aggregate status from counters
|
|
203
|
-
if (clean ===
|
|
350
|
+
if (clean === PROC_JOBS_PREFIX) {
|
|
204
351
|
const items = cache.feedIDs_get().map((feedID) => {
|
|
205
352
|
const feed = cache.feed_get(feedID);
|
|
206
353
|
const status = feed ? feedStatus_derive(feed) : 'unknown';
|
|
@@ -264,6 +411,15 @@ class ProcVfsProvider {
|
|
|
264
411
|
items.push({ name: 'status', type: 'file', size: 0, owner: '', date: '' });
|
|
265
412
|
items.push({ name: 'params', type: 'file', size: 0, owner: '', date: '' });
|
|
266
413
|
items.push({ name: 'log', type: 'file', size: 0, owner: '', date: '' });
|
|
414
|
+
// `data` is structural: list it without fetching the per-instance detail
|
|
415
|
+
// endpoint. Its target is resolved only when `cd data` follows the link.
|
|
416
|
+
// A known null means CUBE has no output directory, so omit it entirely.
|
|
417
|
+
if (inst.outputPath !== null) {
|
|
418
|
+
const dataLink = { name: 'data', type: 'link', size: 0, owner: '', date: '' };
|
|
419
|
+
if (inst.outputPath)
|
|
420
|
+
dataLink.target = inst.outputPath;
|
|
421
|
+
items.push(dataLink);
|
|
422
|
+
}
|
|
267
423
|
for (const childID of childIDs) {
|
|
268
424
|
const child = cache.instance_get(childID);
|
|
269
425
|
if (!child)
|
|
@@ -314,22 +470,21 @@ class ProcVfsProvider {
|
|
|
314
470
|
}
|
|
315
471
|
if (virtualFile === 'params') {
|
|
316
472
|
if (inst.params === null) {
|
|
317
|
-
//
|
|
473
|
+
// CUBE stores the effective run values in the parameter sub-resource;
|
|
474
|
+
// the instance detail payload itself is operational metadata.
|
|
318
475
|
const client = await cumin_1.chrisConnection.client_get();
|
|
319
476
|
if (client) {
|
|
320
477
|
try {
|
|
321
|
-
const
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
}
|
|
478
|
+
const resource = await client
|
|
479
|
+
.getPluginInstance(instanceID);
|
|
480
|
+
if (resource)
|
|
481
|
+
cache.params_update(instanceID, await instanceParams_fetch(resource));
|
|
482
|
+
}
|
|
483
|
+
catch {
|
|
484
|
+
// Deliberate absorption, adjudicated 2026-08: parameters are a
|
|
485
|
+
// lazy display enrichment; a failed fetch renders as
|
|
486
|
+
// "(no parameters)" and the next read retries.
|
|
331
487
|
}
|
|
332
|
-
catch { /* leave null */ }
|
|
333
488
|
}
|
|
334
489
|
}
|
|
335
490
|
return (0, cumin_1.Ok)(params_render(inst));
|
|
@@ -349,12 +504,21 @@ class ProcVfsProvider {
|
|
|
349
504
|
if (feedID !== null && instanceID === null) {
|
|
350
505
|
const allIDs = getAllInstanceIDs_forFeed(feedID, cache);
|
|
351
506
|
const statusMap = await (0, index_js_1.jobs_statusBatch)(allIDs);
|
|
507
|
+
// Every cancel Result is checked: reporting success while jobs kept
|
|
508
|
+
// running would leave the user believing the feed's work was stopped.
|
|
509
|
+
const failedCancels = [];
|
|
352
510
|
await Promise.all(allIDs.map(async (id) => {
|
|
353
511
|
const s = statusMap.get(id) ?? '';
|
|
354
512
|
if (!(0, cumin_1.status_isTerminal)(s)) {
|
|
355
|
-
await (0, index_js_1.job_cancel)(id);
|
|
513
|
+
const cancelResult = await (0, index_js_1.job_cancel)(id);
|
|
514
|
+
if (!cancelResult.ok)
|
|
515
|
+
failedCancels.push(id);
|
|
356
516
|
}
|
|
357
517
|
}));
|
|
518
|
+
if (failedCancels.length > 0) {
|
|
519
|
+
cumin_1.errorStack.stack_push('error', `rm: failed to cancel ${failedCancels.length} running job(s) in feed ${feedID}: instance(s) ${failedCancels.sort((a, b) => a - b).join(', ')}`);
|
|
520
|
+
return false;
|
|
521
|
+
}
|
|
358
522
|
cache.feed_remove(feedID);
|
|
359
523
|
return true;
|
|
360
524
|
}
|
|
@@ -392,7 +556,7 @@ async function procFeed_ensureLoaded(feedID) {
|
|
|
392
556
|
const cache = (0, cumin_1.procCache_get)();
|
|
393
557
|
if (!cache.feed_get(feedID)) {
|
|
394
558
|
cache.feed_add({
|
|
395
|
-
id: feedID, title: `feed_${feedID}`, creationDate: '',
|
|
559
|
+
id: feedID, title: `feed_${feedID}`, ownerUsername: '', public: false, creationDate: '',
|
|
396
560
|
finishedJobs: 0, erroredJobs: 0, startedJobs: 0,
|
|
397
561
|
scheduledJobs: 0, cancelledJobs: 0, createdJobs: 0,
|
|
398
562
|
});
|
|
@@ -424,23 +588,13 @@ async function feedMeta_ensure(feedID) {
|
|
|
424
588
|
return;
|
|
425
589
|
const client = await cumin_1.chrisConnection.client_get();
|
|
426
590
|
if (!client)
|
|
427
|
-
|
|
591
|
+
throw new Error('not connected');
|
|
428
592
|
const typedClient = client;
|
|
429
593
|
const page = await typedClient.getFeeds({ id: feedID, limit: 1, offset: 0 });
|
|
430
594
|
const f = (page.data ?? [])[0];
|
|
431
595
|
if (!f)
|
|
432
596
|
return;
|
|
433
|
-
cache.feed_add(
|
|
434
|
-
id: Number(f.id),
|
|
435
|
-
title: String(f.name),
|
|
436
|
-
creationDate: String(f.creation_date ?? ''),
|
|
437
|
-
finishedJobs: Number(f.finished_jobs ?? 0),
|
|
438
|
-
erroredJobs: Number(f.errored_jobs ?? 0),
|
|
439
|
-
startedJobs: Number(f.started_jobs ?? 0),
|
|
440
|
-
scheduledJobs: Number(f.scheduled_jobs ?? 0),
|
|
441
|
-
cancelledJobs: Number(f.cancelled_jobs ?? 0),
|
|
442
|
-
createdJobs: Number(f.created_jobs ?? 0),
|
|
443
|
-
});
|
|
597
|
+
cache.feed_add(procFeed_create(f));
|
|
444
598
|
}
|
|
445
599
|
async function feedStatus_refresh(feedID) {
|
|
446
600
|
const cache = (0, cumin_1.procCache_get)();
|
|
@@ -449,79 +603,265 @@ async function feedStatus_refresh(feedID) {
|
|
|
449
603
|
return;
|
|
450
604
|
const typedClient = client;
|
|
451
605
|
let offset = 0;
|
|
606
|
+
let total = 0;
|
|
607
|
+
const seenInstanceIDs = new Set();
|
|
452
608
|
while (true) {
|
|
453
609
|
const page = await typedClient.getPluginInstances({
|
|
454
610
|
feed_id: feedID, limit: PAGE, offset,
|
|
455
611
|
});
|
|
456
612
|
const chunk = page.data ?? [];
|
|
613
|
+
if (offset === 0 && typeof page.totalCount === 'number' && page.totalCount >= 0) {
|
|
614
|
+
total = page.totalCount;
|
|
615
|
+
}
|
|
457
616
|
for (const inst of chunk) {
|
|
458
|
-
|
|
617
|
+
const instanceID = Number(inst.id);
|
|
618
|
+
cache.status_update(instanceID, String(inst.status ?? 'unknown'));
|
|
619
|
+
seenInstanceIDs.add(instanceID);
|
|
459
620
|
}
|
|
460
|
-
if (chunk.length
|
|
621
|
+
if (chunk.length === 0 || (total > 0 && seenInstanceIDs.size >= total))
|
|
461
622
|
break;
|
|
462
|
-
|
|
623
|
+
if (total === 0 && chunk.length < PAGE)
|
|
624
|
+
break;
|
|
625
|
+
offset += chunk.length;
|
|
463
626
|
}
|
|
464
627
|
}
|
|
465
628
|
/**
|
|
466
629
|
* Background topology warm-up — single paginated sweep of all plugin instances
|
|
467
630
|
* across all feeds. Replaces the per-feed fan-out that was O(feeds) round trips.
|
|
468
631
|
*
|
|
469
|
-
*
|
|
470
|
-
*
|
|
471
|
-
*
|
|
632
|
+
* Scoped per-feed loads may run concurrently for navigation; the completed
|
|
633
|
+
* global sweep reconciles the cache to its authoritative instance set.
|
|
634
|
+
*
|
|
635
|
+
* @param state - Private pagination and reconciliation state for this sweep.
|
|
636
|
+
* @returns A promise that settles when the complete topology has been reconciled.
|
|
472
637
|
*/
|
|
473
|
-
async function
|
|
638
|
+
async function procTopology_run(state) {
|
|
474
639
|
await cache_ensure();
|
|
475
640
|
const cache = (0, cumin_1.procCache_get)();
|
|
641
|
+
cache.lifecycle_set('reconciling');
|
|
642
|
+
if (state.total > 0) {
|
|
643
|
+
cache.warmup_progress(state.fetchedInstanceIDs.size, state.total);
|
|
644
|
+
}
|
|
476
645
|
const client = await cumin_1.chrisConnection.client_get();
|
|
477
646
|
if (!client)
|
|
478
|
-
|
|
479
|
-
cache.warmup_progress(0, 0);
|
|
647
|
+
throw new Error('not connected');
|
|
480
648
|
const typedClient = client;
|
|
481
|
-
let offset = 0;
|
|
482
|
-
let loaded = 0;
|
|
483
|
-
const seenFeedIDs = new Set();
|
|
484
649
|
while (true) {
|
|
485
650
|
const page = await typedClient.getPluginInstances({
|
|
486
|
-
limit: PAGE, offset,
|
|
651
|
+
limit: PAGE, offset: state.offset,
|
|
487
652
|
});
|
|
488
653
|
const chunk = page.data ?? [];
|
|
654
|
+
if (state.offset === 0 && typeof page.totalCount === 'number' && page.totalCount >= 0) {
|
|
655
|
+
state.total = page.totalCount;
|
|
656
|
+
if (state.total > 0)
|
|
657
|
+
cache.warmup_progress(0, state.total);
|
|
658
|
+
}
|
|
489
659
|
for (const inst of chunk) {
|
|
660
|
+
const instanceID = Number(inst.id);
|
|
490
661
|
const feedID = Number(inst.feed_id);
|
|
662
|
+
state.fetchedInstanceIDs.add(instanceID);
|
|
491
663
|
const prevID = (inst.previous_id !== null && inst.previous_id !== undefined)
|
|
492
664
|
? Number(inst.previous_id)
|
|
493
665
|
: null;
|
|
494
|
-
if (!cache.feed_get(feedID))
|
|
495
|
-
|
|
496
|
-
id: feedID, title: `feed_${feedID}`, creationDate: '',
|
|
497
|
-
finishedJobs: 0, erroredJobs: 0, startedJobs: 0,
|
|
498
|
-
scheduledJobs: 0, cancelledJobs: 0, createdJobs: 0,
|
|
499
|
-
});
|
|
500
|
-
}
|
|
666
|
+
if (!cache.feed_get(feedID))
|
|
667
|
+
continue;
|
|
501
668
|
cache.instance_add({
|
|
502
|
-
id:
|
|
669
|
+
id: instanceID,
|
|
503
670
|
feedID,
|
|
504
671
|
parentID: prevID,
|
|
505
672
|
pluginName: String(inst.plugin_name),
|
|
506
673
|
pluginType: inst.plugin_type !== undefined ? String(inst.plugin_type) : undefined,
|
|
507
674
|
params: null,
|
|
675
|
+
outputPath: outputPath_normalize(inst.output_path) ?? undefined,
|
|
508
676
|
status: String(inst.status ?? 'unknown'),
|
|
509
677
|
});
|
|
510
|
-
|
|
511
|
-
|
|
678
|
+
state.seenInstanceIDs.add(instanceID);
|
|
679
|
+
state.seenFeedIDs.add(feedID);
|
|
512
680
|
}
|
|
513
|
-
|
|
514
|
-
|
|
681
|
+
if (state.total > 0)
|
|
682
|
+
cache.warmup_progress(state.fetchedInstanceIDs.size, state.total);
|
|
683
|
+
if (chunk.length === 0 || (state.total > 0 && state.fetchedInstanceIDs.size >= state.total))
|
|
515
684
|
break;
|
|
516
|
-
|
|
685
|
+
if (state.total === 0 && chunk.length < PAGE)
|
|
686
|
+
break;
|
|
687
|
+
state.offset += chunk.length;
|
|
517
688
|
}
|
|
518
|
-
|
|
689
|
+
cache.topology_reconcile(state.seenInstanceIDs);
|
|
690
|
+
for (const feedID of state.seenFeedIDs)
|
|
519
691
|
cache.topologyLoaded_mark(feedID);
|
|
692
|
+
if (state.total === 0) {
|
|
693
|
+
cache.warmup_progress(state.fetchedInstanceIDs.size, state.fetchedInstanceIDs.size);
|
|
520
694
|
}
|
|
521
695
|
cache.warmup_complete();
|
|
522
696
|
}
|
|
697
|
+
/**
|
|
698
|
+
* Creates an empty continuation for a fresh global topology sweep.
|
|
699
|
+
*
|
|
700
|
+
* @returns New private pagination and reconciliation state.
|
|
701
|
+
*/
|
|
702
|
+
function procTopologyState_create() {
|
|
703
|
+
return {
|
|
704
|
+
offset: 0,
|
|
705
|
+
total: 0,
|
|
706
|
+
fetchedInstanceIDs: new Set(),
|
|
707
|
+
seenInstanceIDs: new Set(),
|
|
708
|
+
seenFeedIDs: new Set(),
|
|
709
|
+
};
|
|
710
|
+
}
|
|
711
|
+
/**
|
|
712
|
+
* Starts one topology sweep from the supplied private continuation state.
|
|
713
|
+
*
|
|
714
|
+
* @param state - Private pagination and reconciliation state to run or resume.
|
|
715
|
+
* @returns The in-flight topology sweep promise.
|
|
716
|
+
*/
|
|
717
|
+
function procTopology_start(state) {
|
|
718
|
+
procTopologyFailure = undefined;
|
|
719
|
+
procTopologyResumeState = state;
|
|
720
|
+
procTopologyScopedResumeFeedIDs = null;
|
|
721
|
+
const sweep = procTopology_run(state);
|
|
722
|
+
procTopologyPromise = sweep;
|
|
723
|
+
sweep.then(() => {
|
|
724
|
+
if (procTopologyPromise !== sweep)
|
|
725
|
+
return;
|
|
726
|
+
procTopologyPromise = null;
|
|
727
|
+
if ((0, cumin_1.procCache_get)().warmupComplete) {
|
|
728
|
+
procTopologyFailure = undefined;
|
|
729
|
+
procTopologyResumeState = null;
|
|
730
|
+
}
|
|
731
|
+
else {
|
|
732
|
+
procTopologyFailure = 'the topology sweep ended before the index completed';
|
|
733
|
+
}
|
|
734
|
+
}, (error) => {
|
|
735
|
+
if (procTopologyPromise !== sweep)
|
|
736
|
+
return;
|
|
737
|
+
procTopologyPromise = null;
|
|
738
|
+
(0, cumin_1.procCache_get)().warmup_abort();
|
|
739
|
+
procTopologyFailure = error instanceof Error ? error.message : String(error);
|
|
740
|
+
});
|
|
741
|
+
return sweep;
|
|
742
|
+
}
|
|
743
|
+
/**
|
|
744
|
+
* Runs a feed-scoped topology reconciliation selected after checkpoint restore.
|
|
745
|
+
*
|
|
746
|
+
* @param feedIDs - Canonical feed IDs whose topology needs a fresh load.
|
|
747
|
+
* @returns A promise that settles after the supplied feeds are reconciled.
|
|
748
|
+
*/
|
|
749
|
+
async function procTopologyScoped_run(feedIDs) {
|
|
750
|
+
const cache = (0, cumin_1.procCache_get)();
|
|
751
|
+
cache.lifecycle_set('reconciling');
|
|
752
|
+
for (const feedID of feedIDs)
|
|
753
|
+
await procCache_refresh(feedID);
|
|
754
|
+
cache.warmup_complete();
|
|
755
|
+
}
|
|
756
|
+
/**
|
|
757
|
+
* Starts one scoped reconciliation and retains its targets for `proc retry`.
|
|
758
|
+
*
|
|
759
|
+
* @param feedIDs - Canonical feed IDs selected by the fresh feed index.
|
|
760
|
+
* @returns The in-flight scoped reconciliation promise.
|
|
761
|
+
*/
|
|
762
|
+
function procTopologyScoped_start(feedIDs) {
|
|
763
|
+
const targets = [...feedIDs];
|
|
764
|
+
procTopologyFailure = undefined;
|
|
765
|
+
procTopologyResumeState = null;
|
|
766
|
+
procTopologyScopedResumeFeedIDs = targets;
|
|
767
|
+
const sweep = procTopologyScoped_run(targets);
|
|
768
|
+
procTopologyPromise = sweep;
|
|
769
|
+
sweep.then(() => {
|
|
770
|
+
if (procTopologyPromise !== sweep)
|
|
771
|
+
return;
|
|
772
|
+
procTopologyPromise = null;
|
|
773
|
+
if ((0, cumin_1.procCache_get)().warmupComplete) {
|
|
774
|
+
procTopologyFailure = undefined;
|
|
775
|
+
procTopologyScopedResumeFeedIDs = null;
|
|
776
|
+
}
|
|
777
|
+
else {
|
|
778
|
+
procTopologyFailure = 'the scoped topology reconciliation ended before the index completed';
|
|
779
|
+
}
|
|
780
|
+
}, (error) => {
|
|
781
|
+
if (procTopologyPromise !== sweep)
|
|
782
|
+
return;
|
|
783
|
+
procTopologyPromise = null;
|
|
784
|
+
(0, cumin_1.procCache_get)().warmup_abort();
|
|
785
|
+
procTopologyFailure = error instanceof Error ? error.message : String(error);
|
|
786
|
+
});
|
|
787
|
+
return sweep;
|
|
788
|
+
}
|
|
789
|
+
/**
|
|
790
|
+
* Starts or joins the session's global plugin-instance topology sweep.
|
|
791
|
+
*
|
|
792
|
+
* @returns The single in-flight sweep promise.
|
|
793
|
+
*/
|
|
794
|
+
function procTopology_warmup() {
|
|
795
|
+
if (procTopologyPromise)
|
|
796
|
+
return procTopologyPromise;
|
|
797
|
+
return procTopology_start(procTopologyState_create());
|
|
798
|
+
}
|
|
799
|
+
/**
|
|
800
|
+
* Reconciles topology only for feed IDs selected by a restored-cache feed-index
|
|
801
|
+
* comparison. This preserves the checkpointed topology for unchanged terminal
|
|
802
|
+
* feeds while refreshing new, changed, and active feeds from CUBE.
|
|
803
|
+
*
|
|
804
|
+
* @param feedIDs - Canonical CUBE feed IDs whose topology must be reloaded.
|
|
805
|
+
* @returns A promise that resolves after all supplied feeds have been refreshed.
|
|
806
|
+
*/
|
|
807
|
+
function procTopology_reconcileFeeds(feedIDs) {
|
|
808
|
+
if (procTopologyPromise)
|
|
809
|
+
return procTopologyPromise;
|
|
810
|
+
return procTopologyScoped_start(feedIDs);
|
|
811
|
+
}
|
|
812
|
+
/**
|
|
813
|
+
* Continues a failed topology reconciliation from its retained global page or
|
|
814
|
+
* scoped feed set.
|
|
815
|
+
*
|
|
816
|
+
* @returns The resumed sweep promise, or the active sweep when one is running.
|
|
817
|
+
* @throws {Error} When there is no failed sweep continuation to retry.
|
|
818
|
+
*/
|
|
819
|
+
function procTopology_retry() {
|
|
820
|
+
if (procTopologyPromise)
|
|
821
|
+
return procTopologyPromise;
|
|
822
|
+
if (procTopologyFailure === undefined) {
|
|
823
|
+
throw new Error('no failed topology sweep to retry');
|
|
824
|
+
}
|
|
825
|
+
if (procTopologyResumeState)
|
|
826
|
+
return procTopology_start(procTopologyResumeState);
|
|
827
|
+
if (procTopologyScopedResumeFeedIDs)
|
|
828
|
+
return procTopologyScoped_start(procTopologyScopedResumeFeedIDs);
|
|
829
|
+
throw new Error('no failed topology sweep to retry');
|
|
830
|
+
}
|
|
831
|
+
/**
|
|
832
|
+
* Reports the session topology sweep lifecycle.
|
|
833
|
+
*
|
|
834
|
+
* @returns A snapshot of the lifecycle state and any failure reason.
|
|
835
|
+
*/
|
|
836
|
+
function procTopology_status() {
|
|
837
|
+
if (procTopologyPromise)
|
|
838
|
+
return { state: 'running', failure: undefined };
|
|
839
|
+
if (procTopologyFailure !== undefined)
|
|
840
|
+
return { state: 'failed', failure: procTopologyFailure };
|
|
841
|
+
if ((0, cumin_1.procCache_get)().warmupComplete)
|
|
842
|
+
return { state: 'complete', failure: undefined };
|
|
843
|
+
return { state: 'idle', failure: undefined };
|
|
844
|
+
}
|
|
845
|
+
/**
|
|
846
|
+
* Waits for the active topology sweep without starting another.
|
|
847
|
+
*
|
|
848
|
+
* @returns A promise that resolves when the active sweep finishes, or
|
|
849
|
+
* immediately when no sweep is running.
|
|
850
|
+
*/
|
|
851
|
+
async function procTopology_await() {
|
|
852
|
+
const sweep = procTopologyPromise;
|
|
853
|
+
if (sweep)
|
|
854
|
+
await sweep;
|
|
855
|
+
}
|
|
523
856
|
/**
|
|
524
857
|
* Rebuilds the ProcCache, optionally scoped to one feed.
|
|
858
|
+
*
|
|
859
|
+
* A full refresh returns feeds whose topology merits reload after the feed-index
|
|
860
|
+
* comparison. Callers performing an explicit `proc refresh` can ignore that
|
|
861
|
+
* result and run a global topology sweep; restored startup uses it to stay scoped.
|
|
862
|
+
*
|
|
863
|
+
* @param feedID - One feed to force-refresh, or undefined for the full feed index.
|
|
864
|
+
* @returns IDs whose topology is new, changed, or currently active.
|
|
525
865
|
*/
|
|
526
866
|
async function procCache_refresh(feedID) {
|
|
527
867
|
if (feedID !== undefined) {
|
|
@@ -534,23 +874,28 @@ async function procCache_refresh(feedID) {
|
|
|
534
874
|
const page = await typedClient.getFeeds({ id: feedID, limit: 1, offset: 0 });
|
|
535
875
|
const f = (page.data ?? [])[0];
|
|
536
876
|
if (f) {
|
|
537
|
-
cache.feed_add(
|
|
538
|
-
id: Number(f.id),
|
|
539
|
-
title: String(f.name),
|
|
540
|
-
creationDate: String(f.creation_date ?? ''),
|
|
541
|
-
finishedJobs: Number(f.finished_jobs ?? 0),
|
|
542
|
-
erroredJobs: Number(f.errored_jobs ?? 0),
|
|
543
|
-
startedJobs: Number(f.started_jobs ?? 0),
|
|
544
|
-
scheduledJobs: Number(f.scheduled_jobs ?? 0),
|
|
545
|
-
cancelledJobs: Number(f.cancelled_jobs ?? 0),
|
|
546
|
-
createdJobs: Number(f.created_jobs ?? 0),
|
|
547
|
-
});
|
|
877
|
+
cache.feed_add(procFeed_create(f));
|
|
548
878
|
}
|
|
549
879
|
}
|
|
550
880
|
await feedInstances_ensureLoaded(feedID);
|
|
881
|
+
return [feedID];
|
|
551
882
|
}
|
|
552
883
|
else {
|
|
553
|
-
|
|
884
|
+
const activeSweep = procTopologyPromise;
|
|
885
|
+
if (activeSweep) {
|
|
886
|
+
try {
|
|
887
|
+
await activeSweep;
|
|
888
|
+
}
|
|
889
|
+
catch {
|
|
890
|
+
// A full refresh replaces any failed sweep with a new idle lifecycle.
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
procTopologyPromise = null;
|
|
894
|
+
procTopologyFailure = undefined;
|
|
895
|
+
procTopologyResumeState = null;
|
|
896
|
+
procTopologyScopedResumeFeedIDs = null;
|
|
897
|
+
(0, cumin_1.procCache_get)().warmup_reset();
|
|
898
|
+
return procCache_build();
|
|
554
899
|
}
|
|
555
900
|
}
|
|
556
901
|
//# sourceMappingURL=proc.js.map
|