@fnndsc/salsa 3.5.1 → 3.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/connect/index.d.ts +11 -0
- package/dist/connect/index.js +11 -0
- package/dist/connect/index.js.map +1 -1
- package/dist/dag/feedJoins.js +8 -7
- package/dist/dag/feedJoins.js.map +1 -1
- package/dist/feeds/index.d.ts +15 -2
- package/dist/feeds/index.js.map +1 -1
- package/dist/files/index.d.ts +7 -0
- package/dist/files/index.js +12 -11
- package/dist/files/index.js.map +1 -1
- package/dist/files/regular_content.js +11 -3
- package/dist/files/regular_content.js.map +1 -1
- package/dist/groups/index.d.ts +59 -0
- package/dist/groups/index.js +89 -0
- package/dist/groups/index.js.map +1 -1
- package/dist/groups/membershipRevision.d.ts +19 -0
- package/dist/groups/membershipRevision.js +29 -0
- package/dist/groups/membershipRevision.js.map +1 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/jobs/jobOps.d.ts +4 -9
- package/dist/jobs/jobOps.js +77 -39
- package/dist/jobs/jobOps.js.map +1 -1
- package/dist/pipelines/diagram.js +19 -9
- package/dist/pipelines/diagram.js.map +1 -1
- package/dist/pipelines/index.d.ts +28 -5
- package/dist/pipelines/index.js +118 -57
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pipelines/invocation.d.ts +80 -0
- package/dist/pipelines/invocation.js +290 -0
- package/dist/pipelines/invocation.js.map +1 -0
- package/dist/pipelines/manifest.d.ts +88 -0
- package/dist/pipelines/manifest.js +216 -0
- package/dist/pipelines/manifest.js.map +1 -0
- package/dist/plugins/index.d.ts +29 -0
- package/dist/plugins/index.js +51 -10
- package/dist/plugins/index.js.map +1 -1
- package/dist/plugins/internal/plugin_executeNewFeed.d.ts +0 -14
- package/dist/plugins/internal/plugin_executeNewFeed.js +3 -3
- package/dist/plugins/internal/plugin_executeNewFeed.js.map +1 -1
- package/dist/plugins/plugin_register.js +5 -5
- package/dist/plugins/plugin_register.js.map +1 -1
- package/dist/plugins/store_import.js +1 -1
- package/dist/plugins/store_import.js.map +1 -1
- package/dist/retrieve/watch.d.ts +114 -0
- package/dist/retrieve/watch.js +358 -0
- package/dist/retrieve/watch.js.map +1 -0
- package/dist/store/index.d.ts +17 -4
- package/dist/store/index.js +9 -4
- package/dist/store/index.js.map +1 -1
- package/dist/users/index.d.ts +36 -0
- package/dist/users/index.js +50 -0
- package/dist/users/index.js.map +1 -0
- package/dist/vfs/dispatcher.d.ts +11 -0
- package/dist/vfs/dispatcher.js +37 -6
- package/dist/vfs/dispatcher.js.map +1 -1
- package/dist/vfs/index.d.ts +1 -0
- package/dist/vfs/index.js +3 -0
- package/dist/vfs/index.js.map +1 -1
- package/dist/vfs/provider.d.ts +13 -0
- package/dist/vfs/providers/etc.d.ts +2 -0
- package/dist/vfs/providers/etc.js +41 -2
- package/dist/vfs/providers/etc.js.map +1 -1
- package/dist/vfs/providers/native.js +41 -7
- package/dist/vfs/providers/native.js.map +1 -1
- package/dist/vfs/providers/pacs.d.ts +13 -1
- package/dist/vfs/providers/pacs.js +86 -107
- package/dist/vfs/providers/pacs.js.map +1 -1
- package/dist/vfs/providers/pacsHelpers.js +9 -5
- package/dist/vfs/providers/pacsHelpers.js.map +1 -1
- package/dist/vfs/providers/proc.d.ts +34 -1
- package/dist/vfs/providers/proc.js +314 -84
- package/dist/vfs/providers/proc.js.map +1 -1
- package/package.json +11 -8
|
@@ -21,19 +21,23 @@ exports.procFeed_ensureLoaded = procFeed_ensureLoaded;
|
|
|
21
21
|
exports.feedMeta_ensure = feedMeta_ensure;
|
|
22
22
|
exports.feedStatus_refresh = feedStatus_refresh;
|
|
23
23
|
exports.procTopology_warmup = procTopology_warmup;
|
|
24
|
+
exports.procTopology_reconcileFeeds = procTopology_reconcileFeeds;
|
|
25
|
+
exports.procTopology_retry = procTopology_retry;
|
|
24
26
|
exports.procTopology_status = procTopology_status;
|
|
25
27
|
exports.procTopology_await = procTopology_await;
|
|
26
28
|
exports.procCache_refresh = procCache_refresh;
|
|
27
29
|
const cumin_1 = require("@fnndsc/cumin");
|
|
28
30
|
const index_js_1 = require("../../jobs/index.js");
|
|
29
31
|
/** Virtual filenames inside each instance directory. */
|
|
30
|
-
const INSTANCE_FILES = new Set(['status', 'params', 'log']);
|
|
32
|
+
const INSTANCE_FILES = new Set(['status', 'params', 'log', 'data']);
|
|
31
33
|
/** Virtual filenames inside each feed directory. */
|
|
32
34
|
const FEED_FILES = new Set(['status', 'title']);
|
|
33
35
|
const PROC_JOBS_PREFIX = '/proc/jobs';
|
|
34
36
|
const PAGE = 100;
|
|
35
37
|
let procTopologyPromise = null;
|
|
36
38
|
let procTopologyFailure;
|
|
39
|
+
let procTopologyResumeState = null;
|
|
40
|
+
let procTopologyScopedResumeFeedIDs = null;
|
|
37
41
|
/** Converts a CUBE feed row into the cache's feed model. */
|
|
38
42
|
function procFeed_create(feed) {
|
|
39
43
|
return {
|
|
@@ -51,18 +55,19 @@ function procFeed_create(feed) {
|
|
|
51
55
|
};
|
|
52
56
|
}
|
|
53
57
|
/** Indexes one complete paginated CUBE feed collection into the shared cache. */
|
|
54
|
-
async function procFeeds_index(
|
|
58
|
+
async function procFeeds_index(page_fetch, indexed) {
|
|
55
59
|
let offset = 0;
|
|
56
60
|
let total = 0;
|
|
57
61
|
const seenFeedIDs = new Set();
|
|
58
62
|
while (true) {
|
|
59
63
|
const page = await page_fetch({ limit: PAGE, offset });
|
|
60
|
-
const chunk = page.data
|
|
61
|
-
if (offset === 0 &&
|
|
64
|
+
const chunk = page.data;
|
|
65
|
+
if (offset === 0 && page.totalCount !== null) {
|
|
62
66
|
total = page.totalCount;
|
|
63
67
|
}
|
|
64
68
|
for (const feed of chunk) {
|
|
65
|
-
|
|
69
|
+
const procFeed = procFeed_create(feed);
|
|
70
|
+
indexed.set(procFeed.id, procFeed);
|
|
66
71
|
seenFeedIDs.add(Number(feed.id));
|
|
67
72
|
}
|
|
68
73
|
if (chunk.length === 0 || (total > 0 && seenFeedIDs.size >= total))
|
|
@@ -79,18 +84,19 @@ async function procFeeds_index(cache, page_fetch) {
|
|
|
79
84
|
*/
|
|
80
85
|
async function procCache_build() {
|
|
81
86
|
const cache = (0, cumin_1.procCache_get)();
|
|
82
|
-
cache.cache_clear();
|
|
83
87
|
const client = await cumin_1.chrisConnection.client_get();
|
|
84
88
|
if (!client) {
|
|
85
89
|
cumin_1.errorStack.stack_push('error', 'procCache_build: not connected');
|
|
86
|
-
|
|
87
|
-
}
|
|
88
|
-
const typedClient = client;
|
|
89
|
-
await procFeeds_index(cache, typedClient.getFeeds.bind(typedClient));
|
|
90
|
-
if (typedClient.getPublicFeeds) {
|
|
91
|
-
await procFeeds_index(cache, typedClient.getPublicFeeds.bind(typedClient));
|
|
90
|
+
throw new Error('not connected');
|
|
92
91
|
}
|
|
92
|
+
const indexed = new Map();
|
|
93
|
+
await procFeeds_index((params) => (0, cumin_1.feedsPage_get)(client, params), indexed);
|
|
94
|
+
// Older CUBEs lack a public-feeds endpoint: the contract answers null
|
|
95
|
+
// without a wire call, and an empty first page ends the index loop.
|
|
96
|
+
await procFeeds_index(async (params) => (await (0, cumin_1.publicFeedsPage_get)(client, params)) ?? { data: [], totalCount: 0 }, indexed);
|
|
97
|
+
const reconciliationTargets = cache.feeds_reconcile(Array.from(indexed.values()));
|
|
93
98
|
cache.built_set();
|
|
99
|
+
return reconciliationTargets;
|
|
94
100
|
}
|
|
95
101
|
/** Ensures the feed index is built, building it on first access. */
|
|
96
102
|
async function cache_ensure() {
|
|
@@ -106,17 +112,16 @@ async function feedInstances_load(feedID) {
|
|
|
106
112
|
const cache = (0, cumin_1.procCache_get)();
|
|
107
113
|
const client = await cumin_1.chrisConnection.client_get();
|
|
108
114
|
if (!client)
|
|
109
|
-
|
|
110
|
-
const typedClient = client;
|
|
115
|
+
throw new Error('not connected');
|
|
111
116
|
const instances = new Map();
|
|
112
117
|
let offset = 0;
|
|
113
118
|
let total = 0;
|
|
114
119
|
while (true) {
|
|
115
|
-
const page = await
|
|
120
|
+
const page = await (0, cumin_1.pluginInstancesPage_get)(client, {
|
|
116
121
|
feed_id: feedID, limit: PAGE, offset,
|
|
117
122
|
});
|
|
118
|
-
const chunk = page.data
|
|
119
|
-
if (offset === 0 &&
|
|
123
|
+
const chunk = page.data;
|
|
124
|
+
if (offset === 0 && page.totalCount !== null) {
|
|
120
125
|
total = page.totalCount;
|
|
121
126
|
}
|
|
122
127
|
for (const instance of chunk)
|
|
@@ -138,6 +143,7 @@ async function feedInstances_load(feedID) {
|
|
|
138
143
|
pluginName: String(inst.plugin_name),
|
|
139
144
|
pluginType: inst.plugin_type !== undefined ? String(inst.plugin_type) : undefined,
|
|
140
145
|
params: null,
|
|
146
|
+
outputPath: outputPath_normalize(inst.output_path) ?? undefined,
|
|
141
147
|
status: String(inst.status ?? 'unknown'),
|
|
142
148
|
});
|
|
143
149
|
}
|
|
@@ -157,8 +163,12 @@ async function feedInstances_ensureLoaded(feedID) {
|
|
|
157
163
|
return inflight;
|
|
158
164
|
const promise = feedInstances_load(feedID);
|
|
159
165
|
cache.loading_set(feedID, promise);
|
|
160
|
-
|
|
161
|
-
|
|
166
|
+
try {
|
|
167
|
+
await promise;
|
|
168
|
+
}
|
|
169
|
+
finally {
|
|
170
|
+
cache.loading_clear(feedID);
|
|
171
|
+
}
|
|
162
172
|
}
|
|
163
173
|
// ── Aggregate status ───────────────────────────────────────────────────────
|
|
164
174
|
/** Derives aggregate feed status from stored job counters — no API call. */
|
|
@@ -218,6 +228,77 @@ function params_render(inst) {
|
|
|
218
228
|
.map(([k, v]) => `${k}=${String(v)}`)
|
|
219
229
|
.join('\n');
|
|
220
230
|
}
|
|
231
|
+
/**
|
|
232
|
+
* Normalizes CUBE's output-path spelling into an absolute CFS path.
|
|
233
|
+
*
|
|
234
|
+
* @param value - Raw `output_path` value from a plugin-instance resource.
|
|
235
|
+
* @returns Absolute CFS path, or null when no usable path was supplied.
|
|
236
|
+
*/
|
|
237
|
+
function outputPath_normalize(value) {
|
|
238
|
+
if (typeof value !== 'string' || value.trim() === '')
|
|
239
|
+
return null;
|
|
240
|
+
const clean = value.trim();
|
|
241
|
+
return clean.startsWith('/') ? clean : `/${clean}`;
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Resolves and memoizes the authoritative CFS output directory for one job.
|
|
245
|
+
*
|
|
246
|
+
* @param instanceID - Plugin-instance ID whose output path is needed.
|
|
247
|
+
* @returns Absolute output path, or null when CUBE has not recorded one.
|
|
248
|
+
*/
|
|
249
|
+
async function instanceOutputPath_ensure(instanceID) {
|
|
250
|
+
const cache = (0, cumin_1.procCache_get)();
|
|
251
|
+
const inst = cache.instance_get(instanceID);
|
|
252
|
+
if (!inst)
|
|
253
|
+
return null;
|
|
254
|
+
if (inst.outputPath !== undefined)
|
|
255
|
+
return inst.outputPath;
|
|
256
|
+
const client = await cumin_1.chrisConnection.client_get();
|
|
257
|
+
if (!client)
|
|
258
|
+
return null;
|
|
259
|
+
try {
|
|
260
|
+
const handle = await (0, cumin_1.pluginInstance_get)(client, instanceID);
|
|
261
|
+
const outputPath = outputPath_normalize(handle?.data?.output_path);
|
|
262
|
+
cache.outputPath_update(instanceID, outputPath);
|
|
263
|
+
return outputPath;
|
|
264
|
+
}
|
|
265
|
+
catch (error) {
|
|
266
|
+
// "No CFS output path" downstream must not mask a failed fetch.
|
|
267
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
268
|
+
cumin_1.errorStack.stack_push("warning", `Could not fetch output path for instance ${instanceID}: ${msg}`);
|
|
269
|
+
return null;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Reads every page of effective parameter values from one plugin instance.
|
|
274
|
+
*
|
|
275
|
+
* @param handle - Typed handle for the instance being inspected.
|
|
276
|
+
* @returns Parameter names mapped to their recorded invocation values.
|
|
277
|
+
*/
|
|
278
|
+
async function instanceParams_fetch(handle) {
|
|
279
|
+
const params = {};
|
|
280
|
+
let offset = 0;
|
|
281
|
+
let total = 0;
|
|
282
|
+
let fetched = 0;
|
|
283
|
+
while (true) {
|
|
284
|
+
const page = await handle.parametersPage_get({ limit: PAGE, offset });
|
|
285
|
+
const items = page.data;
|
|
286
|
+
if (offset === 0 && page.totalCount !== null) {
|
|
287
|
+
total = page.totalCount;
|
|
288
|
+
}
|
|
289
|
+
for (const item of items) {
|
|
290
|
+
if (item.param_name)
|
|
291
|
+
params[item.param_name] = item.value;
|
|
292
|
+
}
|
|
293
|
+
fetched += items.length;
|
|
294
|
+
if (items.length === 0 || (total > 0 && fetched >= total))
|
|
295
|
+
break;
|
|
296
|
+
if (total === 0 && items.length < PAGE)
|
|
297
|
+
break;
|
|
298
|
+
offset += items.length;
|
|
299
|
+
}
|
|
300
|
+
return params;
|
|
301
|
+
}
|
|
221
302
|
/** Collects all instance IDs for a feed recursively. */
|
|
222
303
|
function getAllInstanceIDs_forFeed(feedID, cache) {
|
|
223
304
|
const result = [];
|
|
@@ -235,6 +316,28 @@ function getAllInstanceIDs_forFeed(feedID, cache) {
|
|
|
235
316
|
*/
|
|
236
317
|
class ProcVfsProvider {
|
|
237
318
|
prefix = PROC_JOBS_PREFIX;
|
|
319
|
+
/**
|
|
320
|
+
* Resolves the synthetic `data` link for one job only when navigation asks
|
|
321
|
+
* to follow it. This keeps `/proc` topology listings and recursive trees
|
|
322
|
+
* entirely cache-backed.
|
|
323
|
+
*
|
|
324
|
+
* @param pathStr - Absolute `/proc/jobs/.../data` path to resolve.
|
|
325
|
+
* @returns The authoritative CFS output path, or an error when absent.
|
|
326
|
+
*/
|
|
327
|
+
async linkTarget_resolve(pathStr) {
|
|
328
|
+
await cache_ensure();
|
|
329
|
+
const clean = pathStr.replace(/\/$/, '');
|
|
330
|
+
const { feedID, instanceID, virtualFile } = procPath_parse(clean);
|
|
331
|
+
if (feedID === null || instanceID === null || virtualFile !== 'data') {
|
|
332
|
+
cumin_1.errorStack.stack_push('error', `Not a /proc job data link: ${pathStr}`);
|
|
333
|
+
return (0, cumin_1.Err)();
|
|
334
|
+
}
|
|
335
|
+
const outputPath = await instanceOutputPath_ensure(instanceID);
|
|
336
|
+
if (outputPath)
|
|
337
|
+
return (0, cumin_1.Ok)(outputPath);
|
|
338
|
+
cumin_1.errorStack.stack_push('error', `No CFS output path is available for job ${instanceID}`);
|
|
339
|
+
return (0, cumin_1.Err)();
|
|
340
|
+
}
|
|
238
341
|
async list(pathStr, _options) {
|
|
239
342
|
await cache_ensure();
|
|
240
343
|
const cache = (0, cumin_1.procCache_get)();
|
|
@@ -304,6 +407,15 @@ class ProcVfsProvider {
|
|
|
304
407
|
items.push({ name: 'status', type: 'file', size: 0, owner: '', date: '' });
|
|
305
408
|
items.push({ name: 'params', type: 'file', size: 0, owner: '', date: '' });
|
|
306
409
|
items.push({ name: 'log', type: 'file', size: 0, owner: '', date: '' });
|
|
410
|
+
// `data` is structural: list it without fetching the per-instance detail
|
|
411
|
+
// endpoint. Its target is resolved only when `cd data` follows the link.
|
|
412
|
+
// A known null means CUBE has no output directory, so omit it entirely.
|
|
413
|
+
if (inst.outputPath !== null) {
|
|
414
|
+
const dataLink = { name: 'data', type: 'link', size: 0, owner: '', date: '' };
|
|
415
|
+
if (inst.outputPath)
|
|
416
|
+
dataLink.target = inst.outputPath;
|
|
417
|
+
items.push(dataLink);
|
|
418
|
+
}
|
|
307
419
|
for (const childID of childIDs) {
|
|
308
420
|
const child = cache.instance_get(childID);
|
|
309
421
|
if (!child)
|
|
@@ -354,22 +466,20 @@ class ProcVfsProvider {
|
|
|
354
466
|
}
|
|
355
467
|
if (virtualFile === 'params') {
|
|
356
468
|
if (inst.params === null) {
|
|
357
|
-
//
|
|
469
|
+
// CUBE stores the effective run values in the parameter sub-resource;
|
|
470
|
+
// the instance detail payload itself is operational metadata.
|
|
358
471
|
const client = await cumin_1.chrisConnection.client_get();
|
|
359
472
|
if (client) {
|
|
360
473
|
try {
|
|
361
|
-
const
|
|
362
|
-
if (
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
cache.params_update(instanceID, p);
|
|
370
|
-
}
|
|
474
|
+
const handle = await (0, cumin_1.pluginInstance_get)(client, instanceID);
|
|
475
|
+
if (handle)
|
|
476
|
+
cache.params_update(instanceID, await instanceParams_fetch(handle));
|
|
477
|
+
}
|
|
478
|
+
catch {
|
|
479
|
+
// Deliberate absorption, adjudicated 2026-08: parameters are a
|
|
480
|
+
// lazy display enrichment; a failed fetch renders as
|
|
481
|
+
// "(no parameters)" and the next read retries.
|
|
371
482
|
}
|
|
372
|
-
catch { /* leave null */ }
|
|
373
483
|
}
|
|
374
484
|
}
|
|
375
485
|
return (0, cumin_1.Ok)(params_render(inst));
|
|
@@ -389,12 +499,21 @@ class ProcVfsProvider {
|
|
|
389
499
|
if (feedID !== null && instanceID === null) {
|
|
390
500
|
const allIDs = getAllInstanceIDs_forFeed(feedID, cache);
|
|
391
501
|
const statusMap = await (0, index_js_1.jobs_statusBatch)(allIDs);
|
|
502
|
+
// Every cancel Result is checked: reporting success while jobs kept
|
|
503
|
+
// running would leave the user believing the feed's work was stopped.
|
|
504
|
+
const failedCancels = [];
|
|
392
505
|
await Promise.all(allIDs.map(async (id) => {
|
|
393
506
|
const s = statusMap.get(id) ?? '';
|
|
394
507
|
if (!(0, cumin_1.status_isTerminal)(s)) {
|
|
395
|
-
await (0, index_js_1.job_cancel)(id);
|
|
508
|
+
const cancelResult = await (0, index_js_1.job_cancel)(id);
|
|
509
|
+
if (!cancelResult.ok)
|
|
510
|
+
failedCancels.push(id);
|
|
396
511
|
}
|
|
397
512
|
}));
|
|
513
|
+
if (failedCancels.length > 0) {
|
|
514
|
+
cumin_1.errorStack.stack_push('error', `rm: failed to cancel ${failedCancels.length} running job(s) in feed ${feedID}: instance(s) ${failedCancels.sort((a, b) => a - b).join(', ')}`);
|
|
515
|
+
return false;
|
|
516
|
+
}
|
|
398
517
|
cache.feed_remove(feedID);
|
|
399
518
|
return true;
|
|
400
519
|
}
|
|
@@ -464,10 +583,9 @@ async function feedMeta_ensure(feedID) {
|
|
|
464
583
|
return;
|
|
465
584
|
const client = await cumin_1.chrisConnection.client_get();
|
|
466
585
|
if (!client)
|
|
467
|
-
|
|
468
|
-
const
|
|
469
|
-
const
|
|
470
|
-
const f = (page.data ?? [])[0];
|
|
586
|
+
throw new Error('not connected');
|
|
587
|
+
const page = await (0, cumin_1.feedsPage_get)(client, { id: feedID, limit: 1, offset: 0 });
|
|
588
|
+
const f = page.data[0];
|
|
471
589
|
if (!f)
|
|
472
590
|
return;
|
|
473
591
|
cache.feed_add(procFeed_create(f));
|
|
@@ -477,16 +595,15 @@ async function feedStatus_refresh(feedID) {
|
|
|
477
595
|
const client = await cumin_1.chrisConnection.client_get();
|
|
478
596
|
if (!client)
|
|
479
597
|
return;
|
|
480
|
-
const typedClient = client;
|
|
481
598
|
let offset = 0;
|
|
482
599
|
let total = 0;
|
|
483
600
|
const seenInstanceIDs = new Set();
|
|
484
601
|
while (true) {
|
|
485
|
-
const page = await
|
|
602
|
+
const page = await (0, cumin_1.pluginInstancesPage_get)(client, {
|
|
486
603
|
feed_id: feedID, limit: PAGE, offset,
|
|
487
604
|
});
|
|
488
|
-
const chunk = page.data
|
|
489
|
-
if (offset === 0 &&
|
|
605
|
+
const chunk = page.data;
|
|
606
|
+
if (offset === 0 && page.totalCount !== null) {
|
|
490
607
|
total = page.totalCount;
|
|
491
608
|
}
|
|
492
609
|
for (const inst of chunk) {
|
|
@@ -505,44 +622,41 @@ async function feedStatus_refresh(feedID) {
|
|
|
505
622
|
* Background topology warm-up — single paginated sweep of all plugin instances
|
|
506
623
|
* across all feeds. Replaces the per-feed fan-out that was O(feeds) round trips.
|
|
507
624
|
*
|
|
508
|
-
*
|
|
509
|
-
*
|
|
510
|
-
*
|
|
625
|
+
* Scoped per-feed loads may run concurrently for navigation; the completed
|
|
626
|
+
* global sweep reconciles the cache to its authoritative instance set.
|
|
627
|
+
*
|
|
628
|
+
* @param state - Private pagination and reconciliation state for this sweep.
|
|
629
|
+
* @returns A promise that settles when the complete topology has been reconciled.
|
|
511
630
|
*/
|
|
512
|
-
async function procTopology_run() {
|
|
631
|
+
async function procTopology_run(state) {
|
|
513
632
|
await cache_ensure();
|
|
514
633
|
const cache = (0, cumin_1.procCache_get)();
|
|
634
|
+
cache.lifecycle_set('reconciling');
|
|
635
|
+
if (state.total > 0) {
|
|
636
|
+
cache.warmup_progress(state.fetchedInstanceIDs.size, state.total);
|
|
637
|
+
}
|
|
515
638
|
const client = await cumin_1.chrisConnection.client_get();
|
|
516
639
|
if (!client)
|
|
517
|
-
|
|
518
|
-
const typedClient = client;
|
|
519
|
-
let offset = 0;
|
|
520
|
-
let total = 0;
|
|
521
|
-
const seenInstanceIDs = new Set();
|
|
522
|
-
const seenFeedIDs = new Set();
|
|
640
|
+
throw new Error('not connected');
|
|
523
641
|
while (true) {
|
|
524
|
-
const page = await
|
|
525
|
-
limit: PAGE, offset,
|
|
642
|
+
const page = await (0, cumin_1.pluginInstancesPage_get)(client, {
|
|
643
|
+
limit: PAGE, offset: state.offset,
|
|
526
644
|
});
|
|
527
|
-
const chunk = page.data
|
|
528
|
-
if (offset === 0 &&
|
|
529
|
-
total = page.totalCount;
|
|
530
|
-
if (total > 0)
|
|
531
|
-
cache.warmup_progress(0, total);
|
|
645
|
+
const chunk = page.data;
|
|
646
|
+
if (state.offset === 0 && page.totalCount !== null) {
|
|
647
|
+
state.total = page.totalCount;
|
|
648
|
+
if (state.total > 0)
|
|
649
|
+
cache.warmup_progress(0, state.total);
|
|
532
650
|
}
|
|
533
651
|
for (const inst of chunk) {
|
|
534
652
|
const instanceID = Number(inst.id);
|
|
535
653
|
const feedID = Number(inst.feed_id);
|
|
654
|
+
state.fetchedInstanceIDs.add(instanceID);
|
|
536
655
|
const prevID = (inst.previous_id !== null && inst.previous_id !== undefined)
|
|
537
656
|
? Number(inst.previous_id)
|
|
538
657
|
: null;
|
|
539
|
-
if (!cache.feed_get(feedID))
|
|
540
|
-
|
|
541
|
-
id: feedID, title: `feed_${feedID}`, ownerUsername: '', public: false, creationDate: '',
|
|
542
|
-
finishedJobs: 0, erroredJobs: 0, startedJobs: 0,
|
|
543
|
-
scheduledJobs: 0, cancelledJobs: 0, createdJobs: 0,
|
|
544
|
-
});
|
|
545
|
-
}
|
|
658
|
+
if (!cache.feed_get(feedID))
|
|
659
|
+
continue;
|
|
546
660
|
cache.instance_add({
|
|
547
661
|
id: instanceID,
|
|
548
662
|
feedID,
|
|
@@ -550,36 +664,53 @@ async function procTopology_run() {
|
|
|
550
664
|
pluginName: String(inst.plugin_name),
|
|
551
665
|
pluginType: inst.plugin_type !== undefined ? String(inst.plugin_type) : undefined,
|
|
552
666
|
params: null,
|
|
667
|
+
outputPath: outputPath_normalize(inst.output_path) ?? undefined,
|
|
553
668
|
status: String(inst.status ?? 'unknown'),
|
|
554
669
|
});
|
|
555
|
-
seenInstanceIDs.add(instanceID);
|
|
556
|
-
seenFeedIDs.add(feedID);
|
|
670
|
+
state.seenInstanceIDs.add(instanceID);
|
|
671
|
+
state.seenFeedIDs.add(feedID);
|
|
557
672
|
}
|
|
558
|
-
if (total > 0)
|
|
559
|
-
cache.warmup_progress(
|
|
560
|
-
if (chunk.length === 0 || (total > 0 &&
|
|
673
|
+
if (state.total > 0)
|
|
674
|
+
cache.warmup_progress(state.fetchedInstanceIDs.size, state.total);
|
|
675
|
+
if (chunk.length === 0 || (state.total > 0 && state.fetchedInstanceIDs.size >= state.total))
|
|
561
676
|
break;
|
|
562
|
-
if (total === 0 && chunk.length < PAGE)
|
|
677
|
+
if (state.total === 0 && chunk.length < PAGE)
|
|
563
678
|
break;
|
|
564
|
-
offset += chunk.length;
|
|
679
|
+
state.offset += chunk.length;
|
|
565
680
|
}
|
|
566
|
-
|
|
681
|
+
cache.topology_reconcile(state.seenInstanceIDs);
|
|
682
|
+
for (const feedID of state.seenFeedIDs)
|
|
567
683
|
cache.topologyLoaded_mark(feedID);
|
|
684
|
+
if (state.total === 0) {
|
|
685
|
+
cache.warmup_progress(state.fetchedInstanceIDs.size, state.fetchedInstanceIDs.size);
|
|
568
686
|
}
|
|
569
|
-
if (total === 0)
|
|
570
|
-
cache.warmup_progress(seenInstanceIDs.size, seenInstanceIDs.size);
|
|
571
687
|
cache.warmup_complete();
|
|
572
688
|
}
|
|
573
689
|
/**
|
|
574
|
-
*
|
|
690
|
+
* Creates an empty continuation for a fresh global topology sweep.
|
|
575
691
|
*
|
|
576
|
-
* @returns
|
|
692
|
+
* @returns New private pagination and reconciliation state.
|
|
577
693
|
*/
|
|
578
|
-
function
|
|
579
|
-
|
|
580
|
-
|
|
694
|
+
function procTopologyState_create() {
|
|
695
|
+
return {
|
|
696
|
+
offset: 0,
|
|
697
|
+
total: 0,
|
|
698
|
+
fetchedInstanceIDs: new Set(),
|
|
699
|
+
seenInstanceIDs: new Set(),
|
|
700
|
+
seenFeedIDs: new Set(),
|
|
701
|
+
};
|
|
702
|
+
}
|
|
703
|
+
/**
|
|
704
|
+
* Starts one topology sweep from the supplied private continuation state.
|
|
705
|
+
*
|
|
706
|
+
* @param state - Private pagination and reconciliation state to run or resume.
|
|
707
|
+
* @returns The in-flight topology sweep promise.
|
|
708
|
+
*/
|
|
709
|
+
function procTopology_start(state) {
|
|
581
710
|
procTopologyFailure = undefined;
|
|
582
|
-
|
|
711
|
+
procTopologyResumeState = state;
|
|
712
|
+
procTopologyScopedResumeFeedIDs = null;
|
|
713
|
+
const sweep = procTopology_run(state);
|
|
583
714
|
procTopologyPromise = sweep;
|
|
584
715
|
sweep.then(() => {
|
|
585
716
|
if (procTopologyPromise !== sweep)
|
|
@@ -587,6 +718,7 @@ function procTopology_warmup() {
|
|
|
587
718
|
procTopologyPromise = null;
|
|
588
719
|
if ((0, cumin_1.procCache_get)().warmupComplete) {
|
|
589
720
|
procTopologyFailure = undefined;
|
|
721
|
+
procTopologyResumeState = null;
|
|
590
722
|
}
|
|
591
723
|
else {
|
|
592
724
|
procTopologyFailure = 'the topology sweep ended before the index completed';
|
|
@@ -600,6 +732,94 @@ function procTopology_warmup() {
|
|
|
600
732
|
});
|
|
601
733
|
return sweep;
|
|
602
734
|
}
|
|
735
|
+
/**
|
|
736
|
+
* Runs a feed-scoped topology reconciliation selected after checkpoint restore.
|
|
737
|
+
*
|
|
738
|
+
* @param feedIDs - Canonical feed IDs whose topology needs a fresh load.
|
|
739
|
+
* @returns A promise that settles after the supplied feeds are reconciled.
|
|
740
|
+
*/
|
|
741
|
+
async function procTopologyScoped_run(feedIDs) {
|
|
742
|
+
const cache = (0, cumin_1.procCache_get)();
|
|
743
|
+
cache.lifecycle_set('reconciling');
|
|
744
|
+
for (const feedID of feedIDs)
|
|
745
|
+
await procCache_refresh(feedID);
|
|
746
|
+
cache.warmup_complete();
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* Starts one scoped reconciliation and retains its targets for `proc retry`.
|
|
750
|
+
*
|
|
751
|
+
* @param feedIDs - Canonical feed IDs selected by the fresh feed index.
|
|
752
|
+
* @returns The in-flight scoped reconciliation promise.
|
|
753
|
+
*/
|
|
754
|
+
function procTopologyScoped_start(feedIDs) {
|
|
755
|
+
const targets = [...feedIDs];
|
|
756
|
+
procTopologyFailure = undefined;
|
|
757
|
+
procTopologyResumeState = null;
|
|
758
|
+
procTopologyScopedResumeFeedIDs = targets;
|
|
759
|
+
const sweep = procTopologyScoped_run(targets);
|
|
760
|
+
procTopologyPromise = sweep;
|
|
761
|
+
sweep.then(() => {
|
|
762
|
+
if (procTopologyPromise !== sweep)
|
|
763
|
+
return;
|
|
764
|
+
procTopologyPromise = null;
|
|
765
|
+
if ((0, cumin_1.procCache_get)().warmupComplete) {
|
|
766
|
+
procTopologyFailure = undefined;
|
|
767
|
+
procTopologyScopedResumeFeedIDs = null;
|
|
768
|
+
}
|
|
769
|
+
else {
|
|
770
|
+
procTopologyFailure = 'the scoped topology reconciliation ended before the index completed';
|
|
771
|
+
}
|
|
772
|
+
}, (error) => {
|
|
773
|
+
if (procTopologyPromise !== sweep)
|
|
774
|
+
return;
|
|
775
|
+
procTopologyPromise = null;
|
|
776
|
+
(0, cumin_1.procCache_get)().warmup_abort();
|
|
777
|
+
procTopologyFailure = error instanceof Error ? error.message : String(error);
|
|
778
|
+
});
|
|
779
|
+
return sweep;
|
|
780
|
+
}
|
|
781
|
+
/**
|
|
782
|
+
* Starts or joins the session's global plugin-instance topology sweep.
|
|
783
|
+
*
|
|
784
|
+
* @returns The single in-flight sweep promise.
|
|
785
|
+
*/
|
|
786
|
+
function procTopology_warmup() {
|
|
787
|
+
if (procTopologyPromise)
|
|
788
|
+
return procTopologyPromise;
|
|
789
|
+
return procTopology_start(procTopologyState_create());
|
|
790
|
+
}
|
|
791
|
+
/**
|
|
792
|
+
* Reconciles topology only for feed IDs selected by a restored-cache feed-index
|
|
793
|
+
* comparison. This preserves the checkpointed topology for unchanged terminal
|
|
794
|
+
* feeds while refreshing new, changed, and active feeds from CUBE.
|
|
795
|
+
*
|
|
796
|
+
* @param feedIDs - Canonical CUBE feed IDs whose topology must be reloaded.
|
|
797
|
+
* @returns A promise that resolves after all supplied feeds have been refreshed.
|
|
798
|
+
*/
|
|
799
|
+
function procTopology_reconcileFeeds(feedIDs) {
|
|
800
|
+
if (procTopologyPromise)
|
|
801
|
+
return procTopologyPromise;
|
|
802
|
+
return procTopologyScoped_start(feedIDs);
|
|
803
|
+
}
|
|
804
|
+
/**
|
|
805
|
+
* Continues a failed topology reconciliation from its retained global page or
|
|
806
|
+
* scoped feed set.
|
|
807
|
+
*
|
|
808
|
+
* @returns The resumed sweep promise, or the active sweep when one is running.
|
|
809
|
+
* @throws {Error} When there is no failed sweep continuation to retry.
|
|
810
|
+
*/
|
|
811
|
+
function procTopology_retry() {
|
|
812
|
+
if (procTopologyPromise)
|
|
813
|
+
return procTopologyPromise;
|
|
814
|
+
if (procTopologyFailure === undefined) {
|
|
815
|
+
throw new Error('no failed topology sweep to retry');
|
|
816
|
+
}
|
|
817
|
+
if (procTopologyResumeState)
|
|
818
|
+
return procTopology_start(procTopologyResumeState);
|
|
819
|
+
if (procTopologyScopedResumeFeedIDs)
|
|
820
|
+
return procTopologyScoped_start(procTopologyScopedResumeFeedIDs);
|
|
821
|
+
throw new Error('no failed topology sweep to retry');
|
|
822
|
+
}
|
|
603
823
|
/**
|
|
604
824
|
* Reports the session topology sweep lifecycle.
|
|
605
825
|
*
|
|
@@ -627,6 +847,13 @@ async function procTopology_await() {
|
|
|
627
847
|
}
|
|
628
848
|
/**
|
|
629
849
|
* Rebuilds the ProcCache, optionally scoped to one feed.
|
|
850
|
+
*
|
|
851
|
+
* A full refresh returns feeds whose topology merits reload after the feed-index
|
|
852
|
+
* comparison. Callers performing an explicit `proc refresh` can ignore that
|
|
853
|
+
* result and run a global topology sweep; restored startup uses it to stay scoped.
|
|
854
|
+
*
|
|
855
|
+
* @param feedID - One feed to force-refresh, or undefined for the full feed index.
|
|
856
|
+
* @returns IDs whose topology is new, changed, or currently active.
|
|
630
857
|
*/
|
|
631
858
|
async function procCache_refresh(feedID) {
|
|
632
859
|
if (feedID !== undefined) {
|
|
@@ -635,14 +862,14 @@ async function procCache_refresh(feedID) {
|
|
|
635
862
|
// Re-fetch feed metadata with job counters
|
|
636
863
|
const client = await cumin_1.chrisConnection.client_get();
|
|
637
864
|
if (client) {
|
|
638
|
-
const
|
|
639
|
-
const
|
|
640
|
-
const f = (page.data ?? [])[0];
|
|
865
|
+
const page = await (0, cumin_1.feedsPage_get)(client, { id: feedID, limit: 1, offset: 0 });
|
|
866
|
+
const f = page.data[0];
|
|
641
867
|
if (f) {
|
|
642
868
|
cache.feed_add(procFeed_create(f));
|
|
643
869
|
}
|
|
644
870
|
}
|
|
645
871
|
await feedInstances_ensureLoaded(feedID);
|
|
872
|
+
return [feedID];
|
|
646
873
|
}
|
|
647
874
|
else {
|
|
648
875
|
const activeSweep = procTopologyPromise;
|
|
@@ -656,7 +883,10 @@ async function procCache_refresh(feedID) {
|
|
|
656
883
|
}
|
|
657
884
|
procTopologyPromise = null;
|
|
658
885
|
procTopologyFailure = undefined;
|
|
659
|
-
|
|
886
|
+
procTopologyResumeState = null;
|
|
887
|
+
procTopologyScopedResumeFeedIDs = null;
|
|
888
|
+
(0, cumin_1.procCache_get)().warmup_reset();
|
|
889
|
+
return procCache_build();
|
|
660
890
|
}
|
|
661
891
|
}
|
|
662
892
|
//# sourceMappingURL=proc.js.map
|