arcane-os 0.95.0 → 0.97.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,6 @@
1
- import {mkdir, mkdtemp, open, readFile, readdir, realpath, rename, rm, stat, writeFile} from 'node:fs/promises';
1
+ import {mkdir, mkdtemp, open, readFile, readdir, realpath, rename, rm, rmdir, stat, unlink, writeFile} from 'node:fs/promises';
2
+ import {watch} from 'node:fs';
3
+ import {randomUUID} from 'node:crypto';
2
4
  import {Buffer} from 'node:buffer';
3
5
  import path from 'node:path';
4
6
  import Is from 'strong-type';
@@ -25,149 +27,852 @@ function memberLocation(directory, member) {
25
27
  function sameMembers(left, right) {
26
28
  return is.array(left) && left.length === right.length
27
29
  && left.every(function sameMember(member, index) {
28
- return member.path === right[index].path && member.url === right[index].url;
30
+ return member.path === right[index].path && member.url === right[index].url
31
+ && member.archiveMember === right[index].archiveMember;
29
32
  });
30
33
  }
31
34
 
32
- async function storedAcquisition(directory, members, signal) {
33
- let latest;
35
+ function selectedSources(members = [], archives = []) {
36
+ const selected = members.map(function ordinaryMember(member) { return {path: member.path, url: member.url}; });
37
+ for (const archive of archives) {
38
+ if (archive.format !== 'tar') throw new TypeError('Model asset archives require the uncompressed tar format.');
39
+ for (const member of archive.members) {
40
+ selected.push({path: member.path, url: archive.url, archiveMember: member.member});
41
+ }
42
+ }
43
+ return selected;
44
+ }
45
+
46
+ async function memberAvailability(directory, members, signal) {
47
+ const result = [];
48
+ for (const member of members) {
49
+ signal?.throwIfAborted();
50
+ const location = memberLocation(directory, member);
51
+ let available;
52
+ try { available = (await stat(location.nativePath)).isFile(); }
53
+ catch (error) { if (error.code !== 'ENOENT') throw error; available = false; }
54
+ result.push({...location, available});
55
+ }
56
+ return result;
57
+ }
58
+
59
+ function discoveryRecord(directory, members, source, catalog, entryPath) {
60
+ const missingMembers = members.filter(function missing(member) { return !member.available; })
61
+ .map(function memberPath(member) { return member.path; });
62
+ return {directory, entryPath: entryPath ?? members[0]?.path, members, missingMembers,
63
+ complete: missingMembers.length === 0, source, ...(catalog ? {catalogId: catalog.id} : {})};
64
+ }
65
+
66
+ function groupedMembers(members) {
67
+ const groups = [];
68
+ const shards = new Map();
69
+ for (const member of members) {
70
+ const filename = path.basename(member.path);
71
+ const split = /^(.*)-(\d+)-of-(\d+)(\.gguf)$/i.exec(filename);
72
+ if (!split) {
73
+ groups.push({members: [member], gguf: /\.gguf$/i.test(filename), entryPath: member.path});
74
+ continue;
75
+ }
76
+ const total = Number(split[3]);
77
+ if (!Number.isSafeInteger(total) || total < 1) {
78
+ throw failure('MODEL_ASSET_SHARDS_UNREADABLE', `The split GGUF member count is unreadable: ${member.path}`);
79
+ }
80
+ const directory = member.path.substring(0, member.path.length - filename.length);
81
+ const key = `${directory}${split[1]}-of-${split[3]}${split[4]}`;
82
+ if (!shards.has(key)) {
83
+ const group = {members: [], observed: new Map(), gguf: true, total, directory,
84
+ prefix: split[1], width: split[2].length, totalText: split[3], extension: split[4]};
85
+ shards.set(key, group);
86
+ groups.push(group);
87
+ }
88
+ shards.get(key).observed.set(Number(split[2]), member);
89
+ }
90
+ for (const group of shards.values()) {
91
+ for (let index = 1; index <= group.total; index += 1) {
92
+ group.members.push(group.observed.get(index) ?? {path: `${group.directory}${group.prefix}-${String(index).padStart(group.width, '0')}-of-${group.totalText}${group.extension}`});
93
+ }
94
+ // Preserve any supplied out-of-sequence member instead of dropping it.
95
+ for (const [index, member] of group.observed) if (index < 1 || index > group.total) group.members.push(member);
96
+ group.entryPath = group.members[0].path;
97
+ }
98
+ return groups;
99
+ }
100
+
101
+ function discoveryEntry(groups, selected) {
102
+ if (selected) {
103
+ const group = groups.find(function selectedGroup(candidate) {
104
+ return candidate.members.some(function selectedMember(member) { return member.path === selected; });
105
+ });
106
+ return group?.observed ? group.entryPath : selected;
107
+ }
108
+ return groups.find(function modelGroup(group) { return group.gguf; })?.entryPath ?? groups[0]?.entryPath;
109
+ }
110
+
111
+ async function discoverDirectories(appRoot, {directories, catalog = [], recursive = true, signal} = {}) {
112
+ if (!is.array(directories)) throw new TypeError('Model discovery requires explicit directories.');
113
+ const roots = [];
114
+ const models = [];
115
+ const visited = new Set();
116
+ const candidates = new Set();
117
+ const claimed = new Set();
118
+ const selections = catalog.map(function catalogSelection(item) {
119
+ return {id: item.id, entryPath: item.entryPath, members: selectedSources(item.members, item.archives)};
120
+ });
121
+
122
+ function add(record) {
123
+ const key = JSON.stringify([record.directory, record.members.map(function memberPath(member) { return member.path; })]);
124
+ if (candidates.has(key)) return;
125
+ candidates.add(key);
126
+ models.push(record);
127
+ }
128
+
129
+ async function scan(directory, root = false) {
130
+ signal?.throwIfAborted();
131
+ let entries;
132
+ try { entries = await readdir(directory, {withFileTypes: true}); }
133
+ catch (error) {
134
+ if (!root) throw error;
135
+ roots.push({directory, exists: error.code !== 'ENOENT', ...(error.code === 'ENOENT' ? {} : {error: serializeCoreError(error)})});
136
+ return;
137
+ }
138
+ if (root) roots.push({directory, exists: true});
139
+ const canonical = await realpath(directory);
140
+ if (visited.has(canonical)) return;
141
+ visited.add(canonical);
142
+ entries.sort(function byName(left, right) { return left.name.localeCompare(right.name); });
143
+ const linked = new Map();
144
+ for (const entry of entries) {
145
+ if (!entry.isSymbolicLink()) continue;
146
+ signal?.throwIfAborted();
147
+ try { linked.set(entry, await stat(path.join(directory, entry.name))); }
148
+ catch (error) { if (error.code !== 'ENOENT') throw error; linked.set(entry, null); }
149
+ }
150
+ let acquisition;
151
+ let acquisitionComplete = false;
152
+ if (path.basename(directory).startsWith('acquisition-')) {
153
+ try { acquisition = JSON.parse(await readFile(path.join(directory, 'acquisition.json'), {encoding: 'utf8', signal})); acquisitionComplete = true; }
154
+ catch (error) { if (error.code !== 'ENOENT') throw error; }
155
+ if (!acquisition) {
156
+ try { acquisition = JSON.parse(await readFile(path.join(directory, 'transfer.json'), {encoding: 'utf8', signal})); }
157
+ catch (error) {
158
+ if (error.code !== 'ENOENT') throw error;
159
+ if (path.basename(path.dirname(directory)) === 'model-assets') return;
160
+ }
161
+ }
162
+ }
163
+ if (acquisition) {
164
+ const files = path.join(directory, 'files');
165
+ const groups = groupedMembers(acquisition.members);
166
+ const members = await memberAvailability(files, groups.flatMap(function completeGroup(group) { return group.members; }), signal);
167
+ const selection = selections.find(function matching(item) {
168
+ return item.members.length === acquisition.members.length && item.members.every(function samePath(member, index) {
169
+ const original = acquisition.members[index];
170
+ return member.path === original.path && (member.url === undefined || member.url === original.url)
171
+ && (member.archiveMember === undefined || member.archiveMember === original.archiveMember);
172
+ });
173
+ });
174
+ const record = discoveryRecord(files, members, 'acquisition', selection, discoveryEntry(groups, selection?.entryPath));
175
+ if (!acquisitionComplete) record.complete = false;
176
+ add(record);
177
+ return;
178
+ }
179
+ for (const selection of selections) {
180
+ const groups = groupedMembers(selection.members);
181
+ const members = await memberAvailability(directory, groups.flatMap(function completeGroup(group) { return group.members; }), signal);
182
+ if (!members.some(function present(member) { return member.available; })) continue;
183
+ add(discoveryRecord(directory, members, 'catalog', selection, discoveryEntry(groups, selection.entryPath)));
184
+ for (const member of members) claimed.add(member.nativePath);
185
+ }
186
+ const ggufMembers = entries.filter(function gguf(entry) {
187
+ const file = linked.has(entry) ? linked.get(entry)?.isFile() ?? true : entry.isFile();
188
+ return file && entry.name.toLowerCase().endsWith('.gguf') && !claimed.has(path.join(directory, entry.name));
189
+ }).map(function memberPath(entry) { return {path: entry.name}; });
190
+ for (const group of groupedMembers(ggufMembers)) {
191
+ add(discoveryRecord(directory, await memberAvailability(directory, group.members, signal), 'directory', null, group.entryPath));
192
+ }
193
+ if (recursive) {
194
+ for (const entry of entries) {
195
+ if (linked.has(entry) ? linked.get(entry)?.isDirectory() : entry.isDirectory()) await scan(path.join(directory, entry.name));
196
+ }
197
+ }
198
+ }
199
+
200
+ for (const directory of directories) await scan(path.resolve(appRoot, directory), true);
201
+ signal?.throwIfAborted();
202
+ return {directories: roots, models};
203
+ }
204
+
205
+ async function storedAcquisitions(directory, members, signal) {
206
+ const acquisitions = [];
34
207
  for (const entry of await readdir(directory, {withFileTypes: true})) {
35
208
  signal.throwIfAborted();
36
209
  if (!entry.isDirectory() || !entry.name.startsWith('acquisition-')) continue;
37
210
  const root = path.join(directory, entry.name);
38
211
  let selected;
39
- try { selected = JSON.parse(await readFile(path.join(root, 'acquisition.json'), {encoding: 'utf8', signal})); }
40
- catch (error) { if (error.code === 'ENOENT') continue; throw error; }
212
+ let committed = false;
213
+ try { selected = JSON.parse(await readFile(path.join(root, 'acquisition.json'), {encoding: 'utf8', signal})); committed = true; }
214
+ catch (error) { if (error.code !== 'ENOENT') throw error; }
215
+ if (!selected) {
216
+ try { selected = JSON.parse(await readFile(path.join(root, 'transfer.json'), {encoding: 'utf8', signal})); }
217
+ catch (error) { if (error.code === 'ENOENT') continue; throw error; }
218
+ }
41
219
  if (!sameMembers(selected.members, members)) continue;
42
- nativeAcquisitionOrder = Math.max(nativeAcquisitionOrder, selected.order);
220
+ nativeAcquisitionOrder = Math.max(nativeAcquisitionOrder, selected.order ?? 0);
43
221
  const files = path.join(root, 'files');
44
222
  const locations = members.map(function originalLocation(member) { return memberLocation(files, member); });
45
- let available = true;
46
- for (const member of locations) {
223
+ const completedMembers = [];
224
+ for (const [index, member] of locations.entries()) {
225
+ signal.throwIfAborted();
226
+ // Only an atomically promoted final filename is a completed member
227
+ // in the resumable layout. Older completion records keep their meaning.
228
+ try { if ((await stat(member.nativePath)).isFile()) completedMembers.push(index); }
229
+ catch (error) { if (error.code !== 'ENOENT') throw error; }
230
+ }
231
+ acquisitions.push({root, name: entry.name, order: selected.order ?? 0, refresh: selected.refresh === true,
232
+ complete: committed && completedMembers.length === members.length, completedMembers, directory: files, members: locations});
233
+ }
234
+ acquisitions.sort(function newestAcquisition(left, right) { return right.order - left.order || right.name.localeCompare(left.name); });
235
+ return acquisitions;
236
+ }
237
+
238
+ async function saveAcquisitionState(root, name, record) {
239
+ const pending = path.join(root, `${name}.${randomUUID()}.pending`);
240
+ await writeFile(pending, JSON.stringify(record), {encoding: 'utf8', flag: 'wx'});
241
+ await rename(pending, path.join(root, name));
242
+ }
243
+
244
+ function processExists(pid) {
245
+ try { process.kill(pid, 0); return true; }
246
+ catch (error) { if (error.code === 'ESRCH') return false; throw error; }
247
+ }
248
+
249
+ /** Atomic directory publication keeps an owner record visible with its lock. */
250
+ async function acquireDirectoryOwner(directory, name, signal, waiting) {
251
+ const lock = path.join(directory, name);
252
+ const candidate = await mkdtemp(path.join(directory, `${name}-candidate-`));
253
+ const owner = {pid: process.pid, token: randomUUID(), acquiredAt: new Date().toISOString(),
254
+ scope: lock, expires: 'When the recorded process exits; live ownership has no elapsed-time expiry.',
255
+ release: 'Retire the owned directory atomically after owned I/O settles, then remove its own metadata.',
256
+ recovery: 'A contender may recover this directory only after its recorded process has exited.'};
257
+ let acquired = false;
258
+ async function retireOwner(selectedLock) {
259
+ const retired = path.join(directory, `${name}-released-${randomUUID()}`);
260
+ await rename(selectedLock, retired);
261
+ await unlink(path.join(retired, 'owner.json'));
262
+ await rmdir(retired);
263
+ }
264
+ let acquisitionError;
265
+ try {
266
+ await writeFile(path.join(candidate, 'owner.json'), JSON.stringify(owner), {encoding: 'utf8', flag: 'wx'});
267
+ while (!acquired) {
47
268
  signal.throwIfAborted();
48
- try { if (!(await stat(member.nativePath)).isFile()) available = false; }
49
- catch (error) { if (error.code !== 'ENOENT') throw error; available = false; }
269
+ let notify;
270
+ let rejectWait;
271
+ const changed = new Promise(function observeOwnerChange(resolve, reject) { notify = resolve; rejectWait = reject; });
272
+ changed.catch(function observeWaitRejection() {});
273
+ const watcher = watch(directory, function ownerChanged() { notify(); });
274
+ watcher.on('error', rejectWait);
275
+ // A crashed process emits no filesystem event. This heartbeat checks
276
+ // that exact owner, and also keeps a live wait observable to callers.
277
+ const heartbeat = setTimeout(notify, 5000);
278
+ heartbeat.unref?.();
279
+ function aborted() { rejectWait(signal.reason); }
280
+ signal.addEventListener('abort', aborted, {once: true});
281
+ try {
282
+ try { await rename(candidate, lock); acquired = true; }
283
+ catch (error) {
284
+ if (!['EEXIST', 'ENOTEMPTY', 'EPERM', 'EACCES'].includes(error.code)) throw error;
285
+ let active;
286
+ try { active = JSON.parse(await readFile(path.join(lock, 'owner.json'), 'utf8')); }
287
+ catch (readError) {
288
+ if (readError.code !== 'ENOENT') throw readError;
289
+ try {
290
+ // Empty ownerless directories can remain from an
291
+ // older interrupted release. rmdir cannot remove a
292
+ // newly published nonempty owner directory.
293
+ await rmdir(lock);
294
+ continue;
295
+ } catch (emptyError) {
296
+ if (emptyError.code === 'ENOENT') {
297
+ if (error.code === 'EPERM' || error.code === 'EACCES') throw error;
298
+ continue;
299
+ }
300
+ if (!['ENOTEMPTY', 'EEXIST'].includes(emptyError.code)) throw emptyError;
301
+ }
302
+ }
303
+ if (active && !processExists(active.pid)) {
304
+ let current;
305
+ try { current = JSON.parse(await readFile(path.join(lock, 'owner.json'), 'utf8')); }
306
+ catch (readError) { if (readError.code === 'ENOENT') continue; throw readError; }
307
+ // The old process may have released normally and then
308
+ // exited after our first read. Do not retire its successor.
309
+ if (current.token !== active.token) continue;
310
+ // Keep this nonempty operation record after recovery.
311
+ // A delayed contender for the same dead owner cannot
312
+ // replace it with a newly published live owner's lock.
313
+ const retired = path.join(directory, `${name}-exited-${active.token}`);
314
+ try { await rename(lock, retired); }
315
+ catch (recoveryError) {
316
+ if (recoveryError.code === 'ENOENT') continue;
317
+ if (!['EEXIST', 'ENOTEMPTY', 'EPERM', 'EACCES'].includes(recoveryError.code)) throw recoveryError;
318
+ const recovered = JSON.parse(await readFile(path.join(retired, 'owner.json'), 'utf8'));
319
+ if (recovered.token !== active.token) throw recoveryError;
320
+ }
321
+ continue;
322
+ }
323
+ await waiting?.(active);
324
+ signal.throwIfAborted();
325
+ await changed;
326
+ }
327
+ } finally {
328
+ clearTimeout(heartbeat);
329
+ watcher.close();
330
+ signal.removeEventListener('abort', aborted);
331
+ }
332
+ }
333
+ } catch (error) { acquisitionError = error; throw error; }
334
+ finally {
335
+ if (!acquired) {
336
+ try {
337
+ try { await unlink(path.join(candidate, 'owner.json')); }
338
+ catch (error) { if (error.code !== 'ENOENT') throw error; }
339
+ await rmdir(candidate);
340
+ } catch (error) {
341
+ if (acquisitionError) throw new AggregateError([acquisitionError, error], 'Acquiring model ownership and releasing its candidate failed.');
342
+ throw error;
343
+ }
344
+ }
345
+ }
346
+ return async function releaseDirectoryOwner() {
347
+ const current = JSON.parse(await readFile(path.join(lock, 'owner.json'), 'utf8'));
348
+ if (current.token !== owner.token) throw failure('MODEL_ASSET_OWNER_CHANGED', 'Model acquisition ownership changed before release.');
349
+ await retireOwner(lock);
350
+ };
351
+ }
352
+
353
+ async function retainRestartedPart(attempt, memberIndex) {
354
+ const partial = path.join(attempt, 'parts', `${memberIndex}.part`);
355
+ const previous = path.join(attempt, 'parts', `${memberIndex}.previous.part`);
356
+ try { await stat(previous); }
357
+ catch (error) {
358
+ if (error.code !== 'ENOENT') throw error;
359
+ try { await rename(partial, previous); return; }
360
+ catch (renameError) { if (renameError.code !== 'ENOENT') throw renameError; return; }
361
+ }
362
+ // Retain the earlier resumable original while replacing only this member's
363
+ // failed replacement. Repeated retries do not accumulate duplicate parts.
364
+ try { await unlink(partial); }
365
+ catch (error) { if (error.code !== 'ENOENT') throw error; }
366
+ }
367
+
368
+ async function selectNativeAcquisition(entry, progress) {
369
+ const {directory, members, controller, refresh} = entry;
370
+ const {signal} = controller;
371
+ const release = await acquireDirectoryOwner(directory, '.selection-owner', signal, function waitingForSelection() { progress('waiting'); });
372
+ let selectionError;
373
+ try {
374
+ const stored = await storedAcquisitions(directory, members, signal);
375
+ if (!refresh) {
376
+ const complete = stored.find(function completeAcquisition(candidate) { return candidate.complete; });
377
+ if (complete) return complete;
378
+ }
379
+ const resumable = stored.find(function incompleteAcquisition(candidate) { return !candidate.complete && (!refresh || candidate.refresh); });
380
+ if (resumable) return resumable;
381
+ const root = await mkdtemp(path.join(directory, 'acquisition-'));
382
+ const files = path.join(root, 'files');
383
+ await mkdir(files);
384
+ await mkdir(path.join(root, 'parts'));
385
+ const order = ++nativeAcquisitionOrder;
386
+ await saveAcquisitionState(root, 'transfer.json', {members, order, refresh, completedMembers: [], transports: {}});
387
+ return {root, order, refresh, complete: false, completedMembers: [], directory: files,
388
+ members: members.map(function selectedMember(member) { return memberLocation(files, member); })};
389
+ } catch (error) { selectionError = error; throw error; }
390
+ finally {
391
+ try { await release(); }
392
+ catch (error) {
393
+ if (selectionError) throw new AggregateError([selectionError, error], 'Selecting model assets and releasing selection ownership failed.');
394
+ throw error;
395
+ }
396
+ }
397
+ }
398
+
399
+ async function fetchModelResponse(url, signal, range) {
400
+ const response = await fetch(url, {signal, ...(range ? {headers: {Range: `bytes=${range.start}-${range.end}`}} : {})});
401
+ if (!response.ok) {
402
+ const error = failure('MODEL_ASSET_DOWNLOAD_FAILED', `Model asset request failed: ${response.status} ${response.statusText}`);
403
+ error.url = url;
404
+ error.status = response.status;
405
+ error.response = await response.text();
406
+ throw error;
407
+ }
408
+ if (!response.body) throw failure('MODEL_ASSET_DOWNLOAD_FAILED', `Model asset response has no content stream: ${url}`);
409
+ if (range) {
410
+ const contentRange = /^bytes (\d+)-(\d+)\/(?:\d+|\*)$/.exec(response.headers.get('content-range') ?? '');
411
+ if (response.status !== 206 || !contentRange || Number(contentRange[1]) !== range.start || Number(contentRange[2]) !== range.end) {
412
+ const error = failure('MODEL_ASSET_ARCHIVE_RANGE_FAILED', 'The archive server did not return the requested TAR segment.');
413
+ error.url = url;
414
+ error.status = response.status;
415
+ try { await response.body.cancel(error); }
416
+ catch (cleanupError) { throw new AggregateError([error, cleanupError], 'Rejecting an unusable HTTP range and closing its response failed.'); }
417
+ throw error;
50
418
  }
51
- if (available && (!latest || selected.order > latest.order
52
- || (selected.order === latest.order && entry.name > latest.name))) {
53
- latest = {name: entry.name, order: selected.order, directory: files, members: locations};
419
+ }
420
+ return response;
421
+ }
422
+
423
+ function tarName(name) { return name.replace(/^(?:\.\/)+/, ''); }
424
+
425
+ function tarText(header, start, end) {
426
+ const field = header.subarray(start, end);
427
+ const terminator = field.indexOf(0);
428
+ return field.subarray(0, terminator === -1 ? field.length : terminator).toString('utf8');
429
+ }
430
+
431
+ function paxFields(content) {
432
+ const fields = {};
433
+ let offset = 0;
434
+ while (offset < content.length) {
435
+ const space = content.indexOf(32, offset);
436
+ const length = Number(content.subarray(offset, space).toString('ascii'));
437
+ if (space === -1 || !Number.isSafeInteger(length) || length <= space - offset + 1 || offset + length > content.length) {
438
+ throw failure('MODEL_ASSET_ARCHIVE_FORMAT', 'The TAR PAX record is incomplete or unreadable.');
54
439
  }
440
+ const record = content.subarray(space + 1, offset + length - 1).toString('utf8');
441
+ const equals = record.indexOf('=');
442
+ if (equals !== -1) fields[record.substring(0, equals)] = record.substring(equals + 1);
443
+ offset += length;
55
444
  }
56
- return latest ? {directory: latest.directory, members: latest.members} : null;
445
+ return fields;
446
+ }
447
+
448
+ // TAR framing and HTTP range offsets stay inside this transport reader. They
449
+ // never describe model readiness, download progress or content identity.
450
+ async function extractTarRange(response, range, selected, found, entry, progress, completeMember) {
451
+ const {signal} = entry.controller;
452
+ const reader = response.body.getReader();
453
+ let pending = Buffer.alloc(0);
454
+ let ended = false;
455
+ let localPax = {};
456
+ let globalPax = {};
457
+ let operationError;
458
+
459
+ async function consume(length, consumePart, allowEnd = false) {
460
+ let remaining = length;
461
+ while (remaining) {
462
+ signal.throwIfAborted();
463
+ if (!pending.length && !ended) {
464
+ const chunk = await reader.read();
465
+ ended = chunk.done;
466
+ if (!ended) { pending = Buffer.from(chunk.value); entry.activity(); }
467
+ }
468
+ if (!pending.length && ended) {
469
+ if (allowEnd && remaining === length) return false;
470
+ throw failure('MODEL_ASSET_ARCHIVE_INCOMPLETE', 'The TAR segment ended inside a member or header.');
471
+ }
472
+ const consumed = Math.min(remaining, pending.length);
473
+ if (consumePart) await consumePart(pending.subarray(0, consumed));
474
+ pending = pending.subarray(consumed);
475
+ remaining -= consumed;
476
+ }
477
+ return true;
478
+ }
479
+
480
+ try {
481
+ while (true) {
482
+ const headers = [];
483
+ if (!await consume(512, function headerPart(part) { headers.push(part); }, true)) break;
484
+ const header = Buffer.concat(headers);
485
+ if (header.every(function empty(value) { return value === 0; })) break;
486
+ const type = String.fromCharCode(header[156]);
487
+ const prefix = tarText(header, 345, 500);
488
+ const name = tarText(header, 0, 100);
489
+ const framingLength = Number.parseInt(tarText(header, 124, 136).trim(), 8);
490
+ const length = type === 'x' || type === 'g' ? framingLength : Number(localPax.size ?? globalPax.size ?? framingLength);
491
+ if (!Number.isSafeInteger(length) || length < 0) throw failure('MODEL_ASSET_ARCHIVE_FORMAT', 'The TAR member has unreadable framing.');
492
+ if (type === 'x' || type === 'g') {
493
+ const parts = [];
494
+ await consume(length, function paxPart(part) { parts.push(part); });
495
+ const fields = paxFields(Buffer.concat(parts));
496
+ if (type === 'g') globalPax = {...globalPax, ...fields};
497
+ else localPax = fields;
498
+ await consume((512 - length % 512) % 512);
499
+ continue;
500
+ }
501
+ const memberName = tarName(localPax.path ?? globalPax.path ?? (prefix ? `${prefix}/${name}` : name));
502
+ localPax = {};
503
+ if (range?.stopBefore && (memberName === range.stopBefore || path.posix.basename(memberName) === range.stopBefore)) break;
504
+ const memberIndexes = selected.get(memberName);
505
+ if (memberIndexes && (type === '0' || type === '\0')) {
506
+ if (memberIndexes.some(function alreadyRead(index) { return found.has(index); })) {
507
+ throw failure('MODEL_ASSET_ARCHIVE_FORMAT', `The selected TAR member occurs more than once: ${memberName}`);
508
+ }
509
+ const handles = [];
510
+ const errors = [];
511
+ try {
512
+ for (const memberIndex of memberIndexes) {
513
+ const location = entry.location.members[memberIndex];
514
+ await mkdir(path.dirname(location.nativePath), {recursive: true});
515
+ const partial = await entry.archivePart(memberIndex);
516
+ handles.push(await open(partial, 'wx'));
517
+ progress('download', memberIndex);
518
+ }
519
+ await consume(length, async function originalMemberContent(content) {
520
+ const writes = await Promise.allSettled(handles.map(function appendOriginal(handle) { return handle.appendFile(content); }));
521
+ const failures = writes.filter(function failed(result) { return result.status === 'rejected'; }).map(function reason(result) { return result.reason; });
522
+ if (failures.length) throw new AggregateError(failures, 'Writing selected TAR member destinations failed.');
523
+ for (const memberIndex of memberIndexes) progress('download', memberIndex, true);
524
+ });
525
+ } catch (error) { errors.push(error); }
526
+ const closes = await Promise.allSettled(handles.map(function closeOriginal(handle) { return handle.close(); }));
527
+ for (const result of closes) if (result.status === 'rejected') errors.push(result.reason);
528
+ if (errors.length === 1) throw errors[0];
529
+ if (errors.length) throw new AggregateError(errors, 'Extracting a model member and closing its files failed.');
530
+ for (const memberIndex of memberIndexes) {
531
+ found.add(memberIndex);
532
+ await completeMember(memberIndex);
533
+ }
534
+ } else await consume(length);
535
+ await consume((512 - length % 512) % 512);
536
+ }
537
+ } catch (error) { operationError = error; }
538
+ try { if (!ended) await reader.cancel(operationError); }
539
+ catch (error) { operationError = operationError ? new AggregateError([operationError, error], 'Reading a TAR segment and closing its response failed.') : error; }
540
+ try { reader.releaseLock(); }
541
+ catch (error) { operationError = operationError ? new AggregateError([operationError, error], 'Reading a TAR segment and releasing its reader failed.') : error; }
542
+ if (operationError) throw operationError;
543
+ }
544
+
545
+ async function downloadNativeMember(entry, memberIndex, state, save, progress, completeMember, noteFailure) {
546
+ const member = entry.members[memberIndex];
547
+ const {signal} = entry.controller;
548
+ const partial = path.join(entry.attempt, 'parts', `${memberIndex}.part`);
549
+ let offset = 0;
550
+ try { offset = (await stat(partial)).size; }
551
+ catch (error) { if (error.code !== 'ENOENT') throw error; }
552
+ const previous = state.transports[memberIndex];
553
+ let response;
554
+ let reader;
555
+ let handle;
556
+ let cancellation;
557
+ let ended = false;
558
+ let operationError;
559
+ let remaining;
560
+
561
+ async function restart(reason) {
562
+ progress('restart', memberIndex, false, {reason});
563
+ await retainRestartedPart(entry.attempt, memberIndex);
564
+ offset = 0;
565
+ }
566
+ async function request(headers) {
567
+ try { return await fetch(member.url, {signal, headers}); }
568
+ catch (error) { noteFailure(error, 'interrupted'); throw error; }
569
+ }
570
+ function cancelReader() {
571
+ if (!reader || ended) return;
572
+ cancellation ??= reader.cancel(signal.reason);
573
+ cancellation.catch(function observeReaderCancellation() {});
574
+ }
575
+ try {
576
+ signal.throwIfAborted();
577
+ if (offset && previous?.encoding && previous.encoding !== 'identity') await restart('The previous response used encoded HTTP content and cannot be appended by Range.');
578
+ const validator = previous?.etag && !previous.etag.startsWith('W/') ? previous.etag : previous?.lastModified;
579
+ if (offset && !validator) await restart('The source supplied no HTTP validator for continuing its earlier response.');
580
+ const headers = {'Accept-Encoding': 'identity'};
581
+ if (offset) {
582
+ headers.Range = `bytes=${offset}-`;
583
+ headers['If-Range'] = validator;
584
+ progress('resume', memberIndex);
585
+ } else progress('download', memberIndex);
586
+ response = await request(headers);
587
+ signal.throwIfAborted();
588
+ if (offset && response.status === 416) {
589
+ await response.body?.cancel();
590
+ response = undefined;
591
+ await restart('The server no longer accepts this saved HTTP range.');
592
+ response = await request({'Accept-Encoding': 'identity'});
593
+ }
594
+ if (!response.ok) {
595
+ const error = failure('MODEL_ASSET_DOWNLOAD_FAILED', `Model asset request failed: ${response.status} ${response.statusText}`);
596
+ error.url = member.url;
597
+ error.status = response.status;
598
+ error.response = await response.text();
599
+ noteFailure(error, response.status >= 500 || response.status === 408 || response.status === 429 ? 'interrupted' : 'failed');
600
+ throw error;
601
+ }
602
+ if (!response.body) throw failure('MODEL_ASSET_DOWNLOAD_FAILED', `Model asset response has no content stream: ${member.url}`);
603
+ if (offset && response.status === 200) await restart('The server returned the complete file instead of the saved HTTP range.');
604
+ if (response.status === 206) {
605
+ const range = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(response.headers.get('content-range') ?? '');
606
+ const encoding = response.headers.get('content-encoding');
607
+ const validatorHeader = previous?.etag && !previous.etag.startsWith('W/') ? 'etag' : 'last-modified';
608
+ const currentValidator = response.headers.get(validatorHeader);
609
+ const changed = currentValidator && currentValidator !== validator;
610
+ if (!offset || !range || Number(range[1]) !== offset || Number(range[2]) < offset
611
+ || Number(range[2]) + 1 !== Number(range[3]) || (encoding && encoding !== 'identity') || changed) {
612
+ const error = failure('MODEL_ASSET_DOWNLOAD_RANGE_FAILED', 'The server did not return the requested continuation of this model file.');
613
+ error.url = member.url;
614
+ error.status = response.status;
615
+ throw error;
616
+ }
617
+ // Content-Range framing is private to this HTTP transfer. It never
618
+ // participates in model identity, admission or user-facing progress.
619
+ remaining = Number(range[2]) - Number(range[1]) + 1;
620
+ }
621
+ state.transports[memberIndex] = {
622
+ etag: response.headers.get('etag'), lastModified: response.headers.get('last-modified'),
623
+ encoding: response.headers.get('content-encoding')
624
+ };
625
+ await save();
626
+ handle = await open(partial, 'a');
627
+ reader = response.body.getReader();
628
+ signal.addEventListener('abort', cancelReader, {once: true});
629
+ if (signal.aborted) cancelReader();
630
+ while (true) {
631
+ signal.throwIfAborted();
632
+ let chunk;
633
+ try { chunk = await reader.read(); }
634
+ catch (error) { noteFailure(error, 'interrupted'); throw error; }
635
+ signal.throwIfAborted();
636
+ if (chunk.done) { ended = true; break; }
637
+ if (remaining !== undefined) {
638
+ remaining -= chunk.value.byteLength;
639
+ if (remaining < 0) throw failure('MODEL_ASSET_DOWNLOAD_RANGE_FAILED', 'The server exceeded its HTTP Content-Range.');
640
+ }
641
+ await handle.appendFile(chunk.value);
642
+ progress('download', memberIndex, true);
643
+ }
644
+ if (remaining !== undefined && remaining !== 0) throw failure('MODEL_ASSET_DOWNLOAD_RANGE_FAILED', 'The server ended before its HTTP Content-Range.');
645
+ signal.throwIfAborted();
646
+ } catch (error) { operationError = error; noteFailure(error, 'failed'); }
647
+ signal.removeEventListener('abort', cancelReader);
648
+ try {
649
+ if (reader && !ended) { cancellation ??= reader.cancel(operationError); await cancellation; }
650
+ else if (!reader && response?.body && !response.bodyUsed && !response.body.locked) await response.body.cancel(operationError);
651
+ } catch (error) {
652
+ if (error !== operationError) operationError = operationError ? new AggregateError([operationError, error], 'Downloading a model member and closing its response failed.') : error;
653
+ }
654
+ try { reader?.releaseLock(); }
655
+ catch (error) { operationError = operationError ? new AggregateError([operationError, error], 'Downloading a model member and releasing its reader failed.') : error; }
656
+ try { await handle?.close(); }
657
+ catch (error) { operationError = operationError ? new AggregateError([operationError, error], 'Downloading a model member and closing its file failed.') : error; }
658
+ if (operationError) { noteFailure(operationError, 'failed'); throw operationError; }
659
+ await completeMember(memberIndex);
57
660
  }
58
661
 
59
662
  async function acquireNativeMembers(entry) {
60
- const {directory, members, controller} = entry;
663
+ const {members, controller} = entry;
61
664
  const {signal} = controller;
62
- let attempt;
63
665
  let completed = 0;
64
- let committed = false;
65
666
  let firstFailure;
66
- function progress(phase, memberIndex) {
67
- entry.publish({phase, completed, total: members.length, unit: 'files',
68
- ...(memberIndex === undefined ? {} : {memberIndex, path: members[memberIndex].path})});
667
+ let failureState;
668
+ let releaseOwner;
669
+ let state;
670
+ let stateTail = Promise.resolve();
671
+ let heartbeat;
672
+ let acquisitionError;
673
+ let lastActivity = Date.now();
674
+ entry.activity = function receivedTransportContent() { lastActivity = Date.now(); };
675
+ const activity = new Map();
676
+ function noteFailure(error, selectedState) {
677
+ if (firstFailure || signal.aborted) return;
678
+ firstFailure = error;
679
+ failureState = selectedState;
680
+ controller.abort(error);
681
+ }
682
+ function save() {
683
+ const next = stateTail.then(function saveTransferState() { return saveAcquisitionState(entry.attempt, 'transfer.json', state); });
684
+ stateTail = next.catch(function observeStateFailure(error) { noteFailure(error, 'failed'); });
685
+ return next;
686
+ }
687
+ function progress(phase, memberIndex, repeatedActivity = false, details = {}) {
688
+ if (phase === 'download') lastActivity = Date.now();
689
+ if (repeatedActivity) {
690
+ const now = Date.now();
691
+ if (now - (activity.get(memberIndex) ?? 0) < 1000) return;
692
+ activity.set(memberIndex, now);
693
+ }
694
+ const selectedState = details.state ?? ({waiting: 'waiting', stalled: 'stalled', interrupted: 'interrupted',
695
+ cancelled: 'cancelled', failed: 'failed', ready: 'ready'})[phase] ?? 'downloading';
696
+ const messages = {
697
+ waiting: 'Another application is preparing these model files. Waiting for its acquisition.',
698
+ downloading: 'Downloading model files.', stalled: 'The model download is still open; no recent data has arrived.',
699
+ interrupted: 'The model download was interrupted. Completed files and resumable data were kept.',
700
+ cancelled: 'The model download was cancelled. Completed files and resumable data were kept.',
701
+ failed: 'The model download failed. Existing files and resumable data were kept.', ready: 'Model files are ready.'
702
+ };
703
+ const notification = {phase, state: selectedState, message: phase === 'resume' ? 'Resuming the saved model download.'
704
+ : phase === 'restart' ? 'Restarting this model file because its saved HTTP continuation is unavailable. Earlier partial data was retained.'
705
+ : phase === 'reuse' ? 'Reusing downloaded model files.' : messages[selectedState],
706
+ completed, total: members.length, unit: 'files',
707
+ ...(memberIndex === undefined ? {} : {memberIndex, path: members[memberIndex].path}), ...details};
708
+ if (['interrupted', 'cancelled', 'failed'].includes(selectedState)) notification.recovery = 'Retry to reuse completed files and continue saved downloads.';
709
+ if (selectedState === 'stalled') notification.recovery = 'Wait for the source to respond, or cancel and retry later.';
710
+ if (state) state.progress = notification;
711
+ entry.publish(notification);
69
712
  }
70
713
  try {
71
714
  signal.throwIfAborted();
72
- const stored = await storedAcquisition(directory, members, signal);
73
- if (stored && !entry.refresh) {
74
- entry.location = stored;
715
+ const selected = await selectNativeAcquisition(entry, progress);
716
+ entry.attempt = selected.root;
717
+ entry.location = {directory: selected.directory, members: selected.members};
718
+ if (selected.complete) {
75
719
  completed = members.length;
76
720
  progress('reuse');
77
721
  progress('ready');
78
- return stored;
722
+ return entry.location;
79
723
  }
80
724
  signal.throwIfAborted();
81
- attempt = await mkdtemp(path.join(directory, 'acquisition-'));
82
- const files = path.join(attempt, 'files');
83
- await mkdir(files);
84
- entry.location = {directory: files, members: members.map(function selectedMember(member) { return memberLocation(files, member); })};
725
+ releaseOwner = await acquireDirectoryOwner(selected.root, '.transfer-owner', signal, async function waitingForTransfer() {
726
+ try {
727
+ const observed = JSON.parse(await readFile(path.join(selected.root, 'transfer.json'), 'utf8'));
728
+ completed = observed.completedMembers?.length ?? 0;
729
+ progress('waiting', undefined, false, {activity: observed.progress});
730
+ } catch (error) { if (error.code !== 'ENOENT') throw error; progress('waiting'); }
731
+ });
732
+ const current = (await storedAcquisitions(entry.directory, members, signal)).find(function ownedAttempt(candidate) { return candidate.root === selected.root; });
733
+ if (current?.complete) {
734
+ completed = members.length;
735
+ progress('reuse');
736
+ progress('ready');
737
+ return entry.location;
738
+ }
739
+ try { state = JSON.parse(await readFile(path.join(selected.root, 'transfer.json'), 'utf8')); }
740
+ catch (error) {
741
+ if (error.code !== 'ENOENT') throw error;
742
+ state = {members, order: selected.order, refresh: selected.refresh, transports: {}};
743
+ }
744
+ state.transports ??= {};
745
+ state.completedMembers = current?.completedMembers ?? [];
746
+ const finished = new Set(state.completedMembers);
747
+ completed = finished.size;
748
+ await mkdir(path.join(selected.root, 'parts'), {recursive: true});
85
749
  progress('open');
86
- const results = await Promise.allSettled(members.map(async function downloadMember(member, memberIndex) {
87
- let handle;
88
- let response;
89
- let downloadError;
90
- const failures = [];
91
- let bodyOwned = false;
750
+ await save();
751
+ heartbeat = setInterval(function reportTransferHeartbeat() {
752
+ const stalled = Date.now() - lastActivity >= 15000;
753
+ progress(stalled ? 'stalled' : 'heartbeat');
754
+ save().catch(function observeHeartbeatFailure(error) { noteFailure(error, 'failed'); });
755
+ }, 5000);
756
+ heartbeat.unref?.();
757
+ async function completeMember(memberIndex) {
758
+ const location = entry.location.members[memberIndex];
759
+ await mkdir(path.dirname(location.nativePath), {recursive: true});
760
+ await rename(path.join(entry.attempt, 'parts', `${memberIndex}.part`), location.nativePath);
761
+ finished.add(memberIndex);
762
+ state.completedMembers = [...finished];
763
+ completed = finished.size;
764
+ await save();
765
+ try { await unlink(path.join(entry.attempt, 'parts', `${memberIndex}.previous.part`)); }
766
+ catch (error) { if (error.code !== 'ENOENT') throw error; }
767
+ progress('download', memberIndex);
768
+ }
769
+ entry.archivePart = async function prepareArchivePart(memberIndex) {
770
+ const partial = path.join(entry.attempt, 'parts', `${memberIndex}.part`);
92
771
  try {
93
- signal.throwIfAborted();
94
- const nativePath = entry.location.members[memberIndex].nativePath;
95
- await mkdir(path.dirname(nativePath), {recursive: true});
96
- handle = await open(nativePath, 'wx');
97
- signal.throwIfAborted();
98
- progress('download', memberIndex);
99
- response = await fetch(member.url, {signal});
100
- if (!response.ok) {
101
- const error = failure('MODEL_ASSET_DOWNLOAD_FAILED', `Model asset request failed: ${response.status} ${response.statusText}`);
102
- error.url = member.url;
103
- error.status = response.status;
104
- error.response = await response.text();
772
+ await stat(partial);
773
+ await retainRestartedPart(entry.attempt, memberIndex);
774
+ progress('restart', memberIndex, false, {reason: 'The selected TAR segment must be replayed to recover its member framing.'});
775
+ } catch (error) { if (error.code !== 'ENOENT') throw error; }
776
+ return partial;
777
+ };
778
+ if (completed) progress('reuse');
779
+ const jobs = members.map(function ordinaryJob(member, memberIndex) {
780
+ if (member.archiveMember !== undefined || finished.has(memberIndex)) return null;
781
+ return async function downloadMember() {
782
+ try { await downloadNativeMember(entry, memberIndex, state, save, progress, completeMember, noteFailure); }
783
+ catch (error) { noteFailure(error, 'failed'); throw error; }
784
+ };
785
+ }).filter(Boolean);
786
+ let archiveIndex = members.findIndex(function archiveMember(member) { return member.archiveMember !== undefined; });
787
+ for (const archive of entry.archives) {
788
+ const selected = new Map();
789
+ for (const member of archive.members) {
790
+ const memberIndex = archiveIndex++;
791
+ if (finished.has(memberIndex)) continue;
792
+ const name = tarName(member.member);
793
+ if (!selected.has(name)) selected.set(name, []);
794
+ selected.get(name).push(memberIndex);
795
+ }
796
+ if (!selected.size) continue;
797
+ jobs.push(async function downloadArchiveMembers() {
798
+ const found = new Set();
799
+ try {
800
+ for (const range of archive.ranges ?? [undefined]) {
801
+ signal.throwIfAborted();
802
+ const response = await fetchModelResponse(archive.url, signal, range);
803
+ await extractTarRange(response, range, selected, found, entry, progress, completeMember);
804
+ }
805
+ const missing = [...selected.entries()].filter(function absent([name, indexes]) {
806
+ return indexes.some(function missingDestination(index) { return !found.has(index); });
807
+ })
808
+ .map(function memberName([name]) { return name; });
809
+ if (missing.length) {
810
+ const error = failure('MODEL_ASSET_ARCHIVE_MEMBERS_MISSING', 'The archive did not provide every selected model member.');
811
+ error.members = missing;
812
+ throw error;
813
+ }
814
+ } catch (error) {
815
+ noteFailure(error, error.code?.startsWith('MODEL_ASSET_ARCHIVE_') ? 'failed' : 'interrupted');
105
816
  throw error;
106
817
  }
107
- if (!response.body) throw failure('MODEL_ASSET_DOWNLOAD_FAILED', `Model asset response has no content stream: ${member.url}`);
108
- bodyOwned = true;
109
- for await (const content of response.body) {
110
- signal.throwIfAborted();
111
- await handle.appendFile(content);
112
- progress('download', memberIndex);
113
- }
818
+ });
819
+ }
820
+ let nextJob = 0;
821
+ const results = await Promise.allSettled(Array.from({length: Math.min(4, jobs.length)}, async function downloadWorker() {
822
+ while (nextJob < jobs.length) {
114
823
  signal.throwIfAborted();
115
- } catch (error) {
116
- downloadError = error;
117
- failures.push(error);
118
- firstFailure ??= error;
119
- controller.abort(error);
120
- }
121
- if (response?.body && !bodyOwned && !response.bodyUsed && !response.body.locked) {
122
- try { await response.body.cancel(downloadError); } catch (error) { if (error !== downloadError) failures.push(error); }
123
- }
124
- if (handle) {
125
- try { await handle.close(); } catch (error) { failures.push(error); }
824
+ await jobs[nextJob++]();
126
825
  }
127
- if (failures.length) {
128
- const error = failures.length === 1 ? failures[0] : new AggregateError(failures, 'Downloading a model member and closing its resources failed.');
129
- firstFailure ??= error;
130
- controller.abort(error);
131
- throw error;
132
- }
133
- completed += 1;
134
- progress('download', memberIndex);
135
826
  }));
136
- const failures = results.filter(function rejected(result) { return result.status === 'rejected'; })
137
- .map(function reason(result) { return result.reason; });
827
+ const failures = [...new Set(results.filter(function rejected(result) { return result.status === 'rejected'; })
828
+ .map(function reason(result) { return result.reason; }))];
138
829
  if (failures.length > 1) throw new AggregateError(failures, 'Downloading model asset members failed.', {cause: firstFailure});
139
830
  if (failures.length) throw failures[0];
140
831
  signal.throwIfAborted();
141
832
  progress('complete');
142
833
  // The record describes a completed acquisition, not content identity.
143
834
  // Publish it after every stream and file close, leaving older entries alone.
144
- const pending = path.join(attempt, 'acquisition.pending.json');
145
- await writeFile(pending, JSON.stringify({members, order: ++nativeAcquisitionOrder}), {encoding: 'utf8', flag: 'wx', signal});
835
+ await save();
146
836
  signal.throwIfAborted();
147
- await rename(pending, path.join(attempt, 'acquisition.json'));
148
- committed = true;
837
+ await saveAcquisitionState(entry.attempt, 'acquisition.json', {members, order: state.order, refresh: state.refresh});
838
+ clearInterval(heartbeat);
149
839
  progress('ready');
840
+ await save();
150
841
  return entry.location;
151
842
  } catch (error) {
152
- if (attempt && !committed) {
153
- try { await rm(attempt, {recursive: true}); }
154
- catch (cleanupError) { throw new AggregateError([error, cleanupError], 'Preparing native model assets failed and cleanup failed.'); }
843
+ acquisitionError = error;
844
+ clearInterval(heartbeat);
845
+ progress(failureState ?? (signal.aborted ? 'cancelled' : 'failed'), undefined, false, {error: serializeCoreError(error)});
846
+ if (state) {
847
+ try { await save(); }
848
+ catch (stateError) {
849
+ acquisitionError = new AggregateError([error, stateError], 'Preparing model assets and retaining their interrupted state failed.');
850
+ throw acquisitionError;
851
+ }
155
852
  }
156
853
  throw error;
854
+ } finally {
855
+ clearInterval(heartbeat);
856
+ await stateTail;
857
+ try { await releaseOwner?.(); }
858
+ catch (error) {
859
+ if (acquisitionError) throw new AggregateError([acquisitionError, error], 'Preparing model assets and releasing their acquisition owner failed.');
860
+ throw error;
861
+ }
157
862
  }
158
863
  }
159
864
 
160
- function shareNativeAcquisition({directory, members, refresh, signal, onProgress}) {
865
+ function shareNativeAcquisition({directory, members, archives, refresh, signal, onProgress}) {
161
866
  signal.throwIfAborted();
162
867
  const key = JSON.stringify([directory, members, refresh]);
163
868
  let entry = nativeAcquisitions.get(key);
164
869
  if (entry?.controller.signal.aborted) {
165
870
  return entry.task.catch(function observeRetiringAcquisition() {}).then(function acquireAfterDrain() {
166
- return shareNativeAcquisition({directory, members, refresh, signal, onProgress});
871
+ return shareNativeAcquisition({directory, members, archives, refresh, signal, onProgress});
167
872
  });
168
873
  }
169
874
  if (!entry) {
170
- entry = {directory, members, refresh, controller: new AbortController(), consumers: new Set(), task: null, location: null, progress: null};
875
+ entry = {directory, members, archives, refresh, controller: new AbortController(), consumers: new Set(), task: null, location: null, progress: null};
171
876
  nativeAcquisitions.set(key, entry);
172
877
  entry.publish = function publishProgress(progress) {
173
878
  entry.progress = progress;
@@ -228,6 +933,7 @@ function shareNativeAcquisition({directory, members, refresh, signal, onProgress
228
933
  /** Temporary browser projections and persistent selected native acquisitions. */
229
934
  export function createModelAssetService({appRoot = process.cwd()} = {}) {
230
935
  const projections = new Map();
936
+ const preparations = new Map();
231
937
  let context;
232
938
  let closing;
233
939
 
@@ -238,7 +944,8 @@ export function createModelAssetService({appRoot = process.cwd()} = {}) {
238
944
  return {path: member.path, nativePath: member.nativePath};
239
945
  }),
240
946
  preparationOwned: record.preparationOwned, uses: record.uses.size,
241
- error: record.error ? serializeCoreError(record.error) : null
947
+ error: record.error ? serializeCoreError(record.error) : null,
948
+ ...(record.progress ? {progress: record.progress} : {})
242
949
  };
243
950
  }
244
951
 
@@ -403,13 +1110,11 @@ export function createModelAssetService({appRoot = process.cwd()} = {}) {
403
1110
  return task;
404
1111
  }
405
1112
 
406
- async function prepare({id, workingDirectory, members, signal, onProgress, refresh = false} = {}) {
1113
+ async function prepare({id, workingDirectory, members = [], archives = [], signal, onProgress, refresh = false} = {}) {
407
1114
  const controller = new AbortController();
408
1115
  const operationSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
409
- const record = beginProjection({id, workingDirectory, members}, operationSignal);
410
- let selectedMembers;
411
- try { selectedMembers = members.map(function selectedMember(member) { return {path: member.path, url: member.url}; }); }
412
- catch (error) { return failedOperation(record, error); }
1116
+ const selectedMembers = selectedSources(members, archives);
1117
+ const record = beginProjection({id, workingDirectory, members: selectedMembers}, operationSignal);
413
1118
  record.persistent = true;
414
1119
  record.downloadController = controller;
415
1120
  // This task owns only acquisition. Failure cleanup may join it without
@@ -420,12 +1125,15 @@ export function createModelAssetService({appRoot = process.cwd()} = {}) {
420
1125
  await mkdir(selectedDirectory, {recursive: true});
421
1126
  const directory = await realpath(selectedDirectory);
422
1127
  operationSignal.throwIfAborted();
423
- return shareNativeAcquisition({directory, members: selectedMembers, refresh: refresh === true, signal: operationSignal,
1128
+ return shareNativeAcquisition({directory, members: selectedMembers, archives, refresh: refresh === true, signal: operationSignal,
424
1129
  async onProgress(value, location) {
425
1130
  if (location) {
426
1131
  record.directory = location.directory;
427
1132
  record.members = location.members;
428
1133
  }
1134
+ record.progress = value;
1135
+ record.state = value.state;
1136
+ publish(record);
429
1137
  await onProgress?.(value);
430
1138
  }});
431
1139
  });
@@ -444,6 +1152,84 @@ export function createModelAssetService({appRoot = process.cwd()} = {}) {
444
1152
  }
445
1153
  }
446
1154
 
1155
+ function discover(options) { return discoverDirectories(appRoot, options); }
1156
+
1157
+ async function openExisting({id, directory, members, signal} = {}) {
1158
+ const controller = new AbortController();
1159
+ const operationSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
1160
+ const record = beginProjection({id, workingDirectory: directory, members}, operationSignal);
1161
+ // Borrowed originals never become disposable working files, even when
1162
+ // opening fails or the caller releases while filesystem I/O is pending.
1163
+ record.persistent = true;
1164
+ record.downloadController = controller;
1165
+ record.opening = Promise.resolve().then(async function inspectExistingMembers() {
1166
+ record.directory = path.resolve(appRoot, directory);
1167
+ const locations = await memberAvailability(record.directory, members, operationSignal);
1168
+ const missing = locations.filter(function absent(member) { return !member.available; });
1169
+ if (missing.length) {
1170
+ const error = failure('MODEL_ASSET_MEMBERS_MISSING', 'The selected model has missing members.');
1171
+ error.members = missing.map(function memberPath(member) { return member.path; });
1172
+ throw error;
1173
+ }
1174
+ record.members = locations;
1175
+ });
1176
+ publish(record);
1177
+ try {
1178
+ await record.opening;
1179
+ operationSignal.throwIfAborted();
1180
+ record.state = 'ready';
1181
+ publish(record);
1182
+ return snapshot(record);
1183
+ } catch (error) { return failedOperation(record, error); }
1184
+ }
1185
+
1186
+ function prepareRequest(params, request) {
1187
+ if (preparations.has(params.id)) throw failure('MODEL_ASSET_PROJECTION_ACTIVE', `Model asset preparation ${params.id} already exists.`);
1188
+ const controller = new AbortController();
1189
+ const signal = AbortSignal.any([request.signal, controller.signal]);
1190
+ const operation = {controller, signal, task: null};
1191
+ preparations.set(params.id, operation);
1192
+ operation.task = prepare({...params, signal,
1193
+ onProgress(progress) { request.emit?.('modelAssets.progress', {id: params.id, progress}); }
1194
+ });
1195
+ // Retain failed completion until the caller's explicit cleanup control
1196
+ // joins it; local RPC cancellation alone settles before native cleanup.
1197
+ operation.task.then(function prepared() {
1198
+ preparations.delete(params.id);
1199
+ }, function preparationFailed() {
1200
+ if (!signal.aborted) preparations.delete(params.id);
1201
+ });
1202
+ return operation.task;
1203
+ }
1204
+
1205
+ function cancellationOnly(error, reason) {
1206
+ return error === reason || (error instanceof AggregateError && error.errors.length > 0
1207
+ && error.errors.every(function cancellation(member) { return cancellationOnly(member, reason); }));
1208
+ }
1209
+
1210
+ async function cancelPrepare({id}) {
1211
+ const operation = preparations.get(id);
1212
+ const errors = [];
1213
+ if (operation) {
1214
+ operation.controller.abort(failure('MODEL_ASSET_PROJECTION_RELEASED', 'Model asset preparation was cancelled.'));
1215
+ try { await operation.task; }
1216
+ catch (error) { if (!cancellationOnly(error, operation.signal.reason)) errors.push(error); }
1217
+ preparations.delete(id);
1218
+ }
1219
+ try { await release(id); } catch (error) { errors.push(error); }
1220
+ if (errors.length === 1) throw errors[0];
1221
+ if (errors.length) throw new AggregateError(errors, 'Cancelling native model preparation failed.');
1222
+ return {id, state: 'released'};
1223
+ }
1224
+
1225
+ async function releaseRequest({id}) {
1226
+ const operation = preparations.get(id);
1227
+ if (operation && projections.get(id)?.state !== 'ready') return cancelPrepare({id});
1228
+ const result = await release(id);
1229
+ preparations.delete(id);
1230
+ return result;
1231
+ }
1232
+
447
1233
  async function completeProjection({id}, request) {
448
1234
  const record = requireProjection(id);
449
1235
  await record.opening;
@@ -505,24 +1291,27 @@ export function createModelAssetService({appRoot = process.cwd()} = {}) {
505
1291
  const results = await Promise.allSettled(records.map(function awaitRelease(record) { return record.released; }));
506
1292
  const errors = results.filter(function rejected(result) { return result.status === 'rejected'; })
507
1293
  .map(function reason(result) { return result.reason; });
1294
+ preparations.clear();
508
1295
  if (errors.length) throw new AggregateError(errors, 'Closing model asset preparation failed.');
509
1296
  });
510
1297
  return closing;
511
1298
  }
512
1299
 
513
1300
  return {
514
- name: 'model-assets', current, prepare, retain, release,
1301
+ name: 'model-assets', current, discover, openExisting, prepare, retain, release,
515
1302
  start(serviceContext) { context = serviceContext; },
516
1303
  methods: {
517
1304
  'modelAssets.open': openProjection,
1305
+ 'modelAssets.discover': function discoverRequest(params, request) { return discover({...params, signal: request.signal}); },
1306
+ 'modelAssets.openExisting': function openExistingRequest(params, request) { return openExisting({...params, signal: request.signal}); },
1307
+ 'modelAssets.prepare': prepareRequest,
1308
+ 'modelAssets.cancelPrepare': {lifetime: 'service', handle: cancelPrepare},
518
1309
  'modelAssets.write': writeMember,
519
1310
  'modelAssets.complete': completeProjection,
520
1311
  'modelAssets.status': function status() { return current(); },
521
1312
  'modelAssets.release': {
522
1313
  lifetime: 'service',
523
- handle({id}) {
524
- return release(id);
525
- }
1314
+ handle: releaseRequest
526
1315
  }
527
1316
  },
528
1317
  dispose