@openspecui/server 3.10.0 → 3.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { BatchTranslateInputSchema, CliExecutor, CodeEditorThemeSchema, ConfigManager, CustomSoundHashSchema, DASHBOARD_METRIC_KEYS, DashboardConfigSchema, DocumentTranslationConfigSchema, GitConfigSchema, GlobalSettingsManager, LocalModelAssetStateSchema, MarkdownParser, NotificationPublishInputSchema, NotificationSettingsSchema, OPENSPECUI_HOOKS_VERSION, OpenSpecAdapter, OpenSpecUIGlobalSettingsSchema, OpenSpecWatcher, OpsxConfigSchema, OpsxKernel, PtyClientMessageSchema, ReactiveContext, ServiceTranslationEngineIdSchema, TRANSLATION_ENGINE_MANIFESTS, TerminalConfigSchema, TerminalControlParser, TerminalRendererEngineSchema, TranslationCacheReadInputSchema, TranslationCacheSettingsSchema, TranslationCacheWriteInputSchema, TranslationEngineIdSchema, TranslationLocalSettingsSchema, TranslationOpenAISettingsSchema, buildBackendHealthPayload, buildLocalDownloadPlanFromRepositoryFiles, clearCache, getAllTools, getAvailableTools, getConfiguredTools, getDefaultCliCommandString, getDetectedProjectTools, getToolInitStates, getWatcherRuntimeStatus, initWatcherPool, isWatcherPoolInitialized, parseOpsxEntityMetadata, parseOpsxSchemaDetail, resolveTerminalShellDefaults, selectLocalDownloadGroup, sniffGlobalCli, subscribeWatcherRuntimeStatus, terminalNotificationEventToPublishInput } from "@openspecui/core";
|
|
2
|
-
import { basename, dirname, join, matchesGlob, relative, resolve, sep } from "node:path";
|
|
1
|
+
import { BatchTranslateInputSchema, CliExecutor, CodeEditorThemeSchema, ConfigManager, CustomSoundHashSchema, DASHBOARD_METRIC_KEYS, DashboardConfigSchema, DocumentTranslationConfigSchema, GitConfigSchema, GlobalSettingsManager, LocalModelAssetStateSchema, LocalModelLifecycleFileStateSchema, LocalModelLifecycleGroupStateSchema, LocalModelProfileManifestSchema, MarkdownParser, NotificationPublishInputSchema, NotificationSettingsSchema, OPENSPECUI_HOOKS_VERSION, OpenSpecAdapter, OpenSpecUIGlobalSettingsSchema, OpenSpecWatcher, OpsxConfigSchema, OpsxKernel, PtyClientMessageSchema, ReactiveContext, ServiceTranslationEngineIdSchema, TRANSLATION_ENGINE_MANIFESTS, TerminalConfigSchema, TerminalControlParser, TerminalRendererEngineSchema, TranslationCacheReadInputSchema, TranslationCacheSettingsSchema, TranslationCacheWriteInputSchema, TranslationEngineIdSchema, TranslationLocalSettingsSchema, TranslationOpenAISettingsSchema, buildBackendHealthPayload, buildLocalDownloadPlanFromRepositoryFiles, checkLocalDirectionalModelLanguagePair, clearCache, getAllTools, getAvailableTools, getConfiguredTools, getDefaultCliCommandString, getDetectedProjectTools, getOpsxEntityRootRelativePath, getToolInitStates, getWatcherRuntimeStatus, inferFileMime, inferFilePreviewKind, initWatcherPool, isWatcherPoolInitialized, normalizeOpsxEntityPath, parseOpsxEntityMetadata, parseOpsxSchemaDetail, resolveTerminalShellDefaults, selectLocalDownloadGroup, sniffGlobalCli, subscribeWatcherRuntimeStatus, terminalNotificationEventToPublishInput } from "@openspecui/core";
|
|
2
|
+
import { basename, dirname, extname, join, matchesGlob, relative, resolve, sep } from "node:path";
|
|
3
3
|
import { access, copyFile, lstat, mkdir, open, readFile, readlink, realpath, rename, rm, stat, symlink, unlink, writeFile } from "node:fs/promises";
|
|
4
4
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
5
5
|
import { createServer as createServer$1 } from "node:net";
|
|
@@ -8,7 +8,7 @@ import { fetchRequestHandler } from "@trpc/server/adapters/fetch";
|
|
|
8
8
|
import { applyWSSHandler } from "@trpc/server/adapters/ws";
|
|
9
9
|
import { Hono } from "hono";
|
|
10
10
|
import { cors } from "hono/cors";
|
|
11
|
-
import { existsSync, readFileSync } from "node:fs";
|
|
11
|
+
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
12
12
|
import { WebSocketServer } from "ws";
|
|
13
13
|
import { CustomSoundHashSchema as CustomSoundHashSchema$1, CustomSoundIdSchema, CustomSoundMetadataFileSchema, customHashFromSoundId, soundIdFromCustomHash } from "@openspecui/core/sounds";
|
|
14
14
|
import { createHash } from "node:crypto";
|
|
@@ -16,7 +16,7 @@ import { homedir, platform } from "node:os";
|
|
|
16
16
|
import { EventEmitter } from "node:events";
|
|
17
17
|
import { execFile } from "node:child_process";
|
|
18
18
|
import { promisify } from "node:util";
|
|
19
|
-
import { downloadFile, fileDownloadInfo, listFiles } from "@huggingface/hub";
|
|
19
|
+
import { downloadFile, fileDownloadInfo, listFiles, modelInfo } from "@huggingface/hub";
|
|
20
20
|
import { observable } from "@trpc/server/observable";
|
|
21
21
|
import { z } from "zod";
|
|
22
22
|
import { Agent, EnvHttpProxyAgent, setGlobalDispatcher } from "undici";
|
|
@@ -357,7 +357,7 @@ function normalizeHooksModule(moduleValue) {
|
|
|
357
357
|
onRunWorkflow: isOnRunWorkflowHook(record.onRunWorkflow) ? record.onRunWorkflow : isOnRunWorkflowHook(defaultRecord.onRunWorkflow) ? defaultRecord.onRunWorkflow : isOnRunWorkflowHook(moduleExportsRecord.onRunWorkflow) ? moduleExportsRecord.onRunWorkflow : void 0
|
|
358
358
|
};
|
|
359
359
|
}
|
|
360
|
-
async function pathExists$
|
|
360
|
+
async function pathExists$1(path) {
|
|
361
361
|
try {
|
|
362
362
|
await access(path);
|
|
363
363
|
return true;
|
|
@@ -386,8 +386,8 @@ var ProjectHookRuntime = class {
|
|
|
386
386
|
await Promise.allSettled(callbacks.map((cleanup) => cleanup()));
|
|
387
387
|
}
|
|
388
388
|
async loadFresh() {
|
|
389
|
-
if (!await pathExists$
|
|
390
|
-
const { tsImport } = await import("
|
|
389
|
+
if (!await pathExists$1(this.hooksPath)) return {};
|
|
390
|
+
const { tsImport } = await import("./api-C9I67iTa.mjs");
|
|
391
391
|
return normalizeHooksModule(await tsImport(`${pathToFileURL(this.hooksPath).href}?t=${Date.now()}`, { parentURL: pathToFileURL(this.hooksPath).href }));
|
|
392
392
|
}
|
|
393
393
|
};
|
|
@@ -799,7 +799,7 @@ function parseRelatedChanges(paths) {
|
|
|
799
799
|
continue;
|
|
800
800
|
}
|
|
801
801
|
}
|
|
802
|
-
return [...related].sort((a, b) => a.localeCompare(b));
|
|
802
|
+
return [...related].sort((a$1, b) => a$1.localeCompare(b));
|
|
803
803
|
}
|
|
804
804
|
async function resolveDefaultBranch(projectDir, runGit) {
|
|
805
805
|
const remoteHead = await runGit(projectDir, [
|
|
@@ -1067,9 +1067,9 @@ async function buildDashboardGitSnapshot(options) {
|
|
|
1067
1067
|
maxCommitEntries,
|
|
1068
1068
|
readPathTimestampMs
|
|
1069
1069
|
})));
|
|
1070
|
-
worktrees.sort((a, b) => {
|
|
1071
|
-
if (a.isCurrent !== b.isCurrent) return a.isCurrent ? -1 : 1;
|
|
1072
|
-
return a.branchName.localeCompare(b.branchName);
|
|
1070
|
+
worktrees.sort((a$1, b) => {
|
|
1071
|
+
if (a$1.isCurrent !== b.isCurrent) return a$1.isCurrent ? -1 : 1;
|
|
1072
|
+
return a$1.branchName.localeCompare(b.branchName);
|
|
1073
1073
|
});
|
|
1074
1074
|
return {
|
|
1075
1075
|
defaultBranch,
|
|
@@ -1092,7 +1092,7 @@ function createEmptyTrendSeries() {
|
|
|
1092
1092
|
return Object.fromEntries(DASHBOARD_METRIC_KEYS.map((metric) => [metric, []]));
|
|
1093
1093
|
}
|
|
1094
1094
|
function normalizeEvents(events, pointLimit) {
|
|
1095
|
-
return events.filter((event) => Number.isFinite(event.ts) && event.ts > 0 && Number.isFinite(event.value)).sort((a, b) => a.ts - b.ts).slice(-pointLimit);
|
|
1095
|
+
return events.filter((event) => Number.isFinite(event.ts) && event.ts > 0 && Number.isFinite(event.value)).sort((a$1, b) => a$1.ts - b.ts).slice(-pointLimit);
|
|
1096
1096
|
}
|
|
1097
1097
|
function buildTimeWindow(options) {
|
|
1098
1098
|
const { probeEvents, targetBars, rightEdgeTs } = options;
|
|
@@ -1450,6 +1450,160 @@ async function buildEntityReadOptions(ctx, stage, id) {
|
|
|
1450
1450
|
}
|
|
1451
1451
|
}
|
|
1452
1452
|
|
|
1453
|
+
//#endregion
|
|
1454
|
+
//#region src/entity-file-paths.ts
|
|
1455
|
+
function ensureInsideRoot(rootPath, candidatePath) {
|
|
1456
|
+
if (candidatePath === rootPath) return;
|
|
1457
|
+
if (!candidatePath.startsWith(rootPath + "/")) throw new Error("Resolved path escaped entity root.");
|
|
1458
|
+
}
|
|
1459
|
+
function getEntityRootPath(projectDir, stage, changeId) {
|
|
1460
|
+
return resolve(projectDir, getOpsxEntityRootRelativePath(stage, changeId));
|
|
1461
|
+
}
|
|
1462
|
+
function resolveEntityEntryPath(input) {
|
|
1463
|
+
const relativePath$1 = normalizeOpsxEntityPath(input.path);
|
|
1464
|
+
if (!relativePath$1) throw new Error("path is required");
|
|
1465
|
+
const entityRoot = getEntityRootPath(input.projectDir, input.stage, input.changeId);
|
|
1466
|
+
const absolutePath = resolve(entityRoot, relativePath$1);
|
|
1467
|
+
ensureInsideRoot(entityRoot, absolutePath);
|
|
1468
|
+
return {
|
|
1469
|
+
entityRoot,
|
|
1470
|
+
relativePath: relativePath$1,
|
|
1471
|
+
absolutePath
|
|
1472
|
+
};
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1475
|
+
//#endregion
|
|
1476
|
+
//#region src/file-preview-service.ts
|
|
1477
|
+
const PREVIEW_ENTRY_FILE_BY_KIND = {
|
|
1478
|
+
image: "image-preview.html",
|
|
1479
|
+
audio: "audio-preview.html",
|
|
1480
|
+
video: "video-preview.html",
|
|
1481
|
+
pdf: "pdf-preview.html"
|
|
1482
|
+
};
|
|
1483
|
+
const SESSION_PREVIEW_KINDS = new Set([
|
|
1484
|
+
"html",
|
|
1485
|
+
"image",
|
|
1486
|
+
"audio",
|
|
1487
|
+
"video",
|
|
1488
|
+
"pdf"
|
|
1489
|
+
]);
|
|
1490
|
+
function isSessionPreviewKind(previewKind) {
|
|
1491
|
+
return SESSION_PREVIEW_KINDS.has(previewKind);
|
|
1492
|
+
}
|
|
1493
|
+
function toHash(input) {
|
|
1494
|
+
return createHash("sha256").update(input).digest("hex");
|
|
1495
|
+
}
|
|
1496
|
+
function stripLeadingSlash(path) {
|
|
1497
|
+
return path.replace(/^\/+/, "");
|
|
1498
|
+
}
|
|
1499
|
+
function inferPreviewAssetContentType(path) {
|
|
1500
|
+
switch (extname(path).toLowerCase()) {
|
|
1501
|
+
case ".html": return "text/html";
|
|
1502
|
+
case ".js":
|
|
1503
|
+
case ".mjs": return "application/javascript";
|
|
1504
|
+
case ".css": return "text/css";
|
|
1505
|
+
case ".json": return "application/json";
|
|
1506
|
+
case ".svg": return "image/svg+xml";
|
|
1507
|
+
case ".png": return "image/png";
|
|
1508
|
+
case ".jpg":
|
|
1509
|
+
case ".jpeg": return "image/jpeg";
|
|
1510
|
+
case ".woff": return "font/woff";
|
|
1511
|
+
case ".woff2": return "font/woff2";
|
|
1512
|
+
default: return inferFileMime(path) ?? "application/octet-stream";
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
function isRewritablePreviewAsset(path) {
|
|
1516
|
+
const extension = extname(path).toLowerCase();
|
|
1517
|
+
return extension === ".html" || extension === ".js" || extension === ".mjs" || extension === ".css";
|
|
1518
|
+
}
|
|
1519
|
+
function rewritePreviewAssetPaths(content, hash) {
|
|
1520
|
+
const sessionAssetPrefix = `/api/file-preview/${hash}/assets/`;
|
|
1521
|
+
return content.replaceAll("/assets/", sessionAssetPrefix);
|
|
1522
|
+
}
|
|
1523
|
+
var FilePreviewService = class {
|
|
1524
|
+
sessions = /* @__PURE__ */ new Map();
|
|
1525
|
+
constructor(projectDir, previewAssetsDir) {
|
|
1526
|
+
this.projectDir = projectDir;
|
|
1527
|
+
this.previewAssetsDir = previewAssetsDir;
|
|
1528
|
+
}
|
|
1529
|
+
prepareEntityFilePreview(input) {
|
|
1530
|
+
const resolved = resolveEntityEntryPath({
|
|
1531
|
+
projectDir: this.projectDir,
|
|
1532
|
+
stage: input.stage,
|
|
1533
|
+
changeId: input.changeId,
|
|
1534
|
+
path: input.path
|
|
1535
|
+
});
|
|
1536
|
+
if (!statSync(resolved.absolutePath, { throwIfNoEntry: false })?.isFile()) throw new Error("Preview target file not found.");
|
|
1537
|
+
const mime = inferFileMime(resolved.relativePath);
|
|
1538
|
+
if (!mime) throw new Error("Preview target mime is unknown.");
|
|
1539
|
+
const previewKind = inferFilePreviewKind(resolved.relativePath, mime);
|
|
1540
|
+
if (!isSessionPreviewKind(previewKind)) throw new Error("Preview route is not supported for this file type.");
|
|
1541
|
+
const directoryPath = resolve(resolved.absolutePath, "..");
|
|
1542
|
+
const hash = toHash(`${directoryPath}:${mime}`);
|
|
1543
|
+
const entryFileName = previewKind === "html" ? null : PREVIEW_ENTRY_FILE_BY_KIND[previewKind];
|
|
1544
|
+
const fileName = basename(resolved.absolutePath);
|
|
1545
|
+
this.sessions.set(hash, {
|
|
1546
|
+
hash,
|
|
1547
|
+
directoryPath,
|
|
1548
|
+
mime,
|
|
1549
|
+
previewKind,
|
|
1550
|
+
entryFileName
|
|
1551
|
+
});
|
|
1552
|
+
const htmlPathname = `/api/file-preview/${hash}/${fileName}`;
|
|
1553
|
+
const resourcePathname = previewKind === "html" ? null : `/api/file-preview/${hash}/resource/${fileName}`;
|
|
1554
|
+
const entryPathname = previewKind === "html" ? htmlPathname : `/api/file-preview/${hash}/${entryFileName}`;
|
|
1555
|
+
return {
|
|
1556
|
+
hash,
|
|
1557
|
+
mime,
|
|
1558
|
+
previewKind,
|
|
1559
|
+
relativePath: resolved.relativePath,
|
|
1560
|
+
resourcePathname,
|
|
1561
|
+
entryPathname,
|
|
1562
|
+
urlPath: previewKind === "html" ? htmlPathname : `${entryPathname}?file=${encodeURIComponent(fileName)}`
|
|
1563
|
+
};
|
|
1564
|
+
}
|
|
1565
|
+
readPreviewRequest(hash, requestPath) {
|
|
1566
|
+
const session = this.sessions.get(hash);
|
|
1567
|
+
if (!session) return null;
|
|
1568
|
+
const normalized = stripLeadingSlash(requestPath);
|
|
1569
|
+
if (session.previewKind === "html") {
|
|
1570
|
+
const absolutePath$1 = resolve(session.directoryPath, normalized);
|
|
1571
|
+
if (!absolutePath$1.startsWith(session.directoryPath + "/")) return null;
|
|
1572
|
+
if (!existsSync(absolutePath$1) || !statSync(absolutePath$1).isFile()) return null;
|
|
1573
|
+
return {
|
|
1574
|
+
content: readFileSync(absolutePath$1),
|
|
1575
|
+
contentType: inferFileMime(absolutePath$1) ?? "application/octet-stream"
|
|
1576
|
+
};
|
|
1577
|
+
}
|
|
1578
|
+
if (normalized.startsWith("resource/")) {
|
|
1579
|
+
const resourcePath = normalized.slice(9);
|
|
1580
|
+
const absolutePath$1 = resolve(session.directoryPath, resourcePath);
|
|
1581
|
+
if (!absolutePath$1.startsWith(session.directoryPath + "/")) return null;
|
|
1582
|
+
if (!existsSync(absolutePath$1) || !statSync(absolutePath$1).isFile()) return null;
|
|
1583
|
+
return {
|
|
1584
|
+
content: readFileSync(absolutePath$1),
|
|
1585
|
+
contentType: inferFileMime(absolutePath$1) ?? "application/octet-stream"
|
|
1586
|
+
};
|
|
1587
|
+
}
|
|
1588
|
+
const assetName = normalized || session.entryFileName;
|
|
1589
|
+
if (!assetName) return null;
|
|
1590
|
+
const absolutePath = resolve(this.previewAssetsDir, assetName);
|
|
1591
|
+
if (!absolutePath.startsWith(resolve(this.previewAssetsDir) + "/")) return null;
|
|
1592
|
+
if (!existsSync(absolutePath) || !statSync(absolutePath).isFile()) return null;
|
|
1593
|
+
if (isRewritablePreviewAsset(assetName)) {
|
|
1594
|
+
const rewritten = rewritePreviewAssetPaths(readFileSync(absolutePath, "utf8"), hash);
|
|
1595
|
+
return {
|
|
1596
|
+
content: Buffer.from(rewritten, "utf8"),
|
|
1597
|
+
contentType: inferPreviewAssetContentType(absolutePath)
|
|
1598
|
+
};
|
|
1599
|
+
}
|
|
1600
|
+
return {
|
|
1601
|
+
content: readFileSync(absolutePath),
|
|
1602
|
+
contentType: inferPreviewAssetContentType(absolutePath)
|
|
1603
|
+
};
|
|
1604
|
+
}
|
|
1605
|
+
};
|
|
1606
|
+
|
|
1453
1607
|
//#endregion
|
|
1454
1608
|
//#region src/huggingface-endpoint.ts
|
|
1455
1609
|
const DEFAULT_HUGGING_FACE_ENDPOINT = "https://huggingface.co";
|
|
@@ -1461,9 +1615,6 @@ function normalizeHuggingFaceEndpoint(endpoint) {
|
|
|
1461
1615
|
function buildHuggingFaceApiBaseUrl(endpoint) {
|
|
1462
1616
|
return `${normalizeHuggingFaceEndpoint(endpoint)}/api`;
|
|
1463
1617
|
}
|
|
1464
|
-
function buildTransformersRemoteHost(endpoint) {
|
|
1465
|
-
return `${normalizeHuggingFaceEndpoint(endpoint)}/`;
|
|
1466
|
-
}
|
|
1467
1618
|
|
|
1468
1619
|
//#endregion
|
|
1469
1620
|
//#region src/local-model-asset-store.ts
|
|
@@ -1532,9 +1683,21 @@ function getDefaultLocalModelCacheDir() {
|
|
|
1532
1683
|
function getDefaultLocalModelIndexPath() {
|
|
1533
1684
|
return join(getDefaultLocalModelCacheRoot(), "models.json");
|
|
1534
1685
|
}
|
|
1686
|
+
function getDefaultLocalModelProfileManifestPath() {
|
|
1687
|
+
return join(getDefaultLocalModelCacheRoot(), "profile-manifests.json");
|
|
1688
|
+
}
|
|
1535
1689
|
function getDefaultLocalModelFetchCachePath() {
|
|
1536
1690
|
return join(getDefaultLocalModelCacheRoot(), "fetch-cache.json");
|
|
1537
1691
|
}
|
|
1692
|
+
function getLocalModelProfileRoot(cacheDir, modelId) {
|
|
1693
|
+
return join(cacheDir, "profiles", sanitizeLocalModelPathSegment(modelId));
|
|
1694
|
+
}
|
|
1695
|
+
function getLocalModelProfileGroupRoot(cacheDir, modelId, groupId) {
|
|
1696
|
+
return join(getLocalModelProfileRoot(cacheDir, modelId), sanitizeLocalModelPathSegment(groupId));
|
|
1697
|
+
}
|
|
1698
|
+
function sanitizeLocalModelPathSegment(value) {
|
|
1699
|
+
return value.replace(/[^a-zA-Z0-9._-]+/g, "-");
|
|
1700
|
+
}
|
|
1538
1701
|
|
|
1539
1702
|
//#endregion
|
|
1540
1703
|
//#region src/local-model-fetch-cache-store.ts
|
|
@@ -1687,60 +1850,80 @@ function getTransformersLocalModelPath(cacheDir, modelId) {
|
|
|
1687
1850
|
function getTransformersFileCacheModelPath(cacheDir, modelId) {
|
|
1688
1851
|
return join(cacheDir, modelId);
|
|
1689
1852
|
}
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1853
|
+
|
|
1854
|
+
//#endregion
|
|
1855
|
+
//#region src/local-model-profile-manifest-store.ts
|
|
1856
|
+
const LocalModelProfileManifestIndexSchema = z.object({
|
|
1857
|
+
version: z.literal(1).default(1),
|
|
1858
|
+
manifests: z.array(LocalModelProfileManifestSchema).default([])
|
|
1859
|
+
});
|
|
1860
|
+
var LocalModelProfileManifestStore = class {
|
|
1861
|
+
constructor(options) {
|
|
1862
|
+
this.options = options;
|
|
1863
|
+
}
|
|
1864
|
+
getManifestPath() {
|
|
1865
|
+
return this.options.manifestPath;
|
|
1866
|
+
}
|
|
1867
|
+
async readAll() {
|
|
1868
|
+
return (await this.readFile()).manifests;
|
|
1869
|
+
}
|
|
1870
|
+
async readMap() {
|
|
1871
|
+
return new Map((await this.readAll()).map((manifest) => [manifest.modelId, manifest]));
|
|
1872
|
+
}
|
|
1873
|
+
async read(modelId) {
|
|
1874
|
+
return (await this.readMap()).get(modelId) ?? null;
|
|
1875
|
+
}
|
|
1876
|
+
async writeAll(manifests) {
|
|
1877
|
+
const normalized = LocalModelProfileManifestIndexSchema.parse({
|
|
1878
|
+
version: 1,
|
|
1879
|
+
manifests: [...manifests].sort((left, right) => left.modelId.localeCompare(right.modelId))
|
|
1880
|
+
});
|
|
1881
|
+
const serialized = JSON.stringify(normalized, null, 2);
|
|
1882
|
+
await mkdir(dirname(this.options.manifestPath), { recursive: true });
|
|
1883
|
+
const tempPath = `${this.options.manifestPath}.${process.pid}.${Date.now()}.tmp`;
|
|
1884
|
+
await writeFile(tempPath, `${serialized}\n`, "utf8");
|
|
1885
|
+
await rename(tempPath, this.options.manifestPath);
|
|
1886
|
+
clearCache();
|
|
1887
|
+
}
|
|
1888
|
+
async upsert(manifest) {
|
|
1889
|
+
const manifests = await this.readMap();
|
|
1890
|
+
manifests.set(manifest.modelId, LocalModelProfileManifestSchema.parse(manifest));
|
|
1891
|
+
await this.writeAll([...manifests.values()]);
|
|
1892
|
+
}
|
|
1893
|
+
async remove(modelId) {
|
|
1894
|
+
const manifests = await this.readMap();
|
|
1895
|
+
if (!manifests.delete(modelId)) return;
|
|
1896
|
+
await this.writeAll([...manifests.values()]);
|
|
1897
|
+
}
|
|
1898
|
+
async readFile() {
|
|
1899
|
+
try {
|
|
1900
|
+
const content = await readFile(this.options.manifestPath, "utf8");
|
|
1901
|
+
const parsed = JSON.parse(content);
|
|
1902
|
+
const result = LocalModelProfileManifestIndexSchema.safeParse(parsed);
|
|
1903
|
+
return result.success ? result.data : LocalModelProfileManifestIndexSchema.parse({});
|
|
1904
|
+
} catch {
|
|
1905
|
+
return LocalModelProfileManifestIndexSchema.parse({});
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
};
|
|
1703
1909
|
|
|
1704
1910
|
//#endregion
|
|
1705
1911
|
//#region src/local-model-runtime.ts
|
|
1706
|
-
async function
|
|
1707
|
-
|
|
1708
|
-
transformers.env.cacheDir = cacheDir;
|
|
1709
|
-
transformers.env.allowLocalModels = false;
|
|
1710
|
-
transformers.env.localModelPath = join(cacheDir, "models");
|
|
1711
|
-
}
|
|
1712
|
-
async function resolveLocalModelRuntimePlan(input) {
|
|
1713
|
-
await configureTransformersRuntime(input.transformers, input.cacheDir);
|
|
1714
|
-
const hubUrl = normalizeHuggingFaceEndpoint(input.hfEndpoint);
|
|
1715
|
-
const repositoryFiles = await readHuggingFaceRepositoryFiles({
|
|
1716
|
-
selectedGroupId: input.selectedGroupId,
|
|
1717
|
-
modelId: input.modelId,
|
|
1718
|
-
hubUrl,
|
|
1719
|
-
fetchCacheStore: input.fetchCacheStore
|
|
1720
|
-
});
|
|
1721
|
-
return buildLocalDownloadPlanFromRepositoryFiles({
|
|
1912
|
+
async function readLocalModelRepositorySnapshot(input) {
|
|
1913
|
+
return readHuggingFaceRepositorySnapshot({
|
|
1722
1914
|
modelId: input.modelId,
|
|
1723
|
-
|
|
1724
|
-
|
|
1915
|
+
hubUrl: normalizeHuggingFaceEndpoint(input.hfEndpoint),
|
|
1916
|
+
fetchCacheStore: input.fetchCacheStore,
|
|
1917
|
+
revision: input.revision
|
|
1725
1918
|
});
|
|
1726
1919
|
}
|
|
1727
|
-
async function
|
|
1728
|
-
const
|
|
1729
|
-
const settings = await input.globalSettingsManager.readSettings();
|
|
1730
|
-
return resolveLocalModelRuntimePlan({
|
|
1731
|
-
modelId: input.modelId,
|
|
1732
|
-
transformers,
|
|
1733
|
-
cacheDir: input.cacheDir ?? getDefaultLocalModelCacheDir(),
|
|
1734
|
-
selectedGroupId: input.selectedGroupId,
|
|
1735
|
-
hfEndpoint: settings.translationEngines.local?.hfEndpoint,
|
|
1736
|
-
fetchCacheStore: input.fetchCacheStore
|
|
1737
|
-
});
|
|
1738
|
-
}
|
|
1739
|
-
async function readHuggingFaceRepositoryFiles(input) {
|
|
1920
|
+
async function readHuggingFaceRepositorySnapshot(input) {
|
|
1921
|
+
const detail = await readHuggingFaceModelSnapshotInfo(input).catch(() => null);
|
|
1740
1922
|
let lastError;
|
|
1741
1923
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
1742
1924
|
try {
|
|
1743
1925
|
const files = [];
|
|
1926
|
+
let commitHash = detail?.commitHash;
|
|
1744
1927
|
for await (const entry of listFiles({
|
|
1745
1928
|
repo: {
|
|
1746
1929
|
type: "model",
|
|
@@ -1748,16 +1931,30 @@ async function readHuggingFaceRepositoryFiles(input) {
|
|
|
1748
1931
|
},
|
|
1749
1932
|
recursive: true,
|
|
1750
1933
|
expand: true,
|
|
1934
|
+
revision: input.revision,
|
|
1751
1935
|
hubUrl: input.hubUrl,
|
|
1752
1936
|
fetch: input.fetchCacheStore ? createProviderFetchCache(input.fetchCacheStore) : void 0
|
|
1753
1937
|
})) {
|
|
1754
1938
|
if (entry.type !== "file") continue;
|
|
1939
|
+
commitHash ??= entry.lastCommit?.id;
|
|
1940
|
+
const etag = entry.lfs?.oid ?? entry.xetHash ?? entry.oid;
|
|
1755
1941
|
files.push({
|
|
1756
1942
|
path: entry.path,
|
|
1757
|
-
sizeBytes: entry.lfs?.size ?? entry.size
|
|
1943
|
+
sizeBytes: entry.lfs?.size ?? entry.size,
|
|
1944
|
+
etag,
|
|
1945
|
+
revision: entry.lastCommit?.id,
|
|
1946
|
+
sourceUrl: `${input.hubUrl}/${input.modelId}/resolve/${entry.lastCommit?.id ?? input.revision ?? "main"}/${entry.path}`,
|
|
1947
|
+
raw: entry
|
|
1758
1948
|
});
|
|
1759
1949
|
}
|
|
1760
|
-
if (files.length > 0) return
|
|
1950
|
+
if (files.length > 0 && commitHash) return {
|
|
1951
|
+
modelId: input.modelId,
|
|
1952
|
+
revision: input.revision ?? "main",
|
|
1953
|
+
commitHash,
|
|
1954
|
+
shortCommitHash: commitHash.slice(0, 6),
|
|
1955
|
+
files,
|
|
1956
|
+
raw: detail?.raw
|
|
1957
|
+
};
|
|
1761
1958
|
lastError = /* @__PURE__ */ new Error(`No repository files were returned for ${input.modelId}.`);
|
|
1762
1959
|
} catch (error) {
|
|
1763
1960
|
lastError = error;
|
|
@@ -1765,17 +1962,40 @@ async function readHuggingFaceRepositoryFiles(input) {
|
|
|
1765
1962
|
if (attempt < 2) await delay$2(300 * (attempt + 1));
|
|
1766
1963
|
}
|
|
1767
1964
|
const cachedFiles = await readCachedHuggingFaceRepositoryFiles(input);
|
|
1768
|
-
if (cachedFiles.length > 0) return
|
|
1965
|
+
if (cachedFiles.files.length > 0 && cachedFiles.commitHash) return {
|
|
1966
|
+
modelId: input.modelId,
|
|
1967
|
+
revision: input.revision ?? "main",
|
|
1968
|
+
commitHash: cachedFiles.commitHash,
|
|
1969
|
+
shortCommitHash: cachedFiles.commitHash.slice(0, 6),
|
|
1970
|
+
files: cachedFiles.files,
|
|
1971
|
+
raw: cachedFiles.raw
|
|
1972
|
+
};
|
|
1769
1973
|
throw lastError instanceof Error ? lastError : /* @__PURE__ */ new Error(`Unable to read repository files for ${input.modelId}.`);
|
|
1770
1974
|
}
|
|
1975
|
+
async function readHuggingFaceModelSnapshotInfo(input) {
|
|
1976
|
+
const detail = await modelInfo({
|
|
1977
|
+
name: input.modelId,
|
|
1978
|
+
hubUrl: input.hubUrl,
|
|
1979
|
+
revision: input.revision,
|
|
1980
|
+
additionalFields: ["sha"]
|
|
1981
|
+
});
|
|
1982
|
+
const raw = detail;
|
|
1983
|
+
const commitHash = typeof detail.sha === "string" ? detail.sha : void 0;
|
|
1984
|
+
if (!commitHash) throw new Error(`Unable to resolve a commit hash for ${input.modelId}.`);
|
|
1985
|
+
return {
|
|
1986
|
+
commitHash,
|
|
1987
|
+
raw
|
|
1988
|
+
};
|
|
1989
|
+
}
|
|
1771
1990
|
async function readCachedHuggingFaceRepositoryFiles(input) {
|
|
1772
|
-
if (!input.fetchCacheStore) return [];
|
|
1991
|
+
if (!input.fetchCacheStore) return { files: [] };
|
|
1773
1992
|
const record = await input.fetchCacheStore.read(input.modelId);
|
|
1774
|
-
if (!record?.detailRaw) return [];
|
|
1993
|
+
if (!record?.detailRaw) return { files: [] };
|
|
1775
1994
|
return extractRepositoryFilesFromCachedDetail(record.detailRaw);
|
|
1776
1995
|
}
|
|
1777
1996
|
function extractRepositoryFilesFromCachedDetail(raw) {
|
|
1778
1997
|
const siblings = Array.isArray(raw.siblings) ? raw.siblings : [];
|
|
1998
|
+
const commitHash = typeof raw.sha === "string" ? raw.sha : void 0;
|
|
1779
1999
|
const files = [];
|
|
1780
2000
|
for (const sibling of siblings) {
|
|
1781
2001
|
if (!sibling || typeof sibling !== "object") continue;
|
|
@@ -1784,10 +2004,16 @@ function extractRepositoryFilesFromCachedDetail(raw) {
|
|
|
1784
2004
|
if (!path) continue;
|
|
1785
2005
|
files.push({
|
|
1786
2006
|
path,
|
|
1787
|
-
sizeBytes: typeof record.size === "number" && Number.isFinite(record.size) ? record.size : void 0
|
|
2007
|
+
sizeBytes: typeof record.size === "number" && Number.isFinite(record.size) ? record.size : void 0,
|
|
2008
|
+
revision: commitHash,
|
|
2009
|
+
raw: record
|
|
1788
2010
|
});
|
|
1789
2011
|
}
|
|
1790
|
-
return
|
|
2012
|
+
return {
|
|
2013
|
+
files,
|
|
2014
|
+
commitHash,
|
|
2015
|
+
raw
|
|
2016
|
+
};
|
|
1791
2017
|
}
|
|
1792
2018
|
function createProviderFetchCache(fetchCacheStore) {
|
|
1793
2019
|
return async (input, init) => {
|
|
@@ -1815,9 +2041,6 @@ function headersToRecord$1(headers) {
|
|
|
1815
2041
|
function delay$2(ms) {
|
|
1816
2042
|
return new Promise((resolve$1) => setTimeout(resolve$1, ms));
|
|
1817
2043
|
}
|
|
1818
|
-
async function loadLocalTransformersModule(_projectDir, _globalSettingsManager) {
|
|
1819
|
-
return await import("@huggingface/transformers");
|
|
1820
|
-
}
|
|
1821
2044
|
|
|
1822
2045
|
//#endregion
|
|
1823
2046
|
//#region src/network-dispatcher.ts
|
|
@@ -2267,6 +2490,7 @@ const DEFAULT_NETWORK_RETRY_DELAY_MAX_MS = 5e3;
|
|
|
2267
2490
|
var LocalModelAssetService = class {
|
|
2268
2491
|
now;
|
|
2269
2492
|
store;
|
|
2493
|
+
profileManifestStore;
|
|
2270
2494
|
cacheDir;
|
|
2271
2495
|
fetchCacheStore;
|
|
2272
2496
|
networkRetryPolicy;
|
|
@@ -2286,6 +2510,7 @@ var LocalModelAssetService = class {
|
|
|
2286
2510
|
maxDelayMs: options.networkRetryPolicy?.maxDelayMs ?? DEFAULT_NETWORK_RETRY_DELAY_MAX_MS
|
|
2287
2511
|
};
|
|
2288
2512
|
this.store = new LocalModelAssetStore({ indexPath: options.indexPath ?? getDefaultLocalModelIndexPath() });
|
|
2513
|
+
this.profileManifestStore = new LocalModelProfileManifestStore({ manifestPath: options.profileManifestPath ?? getDefaultLocalModelProfileManifestPath() });
|
|
2289
2514
|
this.fetchCacheStore = new LocalModelFetchCacheStore({
|
|
2290
2515
|
cachePath: options.fetchCachePath ?? getDefaultLocalModelFetchCachePath(),
|
|
2291
2516
|
now: this.now
|
|
@@ -2387,106 +2612,129 @@ var LocalModelAssetService = class {
|
|
|
2387
2612
|
async readSelectedModelState(modelId, selectedGroupId) {
|
|
2388
2613
|
const state = (await this.store.readMap()).get(modelId);
|
|
2389
2614
|
if (state) return this.refreshCachedState(state, selectedGroupId);
|
|
2390
|
-
const
|
|
2391
|
-
|
|
2392
|
-
const selected = modelId === await this.readSelectedModel();
|
|
2393
|
-
const plan = await this.readPlanForState(modelId, selectedGroupId ?? session.selectedGroupId);
|
|
2394
|
-
const selectedGroup = selectLocalDownloadGroup(plan, selectedGroupId ?? session.selectedGroupId);
|
|
2395
|
-
const files = (selectedGroup?.files ?? plan?.files ?? []).map((file) => ({
|
|
2396
|
-
path: file.path,
|
|
2397
|
-
sizeBytes: file.sizeBytes,
|
|
2398
|
-
downloadedBytes: 0
|
|
2399
|
-
}));
|
|
2400
|
-
return LocalModelAssetStateSchema.parse({
|
|
2401
|
-
modelId,
|
|
2402
|
-
plan: plan ?? void 0,
|
|
2403
|
-
status: "downloading",
|
|
2404
|
-
selected,
|
|
2405
|
-
resumable: true,
|
|
2406
|
-
totalBytes: selectedGroup?.estimatedTotalBytes ?? plan?.estimatedTotalBytes,
|
|
2407
|
-
progress: 0,
|
|
2408
|
-
files,
|
|
2409
|
-
updatedAt: this.now()
|
|
2410
|
-
});
|
|
2411
|
-
}
|
|
2412
|
-
return LocalModelAssetStateSchema.parse({
|
|
2615
|
+
const selected = modelId === await this.readSelectedModel();
|
|
2616
|
+
const baseState = LocalModelAssetStateSchema.parse({
|
|
2413
2617
|
modelId,
|
|
2414
2618
|
status: "not-downloaded",
|
|
2415
|
-
selected
|
|
2619
|
+
selected,
|
|
2620
|
+
selectedGroupId,
|
|
2416
2621
|
updatedAt: this.now()
|
|
2417
2622
|
});
|
|
2623
|
+
return this.refreshCachedState(baseState, selectedGroupId);
|
|
2418
2624
|
}
|
|
2419
|
-
async startDownload(modelId,
|
|
2420
|
-
return this.runDownload(modelId, "downloading", "Downloading local model",
|
|
2625
|
+
async startDownload(modelId, groupId) {
|
|
2626
|
+
return this.runDownload(modelId, "downloading", "Downloading local model", groupId);
|
|
2421
2627
|
}
|
|
2422
|
-
async resumeDownload(modelId,
|
|
2423
|
-
return this.runDownload(modelId, "downloading", "Resuming local model download",
|
|
2628
|
+
async resumeDownload(modelId, groupId) {
|
|
2629
|
+
return this.runDownload(modelId, "downloading", "Resuming local model download", groupId);
|
|
2424
2630
|
}
|
|
2425
|
-
async pauseDownload(modelId) {
|
|
2426
|
-
const
|
|
2631
|
+
async pauseDownload(modelId, groupId) {
|
|
2632
|
+
const requestedGroupId = groupId ?? await this.readSelectedGroupId();
|
|
2633
|
+
if (!requestedGroupId) return { success: true };
|
|
2634
|
+
const current = await this.readSelectedModelState(modelId, requestedGroupId);
|
|
2635
|
+
const effectiveGroupId = current.plan?.selectedGroupId ?? current.selectedGroupId ?? requestedGroupId;
|
|
2636
|
+
const sessionKey = buildSessionKey(modelId, effectiveGroupId);
|
|
2637
|
+
const session = this.sessions.get(sessionKey);
|
|
2427
2638
|
if (session) {
|
|
2428
2639
|
session.abortController.abort();
|
|
2429
|
-
this.sessions.delete(
|
|
2640
|
+
this.sessions.delete(sessionKey);
|
|
2430
2641
|
}
|
|
2431
|
-
const
|
|
2642
|
+
const nextGroupsState = {
|
|
2643
|
+
...current.groupsState,
|
|
2644
|
+
[effectiveGroupId]: LocalModelLifecycleGroupStateSchema.parse({
|
|
2645
|
+
...current.groupsState[effectiveGroupId],
|
|
2646
|
+
groupId: effectiveGroupId,
|
|
2647
|
+
status: "paused",
|
|
2648
|
+
resumable: true,
|
|
2649
|
+
updatedAt: this.now()
|
|
2650
|
+
})
|
|
2651
|
+
};
|
|
2432
2652
|
const nextState = LocalModelAssetStateSchema.parse({
|
|
2433
2653
|
...current,
|
|
2434
|
-
|
|
2435
|
-
resumable: true,
|
|
2654
|
+
groupsState: nextGroupsState,
|
|
2436
2655
|
updatedAt: this.now()
|
|
2437
2656
|
});
|
|
2438
|
-
await this.
|
|
2657
|
+
const projected = await this.refreshCachedState(nextState, effectiveGroupId, { revalidateDisk: true });
|
|
2658
|
+
await this.store.upsert(projected);
|
|
2439
2659
|
this.emitLog({
|
|
2440
2660
|
engineId: "local",
|
|
2441
2661
|
modelId,
|
|
2442
|
-
selectedGroupId:
|
|
2662
|
+
selectedGroupId: effectiveGroupId,
|
|
2663
|
+
groupId: effectiveGroupId,
|
|
2443
2664
|
status: "paused",
|
|
2444
2665
|
message: "Local model download paused.",
|
|
2445
|
-
progress:
|
|
2446
|
-
bytesDownloaded:
|
|
2447
|
-
totalBytes:
|
|
2666
|
+
progress: projected.progress,
|
|
2667
|
+
bytesDownloaded: projected.bytesDownloaded,
|
|
2668
|
+
totalBytes: projected.totalBytes,
|
|
2448
2669
|
resumable: true,
|
|
2449
|
-
files:
|
|
2670
|
+
files: projected.files,
|
|
2450
2671
|
updatedAt: this.now()
|
|
2451
2672
|
});
|
|
2452
2673
|
return { success: true };
|
|
2453
2674
|
}
|
|
2454
|
-
async deleteModel(modelId) {
|
|
2455
|
-
this.
|
|
2456
|
-
|
|
2457
|
-
|
|
2675
|
+
async deleteModel(modelId, groupId) {
|
|
2676
|
+
const requestedGroupId = groupId ?? await this.readSelectedGroupId();
|
|
2677
|
+
if (!requestedGroupId) {
|
|
2678
|
+
await this.store.remove(modelId);
|
|
2679
|
+
await this.profileManifestStore.remove(modelId);
|
|
2680
|
+
return { success: true };
|
|
2681
|
+
}
|
|
2682
|
+
const current = await this.readSelectedModelState(modelId, requestedGroupId);
|
|
2683
|
+
const effectiveGroupId = current.plan?.selectedGroupId ?? current.selectedGroupId ?? requestedGroupId;
|
|
2684
|
+
const sessionKey = buildSessionKey(modelId, effectiveGroupId);
|
|
2685
|
+
this.sessions.get(sessionKey)?.abortController.abort();
|
|
2686
|
+
this.sessions.delete(sessionKey);
|
|
2458
2687
|
await this.store.upsert(LocalModelAssetStateSchema.parse({
|
|
2459
2688
|
...current,
|
|
2460
|
-
|
|
2689
|
+
groupsState: {
|
|
2690
|
+
...current.groupsState,
|
|
2691
|
+
[effectiveGroupId]: LocalModelLifecycleGroupStateSchema.parse({
|
|
2692
|
+
...current.groupsState[effectiveGroupId],
|
|
2693
|
+
groupId: effectiveGroupId,
|
|
2694
|
+
status: "deleting",
|
|
2695
|
+
updatedAt: this.now()
|
|
2696
|
+
})
|
|
2697
|
+
},
|
|
2461
2698
|
updatedAt: this.now()
|
|
2462
2699
|
}));
|
|
2463
2700
|
this.emitLog({
|
|
2464
2701
|
engineId: "local",
|
|
2465
2702
|
modelId,
|
|
2466
|
-
selectedGroupId:
|
|
2703
|
+
selectedGroupId: effectiveGroupId,
|
|
2704
|
+
groupId: effectiveGroupId,
|
|
2467
2705
|
status: "deleting",
|
|
2468
2706
|
message: "Deleting local model files.",
|
|
2469
2707
|
files: current.files,
|
|
2470
2708
|
updatedAt: this.now()
|
|
2471
2709
|
});
|
|
2472
|
-
await
|
|
2473
|
-
await rm(getTransformersLocalModelPath(this.cacheDir, modelId), {
|
|
2710
|
+
await rm(getLocalModelProfileGroupRoot(this.cacheDir, modelId, effectiveGroupId), {
|
|
2474
2711
|
recursive: true,
|
|
2475
2712
|
force: true
|
|
2476
2713
|
});
|
|
2477
|
-
await
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2714
|
+
const persistedManifest = await this.profileManifestStore.read(modelId);
|
|
2715
|
+
const nextGroupsState = { ...current.groupsState };
|
|
2716
|
+
delete nextGroupsState[effectiveGroupId];
|
|
2717
|
+
const isPersistedManifestGroup = Boolean(persistedManifest?.groups[effectiveGroupId]);
|
|
2718
|
+
const nextManifest = isPersistedManifestGroup ? persistedManifest : current.profileManifest ? removeManifestGroup(current.profileManifest, effectiveGroupId) : void 0;
|
|
2719
|
+
const nextPlan = isPersistedManifestGroup ? void 0 : current.plan ? removePlanGroup(current.plan, effectiveGroupId) : void 0;
|
|
2720
|
+
const nextSelectedGroupId = current.selectedGroupId === effectiveGroupId ? void 0 : current.selectedGroupId;
|
|
2721
|
+
const nextState = await this.refreshCachedState(LocalModelAssetStateSchema.parse({
|
|
2722
|
+
...current,
|
|
2723
|
+
selectedGroupId: nextSelectedGroupId,
|
|
2724
|
+
profileManifest: nextManifest,
|
|
2725
|
+
groupsState: nextGroupsState,
|
|
2726
|
+
plan: nextPlan,
|
|
2727
|
+
updatedAt: this.now()
|
|
2728
|
+
}), nextSelectedGroupId, { revalidateDisk: true });
|
|
2729
|
+
if (nextState.profileManifest) await this.profileManifestStore.upsert(nextState.profileManifest);
|
|
2730
|
+
else await this.profileManifestStore.remove(modelId);
|
|
2731
|
+
if (nextState.profileManifest || nextState.plan?.groups?.length) await this.store.upsert(nextState);
|
|
2732
|
+
else await this.store.remove(modelId);
|
|
2486
2733
|
this.emitLog({
|
|
2487
2734
|
engineId: "local",
|
|
2488
2735
|
modelId,
|
|
2489
|
-
selectedGroupId:
|
|
2736
|
+
selectedGroupId: effectiveGroupId,
|
|
2737
|
+
groupId: effectiveGroupId,
|
|
2490
2738
|
status: "not-downloaded",
|
|
2491
2739
|
message: "Local model files were removed.",
|
|
2492
2740
|
progress: 0,
|
|
@@ -2497,6 +2745,49 @@ var LocalModelAssetService = class {
|
|
|
2497
2745
|
});
|
|
2498
2746
|
return { success: true };
|
|
2499
2747
|
}
|
|
2748
|
+
async refreshProfiles(modelId) {
|
|
2749
|
+
const targetModelId = modelId ?? await this.readSelectedModel();
|
|
2750
|
+
const loadingState = LocalModelAssetStateSchema.parse({
|
|
2751
|
+
...await this.readSelectedModelState(targetModelId),
|
|
2752
|
+
profileLoad: {
|
|
2753
|
+
status: "loading",
|
|
2754
|
+
message: "Loading local model profiles.",
|
|
2755
|
+
updatedAt: this.now()
|
|
2756
|
+
},
|
|
2757
|
+
updatedAt: this.now()
|
|
2758
|
+
});
|
|
2759
|
+
await this.store.upsert(loadingState);
|
|
2760
|
+
try {
|
|
2761
|
+
const manifest = await this.createProfileManifest(targetModelId);
|
|
2762
|
+
await this.profileManifestStore.upsert(manifest);
|
|
2763
|
+
const current = await this.readSelectedModelState(targetModelId);
|
|
2764
|
+
const nextState = await this.refreshCachedState(LocalModelAssetStateSchema.parse({
|
|
2765
|
+
...current,
|
|
2766
|
+
profileManifest: manifest,
|
|
2767
|
+
profileLoad: {
|
|
2768
|
+
status: "ready",
|
|
2769
|
+
message: "Local model profiles are ready.",
|
|
2770
|
+
updatedAt: this.now()
|
|
2771
|
+
},
|
|
2772
|
+
updatedAt: this.now()
|
|
2773
|
+
}), void 0, { revalidateDisk: true });
|
|
2774
|
+
await this.store.upsert(nextState);
|
|
2775
|
+
return nextState;
|
|
2776
|
+
} catch (error) {
|
|
2777
|
+
const message = error instanceof Error ? error.message : "Unable to load local model profiles.";
|
|
2778
|
+
const failedState = LocalModelAssetStateSchema.parse({
|
|
2779
|
+
...await this.readSelectedModelState(targetModelId),
|
|
2780
|
+
profileLoad: {
|
|
2781
|
+
status: "error",
|
|
2782
|
+
error: message,
|
|
2783
|
+
updatedAt: this.now()
|
|
2784
|
+
},
|
|
2785
|
+
updatedAt: this.now()
|
|
2786
|
+
});
|
|
2787
|
+
await this.store.upsert(failedState);
|
|
2788
|
+
throw error;
|
|
2789
|
+
}
|
|
2790
|
+
}
|
|
2500
2791
|
async markSelectedModel(modelId) {
|
|
2501
2792
|
const nextStates = (await this.store.readAll()).map((state) => LocalModelAssetStateSchema.parse({
|
|
2502
2793
|
...state,
|
|
@@ -2511,7 +2802,7 @@ var LocalModelAssetService = class {
|
|
|
2511
2802
|
await this.store.writeAll(nextStates);
|
|
2512
2803
|
}
|
|
2513
2804
|
async waitForModelTask(modelId) {
|
|
2514
|
-
await this.sessionTasks.
|
|
2805
|
+
await Promise.all([...this.sessionTasks.entries()].filter(([sessionKey]) => sessionKey.startsWith(`${modelId}:`)).map(([, task]) => task));
|
|
2515
2806
|
}
|
|
2516
2807
|
async close() {
|
|
2517
2808
|
const sessions = [...this.sessions.values()];
|
|
@@ -2570,176 +2861,298 @@ var LocalModelAssetService = class {
|
|
|
2570
2861
|
}));
|
|
2571
2862
|
return [...remoteItems, ...localOnlyItems];
|
|
2572
2863
|
}
|
|
2573
|
-
async refreshCachedState(state, selectedGroupId) {
|
|
2574
|
-
const
|
|
2575
|
-
const
|
|
2576
|
-
const
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
sizeBytes: file.sizeBytes,
|
|
2586
|
-
downloadedBytes: file.sizeBytes
|
|
2587
|
-
}));
|
|
2588
|
-
return LocalModelAssetStateSchema.parse({
|
|
2589
|
-
...state,
|
|
2590
|
-
selected,
|
|
2591
|
-
status: "downloaded",
|
|
2592
|
-
progress: 1,
|
|
2593
|
-
bytesDownloaded: state.totalBytes ?? state.plan.estimatedTotalBytes,
|
|
2594
|
-
totalBytes: state.totalBytes ?? state.plan.estimatedTotalBytes,
|
|
2595
|
-
resumable: false,
|
|
2596
|
-
error: void 0,
|
|
2597
|
-
files: files$1,
|
|
2598
|
-
updatedAt: this.now(),
|
|
2599
|
-
installedAt: state.installedAt ?? this.now()
|
|
2600
|
-
});
|
|
2601
|
-
}
|
|
2602
|
-
const transformers = await this.getTransformersModule();
|
|
2603
|
-
transformers.env.remoteHost = buildTransformersRemoteHost(await this.readHuggingFaceEndpoint());
|
|
2604
|
-
const [plan, persistedSelectedGroupId] = await Promise.all([this.readPlan(state.modelId, transformers, requestedGroupId ?? state.plan?.selectedGroupId), this.readSelectedGroupId()]);
|
|
2605
|
-
if (!plan && state.status !== "downloaded") return LocalModelAssetStateSchema.parse({
|
|
2606
|
-
...state,
|
|
2607
|
-
selected,
|
|
2608
|
-
plan: void 0
|
|
2864
|
+
async refreshCachedState(state, selectedGroupId, options = {}) {
|
|
2865
|
+
const [selectedModel, persistedSelectedGroupId] = await Promise.all([this.readSelectedModel(), this.readSelectedGroupId()]);
|
|
2866
|
+
const selected = state.selected || state.modelId === selectedModel;
|
|
2867
|
+
const selectedGroupIdFromSettings = selectedGroupId ?? persistedSelectedGroupId;
|
|
2868
|
+
const manifest = filterConcreteProfileManifest(state.profileManifest ?? await this.profileManifestStore.read(state.modelId) ?? void 0);
|
|
2869
|
+
const migrated = migrateLegacyStateToGroups(state, manifest, this.now());
|
|
2870
|
+
const manifestWithHistoricalGroups = mergeHistoricalGroupsIntoManifest({
|
|
2871
|
+
cacheDir: this.cacheDir,
|
|
2872
|
+
modelId: state.modelId,
|
|
2873
|
+
manifest,
|
|
2874
|
+
groupsState: migrated.groupsState,
|
|
2875
|
+
fallbackPlan: migrated.plan
|
|
2609
2876
|
});
|
|
2610
|
-
const
|
|
2877
|
+
const selectedGroupIdForProjection = resolveManifestGroupId(manifestWithHistoricalGroups, selectedGroupIdFromSettings ?? migrated.selectedGroupId ?? migrated.plan?.selectedGroupId) ?? selectFirstManifestGroupId(manifestWithHistoricalGroups);
|
|
2878
|
+
const reconciledGroupsState = options.revalidateDisk ? await this.reconcileGroupsFromDisk({
|
|
2611
2879
|
modelId: state.modelId,
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
selectedGroupId: state.plan?.selectedGroupId,
|
|
2619
|
-
groups: state.plan?.groups
|
|
2620
|
-
};
|
|
2621
|
-
const selectedGroup = selectLocalDownloadGroup(effectivePlan, requestedGroupId ?? state.plan?.selectedGroupId ?? persistedSelectedGroupId);
|
|
2622
|
-
if (requestedGroupId && requestedGroupId !== effectivePlan.selectedGroupId && !selectedGroup) return LocalModelAssetStateSchema.parse({
|
|
2623
|
-
...state,
|
|
2624
|
-
selected,
|
|
2625
|
-
status: "not-downloaded",
|
|
2626
|
-
progress: 0,
|
|
2627
|
-
bytesDownloaded: 0,
|
|
2628
|
-
totalBytes: void 0,
|
|
2629
|
-
resumable: false,
|
|
2630
|
-
files: [],
|
|
2631
|
-
updatedAt: this.now()
|
|
2880
|
+
manifest: manifestWithHistoricalGroups,
|
|
2881
|
+
groupsState: migrated.groupsState
|
|
2882
|
+
}) : this.reconcileGroupsFromSnapshot({
|
|
2883
|
+
modelId: state.modelId,
|
|
2884
|
+
manifest: manifestWithHistoricalGroups,
|
|
2885
|
+
groupsState: migrated.groupsState
|
|
2632
2886
|
});
|
|
2633
|
-
const
|
|
2634
|
-
const cacheStatus = await readLocalModelFileStatus({
|
|
2635
|
-
cacheDir: this.cacheDir,
|
|
2887
|
+
const plan = buildPlanFromManifest({
|
|
2636
2888
|
modelId: state.modelId,
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
const
|
|
2642
|
-
const
|
|
2643
|
-
|
|
2644
|
-
const
|
|
2645
|
-
const downloadedBytes = file.sizeBytes === void 0 ? cached ? existingDownloadedBytes : existingDownloadedBytes : cached ? file.sizeBytes : Math.min(existingDownloadedBytes, file.sizeBytes);
|
|
2889
|
+
manifest: manifestWithHistoricalGroups,
|
|
2890
|
+
groupsState: reconciledGroupsState,
|
|
2891
|
+
selectedGroupId: selectedGroupIdForProjection
|
|
2892
|
+
});
|
|
2893
|
+
const selectedPlanGroup = selectLocalDownloadGroup(plan, selectedGroupIdForProjection);
|
|
2894
|
+
const selectedGroupState = selectedPlanGroup && reconciledGroupsState[selectedPlanGroup.id] ? reconciledGroupsState[selectedPlanGroup.id] : void 0;
|
|
2895
|
+
const files = selectedPlanGroup?.files.map((file) => {
|
|
2896
|
+
const stateFile = selectedGroupState?.files.find((entry) => entry.path === file.path);
|
|
2646
2897
|
return {
|
|
2647
2898
|
path: file.path,
|
|
2648
2899
|
sizeBytes: file.sizeBytes,
|
|
2649
|
-
downloadedBytes
|
|
2900
|
+
downloadedBytes: stateFile?.downloadedBytes ?? 0
|
|
2650
2901
|
};
|
|
2651
|
-
});
|
|
2652
|
-
const
|
|
2653
|
-
const detectedProgress = effectivePlan.estimatedTotalBytes !== void 0 && effectivePlan.estimatedTotalBytes > 0 ? detectedBytesDownloaded / effectivePlan.estimatedTotalBytes : runtimeAllCached ? 1 : void 0;
|
|
2654
|
-
const progress = cacheStatus.allCached ? 1 : session ? state.progress : detectedProgress;
|
|
2655
|
-
const hasPartialCache = !runtimeAllCached && detectedBytesDownloaded > 0;
|
|
2902
|
+
}) ?? [];
|
|
2903
|
+
const status = selectedGroupState?.status ?? "not-downloaded";
|
|
2656
2904
|
return LocalModelAssetStateSchema.parse({
|
|
2657
|
-
...
|
|
2905
|
+
...migrated,
|
|
2658
2906
|
selected,
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2907
|
+
selectedGroupId: selectedGroupIdForProjection,
|
|
2908
|
+
profileManifest: manifestWithHistoricalGroups,
|
|
2909
|
+
groupsState: reconciledGroupsState,
|
|
2910
|
+
plan: plan ?? void 0,
|
|
2911
|
+
status,
|
|
2912
|
+
progress: selectedGroupState?.progress,
|
|
2913
|
+
totalBytes: selectedGroupState?.totalBytes ?? selectedPlanGroup?.estimatedTotalBytes,
|
|
2914
|
+
bytesDownloaded: selectedGroupState?.bytesDownloaded,
|
|
2915
|
+
error: selectedGroupState?.error,
|
|
2916
|
+
resumable: selectedGroupState?.resumable ?? false,
|
|
2666
2917
|
files,
|
|
2667
2918
|
updatedAt: this.now(),
|
|
2668
|
-
installedAt:
|
|
2919
|
+
installedAt: selectedGroupState?.installedAt ?? state.installedAt
|
|
2669
2920
|
});
|
|
2670
2921
|
}
|
|
2671
|
-
|
|
2672
|
-
|
|
2922
|
+
reconcileGroupsFromSnapshot(input) {
|
|
2923
|
+
if (!input.manifest) return input.groupsState;
|
|
2924
|
+
const nextGroupsState = { ...input.groupsState };
|
|
2925
|
+
for (const groupId of input.manifest.groupOrder) {
|
|
2926
|
+
const manifestGroup = input.manifest.groups[groupId];
|
|
2927
|
+
if (!manifestGroup) continue;
|
|
2928
|
+
const current = nextGroupsState[groupId];
|
|
2929
|
+
const files = reconcileGroupFilesFromSnapshot({
|
|
2930
|
+
manifestGroup,
|
|
2931
|
+
currentFiles: current?.files ?? [],
|
|
2932
|
+
currentStatus: current?.status ?? "not-downloaded"
|
|
2933
|
+
});
|
|
2934
|
+
const bytesDownloaded = sumDownloadedBytes(files);
|
|
2935
|
+
const totalBytes = manifestGroup.estimatedTotalBytes;
|
|
2936
|
+
const status = current?.status ?? "not-downloaded";
|
|
2937
|
+
nextGroupsState[groupId] = LocalModelLifecycleGroupStateSchema.parse({
|
|
2938
|
+
...current,
|
|
2939
|
+
groupId,
|
|
2940
|
+
baseGroupId: manifestGroup.baseGroupId,
|
|
2941
|
+
status,
|
|
2942
|
+
rootDir: manifestGroup.rootDir,
|
|
2943
|
+
bytesDownloaded,
|
|
2944
|
+
totalBytes,
|
|
2945
|
+
progress: totalBytes && totalBytes > 0 ? Math.max(0, Math.min(1, bytesDownloaded / totalBytes)) : current?.progress,
|
|
2946
|
+
resumable: current?.resumable ?? (status === "paused" || status === "error" || status === "downloading"),
|
|
2947
|
+
error: current?.error,
|
|
2948
|
+
installedAt: current?.installedAt,
|
|
2949
|
+
updatedAt: current?.updatedAt ?? this.now(),
|
|
2950
|
+
files
|
|
2951
|
+
});
|
|
2952
|
+
}
|
|
2953
|
+
return nextGroupsState;
|
|
2954
|
+
}
|
|
2955
|
+
async reconcileGroupsFromDisk(input) {
|
|
2956
|
+
if (!input.manifest) return input.groupsState;
|
|
2957
|
+
const nextGroupsState = { ...input.groupsState };
|
|
2958
|
+
for (const groupId of input.manifest.groupOrder) {
|
|
2959
|
+
const manifestGroup = input.manifest.groups[groupId];
|
|
2960
|
+
if (!manifestGroup) continue;
|
|
2961
|
+
const current = nextGroupsState[groupId];
|
|
2962
|
+
if (isActiveDownloadStatus(current?.status ?? "not-downloaded")) {
|
|
2963
|
+
nextGroupsState[groupId] = LocalModelLifecycleGroupStateSchema.parse({
|
|
2964
|
+
...current,
|
|
2965
|
+
groupId,
|
|
2966
|
+
baseGroupId: manifestGroup.baseGroupId,
|
|
2967
|
+
rootDir: manifestGroup.rootDir,
|
|
2968
|
+
totalBytes: manifestGroup.estimatedTotalBytes,
|
|
2969
|
+
files: reconcileGroupFiles({
|
|
2970
|
+
manifestGroup,
|
|
2971
|
+
currentFiles: current?.files ?? []
|
|
2972
|
+
})
|
|
2973
|
+
});
|
|
2974
|
+
continue;
|
|
2975
|
+
}
|
|
2976
|
+
const files = await reconcileGroupFilesFromDisk({
|
|
2977
|
+
rootDir: manifestGroup.rootDir,
|
|
2978
|
+
manifestGroup,
|
|
2979
|
+
currentFiles: current?.files ?? []
|
|
2980
|
+
});
|
|
2981
|
+
const bytesDownloaded = sumDownloadedBytes(files);
|
|
2982
|
+
const totalBytes = manifestGroup.estimatedTotalBytes;
|
|
2983
|
+
const allComplete = files.length > 0 && files.every((file) => file.sizeBytes !== void 0 && (file.downloadedBytes ?? 0) >= file.sizeBytes && file.status === "downloaded");
|
|
2984
|
+
const hasPartial = files.some((file) => (file.downloadedBytes ?? 0) > 0);
|
|
2985
|
+
const status = allComplete ? "downloaded" : current?.status === "error" ? "error" : current?.status === "paused" ? "paused" : hasPartial ? "paused" : "not-downloaded";
|
|
2986
|
+
nextGroupsState[groupId] = LocalModelLifecycleGroupStateSchema.parse({
|
|
2987
|
+
...current,
|
|
2988
|
+
groupId,
|
|
2989
|
+
baseGroupId: manifestGroup.baseGroupId,
|
|
2990
|
+
status,
|
|
2991
|
+
rootDir: manifestGroup.rootDir,
|
|
2992
|
+
bytesDownloaded,
|
|
2993
|
+
totalBytes,
|
|
2994
|
+
progress: totalBytes && totalBytes > 0 ? Math.max(0, Math.min(1, bytesDownloaded / totalBytes)) : void 0,
|
|
2995
|
+
resumable: status === "paused" || status === "error",
|
|
2996
|
+
error: status === "error" ? current?.error : void 0,
|
|
2997
|
+
installedAt: status === "downloaded" ? current?.installedAt ?? this.now() : current?.installedAt,
|
|
2998
|
+
updatedAt: this.now(),
|
|
2999
|
+
files
|
|
3000
|
+
});
|
|
3001
|
+
}
|
|
3002
|
+
return nextGroupsState;
|
|
3003
|
+
}
|
|
3004
|
+
async runDownload(modelId, targetStatus, messagePrefix, groupId) {
|
|
3005
|
+
const effectiveGroupId = groupId ?? await this.readSelectedGroupId();
|
|
3006
|
+
if (!effectiveGroupId) throw new Error("No local model profile is selected.");
|
|
3007
|
+
const manifest = await this.ensureProfileManifest(modelId);
|
|
3008
|
+
const resolvedGroupId = resolveManifestGroupId(manifest, effectiveGroupId);
|
|
3009
|
+
if (!resolvedGroupId) throw new Error("No concrete local model download plan is available.");
|
|
3010
|
+
const sessionKey = buildSessionKey(modelId, resolvedGroupId);
|
|
3011
|
+
const existing = this.sessions.get(sessionKey);
|
|
2673
3012
|
if (existing) return { sessionId: existing.sessionId };
|
|
2674
|
-
const sessionId = `local-model-${sanitizeId(modelId)}-${this.now()}`;
|
|
3013
|
+
const sessionId = `local-model-${sanitizeId(modelId)}-${sanitizeId(resolvedGroupId)}-${this.now()}`;
|
|
2675
3014
|
const abortController = new AbortController();
|
|
2676
|
-
this.sessions.set(
|
|
3015
|
+
this.sessions.set(sessionKey, {
|
|
2677
3016
|
modelId,
|
|
2678
3017
|
sessionId,
|
|
2679
3018
|
abortController,
|
|
2680
|
-
|
|
3019
|
+
groupId: resolvedGroupId
|
|
2681
3020
|
});
|
|
2682
|
-
const current = await this.readSelectedModelState(modelId);
|
|
2683
|
-
const
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
this.sessions.delete(modelId);
|
|
3021
|
+
const current = await this.readSelectedModelState(modelId, resolvedGroupId);
|
|
3022
|
+
const manifestGroup = manifest.groups[resolvedGroupId];
|
|
3023
|
+
if (!manifestGroup || manifestGroup.files.length === 0 || manifestGroup.estimatedTotalBytes === void 0) {
|
|
3024
|
+
this.sessions.delete(sessionKey);
|
|
2687
3025
|
throw new Error("No concrete local model download plan is available.");
|
|
2688
3026
|
}
|
|
2689
|
-
const totalBytes =
|
|
2690
|
-
const
|
|
2691
|
-
|
|
2692
|
-
|
|
3027
|
+
const totalBytes = manifestGroup.estimatedTotalBytes;
|
|
3028
|
+
const currentGroup = current.groupsState[resolvedGroupId];
|
|
3029
|
+
const resumedFiles = await reconcileGroupFilesFromDisk({
|
|
3030
|
+
rootDir: manifestGroup.rootDir,
|
|
3031
|
+
manifestGroup,
|
|
3032
|
+
currentFiles: currentGroup?.files ?? []
|
|
2693
3033
|
});
|
|
2694
3034
|
const resumedBytesDownloaded = sumDownloadedBytes(resumedFiles);
|
|
2695
3035
|
const nextState = LocalModelAssetStateSchema.parse({
|
|
2696
3036
|
...current,
|
|
2697
3037
|
modelId,
|
|
2698
|
-
plan,
|
|
2699
|
-
status: targetStatus,
|
|
2700
3038
|
selected: true,
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
3039
|
+
profileManifest: manifest,
|
|
3040
|
+
groupsState: {
|
|
3041
|
+
...current.groupsState,
|
|
3042
|
+
[resolvedGroupId]: LocalModelLifecycleGroupStateSchema.parse({
|
|
3043
|
+
...currentGroup,
|
|
3044
|
+
groupId: resolvedGroupId,
|
|
3045
|
+
baseGroupId: manifestGroup.baseGroupId,
|
|
3046
|
+
status: targetStatus,
|
|
3047
|
+
rootDir: manifestGroup.rootDir,
|
|
3048
|
+
bytesDownloaded: resumedBytesDownloaded,
|
|
3049
|
+
progress: totalBytes > 0 ? resumedBytesDownloaded / totalBytes : currentGroup?.progress,
|
|
3050
|
+
totalBytes,
|
|
3051
|
+
resumable: true,
|
|
3052
|
+
files: resumedFiles,
|
|
3053
|
+
updatedAt: this.now()
|
|
3054
|
+
})
|
|
3055
|
+
},
|
|
2706
3056
|
updatedAt: this.now()
|
|
2707
3057
|
});
|
|
2708
|
-
await this.
|
|
3058
|
+
const projected = await this.refreshCachedState(nextState, resolvedGroupId, { revalidateDisk: true });
|
|
3059
|
+
await this.store.upsert(projected);
|
|
2709
3060
|
this.emitLog({
|
|
2710
3061
|
engineId: "local",
|
|
2711
3062
|
modelId,
|
|
2712
|
-
selectedGroupId:
|
|
3063
|
+
selectedGroupId: resolvedGroupId,
|
|
3064
|
+
groupId: resolvedGroupId,
|
|
2713
3065
|
status: targetStatus,
|
|
2714
3066
|
message: `${messagePrefix} ${modelId}.`,
|
|
2715
|
-
progress:
|
|
2716
|
-
bytesDownloaded:
|
|
3067
|
+
progress: projected.progress,
|
|
3068
|
+
bytesDownloaded: projected.bytesDownloaded,
|
|
2717
3069
|
totalBytes,
|
|
2718
3070
|
sessionId,
|
|
2719
3071
|
resumable: true,
|
|
2720
|
-
files:
|
|
3072
|
+
files: projected.files,
|
|
2721
3073
|
updatedAt: this.now()
|
|
2722
3074
|
});
|
|
2723
|
-
const task = this.performDownload(modelId, sessionId, abortController.signal
|
|
2724
|
-
if (this.sessionTasks.get(
|
|
3075
|
+
const task = this.performDownload(modelId, resolvedGroupId, sessionId, abortController.signal).catch((error) => this.finishDownload(modelId, resolvedGroupId, sessionId, false, error instanceof Error ? error.message : String(error))).finally(() => {
|
|
3076
|
+
if (this.sessionTasks.get(sessionKey) === task) this.sessionTasks.delete(sessionKey);
|
|
2725
3077
|
});
|
|
2726
|
-
this.sessionTasks.set(
|
|
3078
|
+
this.sessionTasks.set(sessionKey, task);
|
|
2727
3079
|
return { sessionId };
|
|
2728
3080
|
}
|
|
2729
|
-
async
|
|
2730
|
-
const
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
3081
|
+
async ensureProfileManifest(modelId) {
|
|
3082
|
+
const existing = await this.profileManifestStore.read(modelId);
|
|
3083
|
+
if (existing) return existing;
|
|
3084
|
+
const manifest = await this.createProfileManifest(modelId);
|
|
3085
|
+
await this.profileManifestStore.upsert(manifest);
|
|
3086
|
+
return manifest;
|
|
3087
|
+
}
|
|
3088
|
+
async createProfileManifest(modelId) {
|
|
3089
|
+
const hfEndpoint = await this.readHuggingFaceEndpoint();
|
|
3090
|
+
const snapshot = await readLocalModelRepositorySnapshot({
|
|
3091
|
+
modelId,
|
|
3092
|
+
hfEndpoint,
|
|
3093
|
+
fetchCacheStore: this.fetchCacheStore
|
|
3094
|
+
});
|
|
3095
|
+
const basePlan = buildLocalDownloadPlanFromRepositoryFiles({
|
|
3096
|
+
modelId,
|
|
3097
|
+
files: snapshot.files.map((file) => ({
|
|
3098
|
+
...file,
|
|
3099
|
+
revision: snapshot.commitHash
|
|
3100
|
+
}))
|
|
3101
|
+
});
|
|
3102
|
+
if (!basePlan?.groups?.length) throw new Error(`No recognizable local model profiles were found for ${modelId}.`);
|
|
3103
|
+
const groupsEntries = basePlan.groups.flatMap((group) => {
|
|
3104
|
+
if (!group.selectable || group.estimatedTotalBytes === void 0) return [];
|
|
3105
|
+
const groupId = buildVersionedGroupId(group.id, snapshot.shortCommitHash);
|
|
3106
|
+
const rootDir = getLocalModelProfileGroupRoot(this.cacheDir, modelId, groupId);
|
|
3107
|
+
return [[groupId, {
|
|
3108
|
+
id: groupId,
|
|
3109
|
+
baseGroupId: group.id,
|
|
3110
|
+
label: group.label,
|
|
3111
|
+
displayLabel: group.label,
|
|
3112
|
+
description: group.description,
|
|
3113
|
+
profile: group.profile,
|
|
3114
|
+
dtype: group.dtype,
|
|
3115
|
+
commitHash: snapshot.commitHash,
|
|
3116
|
+
shortCommitHash: snapshot.shortCommitHash,
|
|
3117
|
+
rootDir,
|
|
3118
|
+
estimatedTotalBytes: group.estimatedTotalBytes,
|
|
3119
|
+
selectable: group.selectable,
|
|
3120
|
+
files: group.files.map((file) => ({
|
|
3121
|
+
...file,
|
|
3122
|
+
revision: snapshot.commitHash,
|
|
3123
|
+
sourceUrl: file.sourceUrl ?? `${normalizeHuggingFaceEndpoint(hfEndpoint)}/${modelId}/resolve/${snapshot.commitHash}/${file.path}`
|
|
3124
|
+
}))
|
|
3125
|
+
}]];
|
|
3126
|
+
});
|
|
3127
|
+
if (groupsEntries.length === 0) throw new Error(`No selectable local model profiles were found for ${modelId}.`);
|
|
3128
|
+
return LocalModelProfileManifestSchema.parse({
|
|
3129
|
+
modelId,
|
|
3130
|
+
source: "huggingface",
|
|
3131
|
+
endpoint: normalizeHuggingFaceEndpoint(hfEndpoint),
|
|
3132
|
+
revision: snapshot.revision,
|
|
3133
|
+
commitHash: snapshot.commitHash,
|
|
3134
|
+
shortCommitHash: snapshot.shortCommitHash,
|
|
3135
|
+
fetchedAt: this.now(),
|
|
3136
|
+
updatedAt: this.now(),
|
|
3137
|
+
raw: snapshot.raw,
|
|
3138
|
+
groups: Object.fromEntries(groupsEntries),
|
|
3139
|
+
groupOrder: groupsEntries.map(([groupId]) => groupId)
|
|
3140
|
+
});
|
|
3141
|
+
}
|
|
3142
|
+
async performDownload(modelId, groupId, sessionId, signal) {
|
|
3143
|
+
const manifestGroup = (await this.ensureProfileManifest(modelId)).groups[groupId];
|
|
3144
|
+
if (!manifestGroup) throw new Error(`Unknown local model profile: ${groupId}.`);
|
|
3145
|
+
const files = manifestGroup.files;
|
|
3146
|
+
const totalBytes = manifestGroup.estimatedTotalBytes;
|
|
2736
3147
|
const hfEndpoint = normalizeHuggingFaceEndpoint(await this.readHuggingFaceEndpoint());
|
|
2737
|
-
const
|
|
2738
|
-
|
|
2739
|
-
|
|
3148
|
+
const currentGroup = (await this.readSelectedModelState(modelId, groupId)).groupsState[groupId];
|
|
3149
|
+
const downloadedFiles = await reconcileGroupFilesFromDisk({
|
|
3150
|
+
rootDir: manifestGroup.rootDir,
|
|
3151
|
+
manifestGroup,
|
|
3152
|
+
currentFiles: currentGroup?.files ?? []
|
|
2740
3153
|
});
|
|
2741
3154
|
let bytesDownloaded = sumDownloadedBytes(downloadedFiles);
|
|
2742
|
-
if (files.length === 0) throw new Error("No concrete local model download files were selected.");
|
|
3155
|
+
if (files.length === 0 || totalBytes === void 0) throw new Error("No concrete local model download files were selected.");
|
|
2743
3156
|
for (const [fileIndex, file] of files.entries()) {
|
|
2744
3157
|
throwIfAborted(signal);
|
|
2745
3158
|
const previousFileBytes = downloadedFiles[fileIndex]?.downloadedBytes ?? 0;
|
|
@@ -2747,12 +3160,14 @@ var LocalModelAssetService = class {
|
|
|
2747
3160
|
downloadedFiles[fileIndex] = {
|
|
2748
3161
|
path: file.path,
|
|
2749
3162
|
sizeBytes: file.sizeBytes,
|
|
2750
|
-
downloadedBytes: previousFileBytes
|
|
3163
|
+
downloadedBytes: previousFileBytes,
|
|
3164
|
+
required: file.required,
|
|
3165
|
+
status: previousFileBytes > 0 ? "paused" : "not-downloaded"
|
|
2751
3166
|
};
|
|
2752
3167
|
await this.emitDownloadProgress({
|
|
2753
3168
|
modelId,
|
|
3169
|
+
groupId,
|
|
2754
3170
|
sessionId,
|
|
2755
|
-
state,
|
|
2756
3171
|
message: `Downloading ${file.path}.`,
|
|
2757
3172
|
totalBytes,
|
|
2758
3173
|
bytesDownloaded,
|
|
@@ -2765,7 +3180,10 @@ var LocalModelAssetService = class {
|
|
|
2765
3180
|
},
|
|
2766
3181
|
path: file.path,
|
|
2767
3182
|
cacheDir: this.cacheDir,
|
|
3183
|
+
targetPath: join(manifestGroup.rootDir, file.path),
|
|
2768
3184
|
hubUrl: hfEndpoint,
|
|
3185
|
+
revision: manifestGroup.commitHash,
|
|
3186
|
+
etag: file.etag,
|
|
2769
3187
|
expectedSizeBytes: file.sizeBytes,
|
|
2770
3188
|
retryPolicy: this.networkRetryPolicy,
|
|
2771
3189
|
fetch: createAbortableFetch(signal),
|
|
@@ -2776,12 +3194,14 @@ var LocalModelAssetService = class {
|
|
|
2776
3194
|
downloadedFiles[fileIndex] = {
|
|
2777
3195
|
path: file.path,
|
|
2778
3196
|
sizeBytes: file.sizeBytes,
|
|
2779
|
-
downloadedBytes: boundedFileBytes
|
|
3197
|
+
downloadedBytes: boundedFileBytes,
|
|
3198
|
+
required: file.required,
|
|
3199
|
+
status: boundedFileBytes >= (file.sizeBytes ?? Number.POSITIVE_INFINITY) ? "downloaded" : "downloading"
|
|
2780
3200
|
};
|
|
2781
3201
|
await this.emitDownloadProgress({
|
|
2782
3202
|
modelId,
|
|
3203
|
+
groupId,
|
|
2783
3204
|
sessionId,
|
|
2784
|
-
state,
|
|
2785
3205
|
message: `Downloading ${file.path}.`,
|
|
2786
3206
|
totalBytes,
|
|
2787
3207
|
bytesDownloaded: bytesDownloaded - previousFileBytes + boundedFileBytes,
|
|
@@ -2792,8 +3212,8 @@ var LocalModelAssetService = class {
|
|
|
2792
3212
|
const retryTarget = phase === "metadata" ? `metadata for ${file.path}` : `${file.path}`;
|
|
2793
3213
|
await this.emitDownloadProgress({
|
|
2794
3214
|
modelId,
|
|
3215
|
+
groupId,
|
|
2795
3216
|
sessionId,
|
|
2796
|
-
state,
|
|
2797
3217
|
message: `Connection interrupted while downloading ${retryTarget}. Retrying automatically in ${formatDuration(retryDelayMs)}.`,
|
|
2798
3218
|
totalBytes,
|
|
2799
3219
|
bytesDownloaded: bytesDownloaded - previousFileBytes + (downloadedFiles[fileIndex]?.downloadedBytes ?? 0),
|
|
@@ -2804,6 +3224,7 @@ var LocalModelAssetService = class {
|
|
|
2804
3224
|
await mirrorHubCacheFileForTransformers({
|
|
2805
3225
|
cacheDir: this.cacheDir,
|
|
2806
3226
|
modelId,
|
|
3227
|
+
profileRoot: manifestGroup.rootDir,
|
|
2807
3228
|
filePath: file.path,
|
|
2808
3229
|
cachedPath
|
|
2809
3230
|
});
|
|
@@ -2813,95 +3234,123 @@ var LocalModelAssetService = class {
|
|
|
2813
3234
|
downloadedFiles[fileIndex] = {
|
|
2814
3235
|
path: file.path,
|
|
2815
3236
|
sizeBytes: file.sizeBytes,
|
|
2816
|
-
downloadedBytes: file.sizeBytes
|
|
3237
|
+
downloadedBytes: file.sizeBytes,
|
|
3238
|
+
required: file.required,
|
|
3239
|
+
status: "downloaded"
|
|
2817
3240
|
};
|
|
2818
3241
|
await this.emitDownloadProgress({
|
|
2819
3242
|
modelId,
|
|
3243
|
+
groupId,
|
|
2820
3244
|
sessionId,
|
|
2821
|
-
state,
|
|
2822
3245
|
message: `Downloaded ${file.path}.`,
|
|
2823
3246
|
totalBytes,
|
|
2824
3247
|
bytesDownloaded,
|
|
2825
3248
|
files: downloadedFiles
|
|
2826
3249
|
});
|
|
2827
3250
|
}
|
|
2828
|
-
await this.finishDownload(modelId, sessionId, true, `Local model ${modelId} is ready.`);
|
|
3251
|
+
await this.finishDownload(modelId, groupId, sessionId, true, `Local model ${modelId} is ready.`);
|
|
2829
3252
|
}
|
|
2830
3253
|
async emitDownloadProgress(input) {
|
|
2831
|
-
if (!this.isActiveSession(input.modelId, input.sessionId)) return;
|
|
3254
|
+
if (!this.isActiveSession(input.modelId, input.groupId, input.sessionId)) return;
|
|
2832
3255
|
const progress = input.totalBytes && input.totalBytes > 0 ? Math.max(0, Math.min(1, input.bytesDownloaded / input.totalBytes)) : void 0;
|
|
3256
|
+
const current = await this.readSelectedModelState(input.modelId, input.groupId);
|
|
3257
|
+
const currentGroup = current.groupsState[input.groupId];
|
|
2833
3258
|
const nextState = LocalModelAssetStateSchema.parse({
|
|
2834
|
-
...
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
3259
|
+
...current,
|
|
3260
|
+
groupsState: {
|
|
3261
|
+
...current.groupsState,
|
|
3262
|
+
[input.groupId]: LocalModelLifecycleGroupStateSchema.parse({
|
|
3263
|
+
...currentGroup,
|
|
3264
|
+
groupId: input.groupId,
|
|
3265
|
+
status: "downloading",
|
|
3266
|
+
bytesDownloaded: input.bytesDownloaded,
|
|
3267
|
+
totalBytes: input.totalBytes,
|
|
3268
|
+
progress,
|
|
3269
|
+
resumable: true,
|
|
3270
|
+
files: input.files,
|
|
3271
|
+
updatedAt: this.now()
|
|
3272
|
+
})
|
|
3273
|
+
},
|
|
3274
|
+
updatedAt: this.now()
|
|
2842
3275
|
});
|
|
2843
|
-
await this.
|
|
3276
|
+
const projected = await this.refreshCachedState(nextState, input.groupId, { revalidateDisk: true });
|
|
3277
|
+
await this.store.upsert(projected);
|
|
2844
3278
|
this.emitLog({
|
|
2845
3279
|
engineId: "local",
|
|
2846
3280
|
modelId: input.modelId,
|
|
2847
|
-
selectedGroupId: input.
|
|
3281
|
+
selectedGroupId: input.groupId,
|
|
3282
|
+
groupId: input.groupId,
|
|
2848
3283
|
status: "downloading",
|
|
2849
3284
|
message: input.message,
|
|
2850
3285
|
progress,
|
|
2851
3286
|
bytesDownloaded: input.bytesDownloaded,
|
|
2852
3287
|
totalBytes: input.totalBytes,
|
|
2853
|
-
files: input.files
|
|
3288
|
+
files: input.files.map((file) => ({
|
|
3289
|
+
path: file.path,
|
|
3290
|
+
sizeBytes: file.sizeBytes,
|
|
3291
|
+
downloadedBytes: file.downloadedBytes
|
|
3292
|
+
})),
|
|
2854
3293
|
sessionId: input.sessionId,
|
|
2855
3294
|
resumable: true,
|
|
2856
3295
|
updatedAt: this.now()
|
|
2857
3296
|
});
|
|
2858
3297
|
}
|
|
2859
|
-
async finishDownload(modelId, sessionId, success, message) {
|
|
2860
|
-
if (!this.isActiveSession(modelId, sessionId)) return;
|
|
2861
|
-
const
|
|
3298
|
+
async finishDownload(modelId, groupId, sessionId, success, message) {
|
|
3299
|
+
if (!this.isActiveSession(modelId, groupId, sessionId)) return;
|
|
3300
|
+
const sessionKey = buildSessionKey(modelId, groupId);
|
|
3301
|
+
const current = await this.readSelectedModelState(modelId, groupId);
|
|
3302
|
+
const currentGroup = current.groupsState[groupId];
|
|
3303
|
+
const totalBytes = currentGroup?.totalBytes ?? current.totalBytes;
|
|
3304
|
+
const files = success ? current.files.map((file) => LocalModelLifecycleFileStateSchema.parse({
|
|
3305
|
+
...file,
|
|
3306
|
+
required: true,
|
|
3307
|
+
downloadedBytes: file.sizeBytes,
|
|
3308
|
+
status: "downloaded",
|
|
3309
|
+
updatedAt: this.now()
|
|
3310
|
+
})) : (currentGroup?.files ?? []).map((file) => LocalModelLifecycleFileStateSchema.parse({
|
|
3311
|
+
...file,
|
|
3312
|
+
status: file.status === "downloaded" ? "downloaded" : "paused",
|
|
3313
|
+
updatedAt: this.now()
|
|
3314
|
+
}));
|
|
2862
3315
|
const nextState = LocalModelAssetStateSchema.parse({
|
|
2863
3316
|
...current,
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
3317
|
+
groupsState: {
|
|
3318
|
+
...current.groupsState,
|
|
3319
|
+
[groupId]: LocalModelLifecycleGroupStateSchema.parse({
|
|
3320
|
+
...currentGroup,
|
|
3321
|
+
groupId,
|
|
3322
|
+
status: success ? "downloaded" : "error",
|
|
3323
|
+
progress: success ? 1 : current.progress,
|
|
3324
|
+
bytesDownloaded: success ? totalBytes : current.bytesDownloaded,
|
|
3325
|
+
totalBytes,
|
|
3326
|
+
installedAt: success ? this.now() : currentGroup?.installedAt,
|
|
3327
|
+
updatedAt: this.now(),
|
|
3328
|
+
error: success ? void 0 : message,
|
|
3329
|
+
resumable: !success,
|
|
3330
|
+
files
|
|
3331
|
+
})
|
|
3332
|
+
},
|
|
3333
|
+
updatedAt: this.now()
|
|
2872
3334
|
});
|
|
2873
|
-
await this.
|
|
2874
|
-
this.
|
|
3335
|
+
const projected = await this.refreshCachedState(nextState, groupId, { revalidateDisk: true });
|
|
3336
|
+
await this.store.upsert(projected);
|
|
3337
|
+
this.sessions.delete(sessionKey);
|
|
2875
3338
|
this.emitLog({
|
|
2876
3339
|
engineId: "local",
|
|
2877
3340
|
modelId,
|
|
2878
|
-
selectedGroupId:
|
|
2879
|
-
|
|
3341
|
+
selectedGroupId: groupId,
|
|
3342
|
+
groupId,
|
|
3343
|
+
status: projected.status,
|
|
2880
3344
|
message,
|
|
2881
|
-
progress:
|
|
2882
|
-
bytesDownloaded:
|
|
2883
|
-
totalBytes:
|
|
3345
|
+
progress: projected.progress,
|
|
3346
|
+
bytesDownloaded: projected.bytesDownloaded,
|
|
3347
|
+
totalBytes: projected.totalBytes,
|
|
2884
3348
|
sessionId,
|
|
2885
|
-
resumable:
|
|
2886
|
-
files:
|
|
3349
|
+
resumable: projected.resumable,
|
|
3350
|
+
files: projected.files,
|
|
2887
3351
|
updatedAt: this.now()
|
|
2888
3352
|
});
|
|
2889
3353
|
}
|
|
2890
|
-
async readPlan(modelId, transformers, selectedGroupId) {
|
|
2891
|
-
return resolveLocalModelRuntimePlan({
|
|
2892
|
-
modelId,
|
|
2893
|
-
transformers,
|
|
2894
|
-
cacheDir: this.cacheDir,
|
|
2895
|
-
selectedGroupId: selectedGroupId ?? await this.readSelectedGroupId(),
|
|
2896
|
-
hfEndpoint: await this.readHuggingFaceEndpoint(),
|
|
2897
|
-
fetchCacheStore: this.fetchCacheStore
|
|
2898
|
-
}).catch(() => null);
|
|
2899
|
-
}
|
|
2900
|
-
async readPlanForState(modelId, selectedGroupId) {
|
|
2901
|
-
const transformers = await this.getTransformersModule();
|
|
2902
|
-
transformers.env.remoteHost = buildTransformersRemoteHost(await this.readHuggingFaceEndpoint());
|
|
2903
|
-
return this.readPlan(modelId, transformers, selectedGroupId);
|
|
2904
|
-
}
|
|
2905
3354
|
async readSelectedModel() {
|
|
2906
3355
|
return (await this.options.globalSettingsManager.readSettings()).translationEngines.local.model;
|
|
2907
3356
|
}
|
|
@@ -2911,8 +3360,8 @@ var LocalModelAssetService = class {
|
|
|
2911
3360
|
async readHuggingFaceEndpoint() {
|
|
2912
3361
|
return (await this.options.globalSettingsManager.readSettings()).translationEngines.local.hfEndpoint;
|
|
2913
3362
|
}
|
|
2914
|
-
isActiveSession(modelId, sessionId) {
|
|
2915
|
-
return this.sessions.get(modelId)?.sessionId === sessionId;
|
|
3363
|
+
isActiveSession(modelId, groupId, sessionId) {
|
|
3364
|
+
return this.sessions.get(buildSessionKey(modelId, groupId))?.sessionId === sessionId;
|
|
2916
3365
|
}
|
|
2917
3366
|
emitLog(log) {
|
|
2918
3367
|
this.logs.set(log.modelId, log);
|
|
@@ -2926,17 +3375,279 @@ var LocalModelAssetService = class {
|
|
|
2926
3375
|
return import("@huggingface/transformers");
|
|
2927
3376
|
}
|
|
2928
3377
|
};
|
|
2929
|
-
function
|
|
3378
|
+
function buildSessionKey(modelId, groupId) {
|
|
3379
|
+
return `${modelId}::${groupId}`;
|
|
3380
|
+
}
|
|
3381
|
+
function buildVersionedGroupId(baseGroupId, shortCommitHash) {
|
|
3382
|
+
return `${sanitizeId(baseGroupId)}-${sanitizeId(shortCommitHash)}`;
|
|
3383
|
+
}
|
|
3384
|
+
function selectFirstManifestGroupId(manifest) {
|
|
3385
|
+
return manifest?.groupOrder.find((groupId) => manifest.groups[groupId]?.selectable);
|
|
3386
|
+
}
|
|
3387
|
+
function resolveManifestGroupId(manifest, requestedGroupId) {
|
|
3388
|
+
if (!manifest || !requestedGroupId) return requestedGroupId;
|
|
3389
|
+
if (manifest.groups[requestedGroupId]?.selectable) return requestedGroupId;
|
|
3390
|
+
return manifest.groupOrder.find((groupId) => {
|
|
3391
|
+
const group = manifest.groups[groupId];
|
|
3392
|
+
return group?.selectable && group.baseGroupId === requestedGroupId;
|
|
3393
|
+
});
|
|
3394
|
+
}
|
|
3395
|
+
function removeManifestGroup(manifest, groupId) {
|
|
3396
|
+
const groups = { ...manifest.groups };
|
|
3397
|
+
delete groups[groupId];
|
|
3398
|
+
const groupOrder = manifest.groupOrder.filter((id) => id !== groupId);
|
|
3399
|
+
if (groupOrder.length === 0) return void 0;
|
|
3400
|
+
return LocalModelProfileManifestSchema.parse({
|
|
3401
|
+
...manifest,
|
|
3402
|
+
groups,
|
|
3403
|
+
groupOrder
|
|
3404
|
+
});
|
|
3405
|
+
}
|
|
3406
|
+
function removePlanGroup(plan, groupId) {
|
|
3407
|
+
const groups = plan.groups?.filter((group) => group.id !== groupId);
|
|
3408
|
+
if (!groups?.length) return void 0;
|
|
3409
|
+
const selectedGroup = groups.find((group) => group.selected) ?? groups[0];
|
|
3410
|
+
return {
|
|
3411
|
+
...plan,
|
|
3412
|
+
selectedGroupId: selectedGroup?.id,
|
|
3413
|
+
estimatedTotalBytes: selectedGroup?.estimatedTotalBytes,
|
|
3414
|
+
files: selectedGroup?.files ?? [],
|
|
3415
|
+
groups: groups.map((group) => ({
|
|
3416
|
+
...group,
|
|
3417
|
+
selected: group.id === selectedGroup?.id
|
|
3418
|
+
}))
|
|
3419
|
+
};
|
|
3420
|
+
}
|
|
3421
|
+
function migrateLegacyStateToGroups(state, manifest, now) {
|
|
3422
|
+
const selectedGroupId = state.selectedGroupId ?? state.plan?.selectedGroupId;
|
|
3423
|
+
const groupsState = { ...state.groupsState };
|
|
3424
|
+
for (const group of manifest ? [] : state.plan?.groups ?? []) {
|
|
3425
|
+
if (groupsState[group.id] || !group.status || group.status === "not-downloaded") continue;
|
|
3426
|
+
const groupStatus = group.status;
|
|
3427
|
+
const manifestGroup = manifest?.groups[group.id];
|
|
3428
|
+
const groupFiles = group.id === selectedGroupId && state.files.length > 0 ? state.files.map((file) => ({
|
|
3429
|
+
...file,
|
|
3430
|
+
required: true,
|
|
3431
|
+
status: file.sizeBytes !== void 0 && (file.downloadedBytes ?? 0) >= file.sizeBytes ? "downloaded" : normalizeLiveStatusForStoredState(groupStatus)
|
|
3432
|
+
})) : group.files.map((file) => ({
|
|
3433
|
+
...file,
|
|
3434
|
+
downloadedBytes: groupStatus === "downloaded" ? file.sizeBytes : 0,
|
|
3435
|
+
status: normalizeLiveStatusForStoredState(groupStatus)
|
|
3436
|
+
}));
|
|
3437
|
+
const bytesDownloaded = sumDownloadedBytes(groupFiles);
|
|
3438
|
+
const totalBytes = group.estimatedTotalBytes;
|
|
3439
|
+
groupsState[group.id] = LocalModelLifecycleGroupStateSchema.parse({
|
|
3440
|
+
groupId: group.id,
|
|
3441
|
+
baseGroupId: group.baseGroupId ?? manifestGroup?.baseGroupId ?? group.id,
|
|
3442
|
+
status: normalizeLiveStatusForStoredState(groupStatus),
|
|
3443
|
+
rootDir: group.rootDir ?? manifestGroup?.rootDir,
|
|
3444
|
+
bytesDownloaded,
|
|
3445
|
+
totalBytes,
|
|
3446
|
+
progress: totalBytes && totalBytes > 0 ? Math.max(0, Math.min(1, bytesDownloaded / totalBytes)) : group.progress,
|
|
3447
|
+
resumable: group.resumable ?? (groupStatus === "paused" || groupStatus === "downloading" || groupStatus === "queued" || groupStatus === "error"),
|
|
3448
|
+
error: group.error,
|
|
3449
|
+
installedAt: groupStatus === "downloaded" ? state.installedAt ?? now : void 0,
|
|
3450
|
+
updatedAt: state.updatedAt ?? now,
|
|
3451
|
+
files: groupFiles
|
|
3452
|
+
});
|
|
3453
|
+
}
|
|
3454
|
+
if (selectedGroupId && !groupsState[selectedGroupId] && state.files.length > 0 && state.status !== "not-downloaded") {
|
|
3455
|
+
const manifestGroup = manifest?.groups[selectedGroupId];
|
|
3456
|
+
groupsState[selectedGroupId] = LocalModelLifecycleGroupStateSchema.parse({
|
|
3457
|
+
groupId: selectedGroupId,
|
|
3458
|
+
baseGroupId: manifestGroup?.baseGroupId ?? selectedGroupId,
|
|
3459
|
+
status: normalizeLiveStatusForStoredState(state.status),
|
|
3460
|
+
rootDir: manifestGroup?.rootDir,
|
|
3461
|
+
bytesDownloaded: state.bytesDownloaded,
|
|
3462
|
+
totalBytes: state.totalBytes,
|
|
3463
|
+
progress: state.progress,
|
|
3464
|
+
resumable: state.resumable,
|
|
3465
|
+
error: state.error,
|
|
3466
|
+
installedAt: state.installedAt,
|
|
3467
|
+
updatedAt: state.updatedAt ?? now,
|
|
3468
|
+
files: state.files.map((file) => LocalModelLifecycleFileStateSchema.parse({
|
|
3469
|
+
...file,
|
|
3470
|
+
required: true,
|
|
3471
|
+
status: file.sizeBytes !== void 0 && (file.downloadedBytes ?? 0) >= file.sizeBytes ? "downloaded" : state.status === "downloaded" ? "downloaded" : normalizeLiveStatusForStoredState(state.status)
|
|
3472
|
+
}))
|
|
3473
|
+
});
|
|
3474
|
+
}
|
|
3475
|
+
return LocalModelAssetStateSchema.parse({
|
|
3476
|
+
...state,
|
|
3477
|
+
selectedGroupId,
|
|
3478
|
+
groupsState
|
|
3479
|
+
});
|
|
3480
|
+
}
|
|
3481
|
+
function mergeHistoricalGroupsIntoManifest(input) {
|
|
3482
|
+
const existing = input.manifest;
|
|
3483
|
+
const groups = existing ? { ...existing.groups } : {};
|
|
3484
|
+
const groupOrder = existing ? [...existing.groupOrder] : [];
|
|
3485
|
+
const fallbackGroups = input.fallbackPlan?.groups ?? [];
|
|
3486
|
+
for (const fallbackGroup of fallbackGroups) {
|
|
3487
|
+
if (groups[fallbackGroup.id]) continue;
|
|
3488
|
+
if (!isConcreteCommitHash(fallbackGroup.commitHash) || !isConcreteCommitHash(fallbackGroup.shortCommitHash)) continue;
|
|
3489
|
+
const state = input.groupsState[fallbackGroup.id];
|
|
3490
|
+
const commitHash = fallbackGroup.commitHash;
|
|
3491
|
+
const shortCommitHash = fallbackGroup.shortCommitHash;
|
|
3492
|
+
groups[fallbackGroup.id] = {
|
|
3493
|
+
id: fallbackGroup.id,
|
|
3494
|
+
baseGroupId: fallbackGroup.baseGroupId ?? fallbackGroup.id,
|
|
3495
|
+
label: fallbackGroup.label,
|
|
3496
|
+
displayLabel: `${fallbackGroup.label} · ${shortCommitHash}`,
|
|
3497
|
+
description: fallbackGroup.description,
|
|
3498
|
+
profile: fallbackGroup.profile,
|
|
3499
|
+
dtype: fallbackGroup.dtype,
|
|
3500
|
+
commitHash,
|
|
3501
|
+
shortCommitHash,
|
|
3502
|
+
rootDir: fallbackGroup.rootDir ?? state?.rootDir ?? getLocalModelProfileGroupRoot(input.cacheDir, input.modelId, fallbackGroup.id),
|
|
3503
|
+
estimatedTotalBytes: fallbackGroup.estimatedTotalBytes,
|
|
3504
|
+
selectable: fallbackGroup.selectable,
|
|
3505
|
+
files: fallbackGroup.files.map((file) => ({
|
|
3506
|
+
...file,
|
|
3507
|
+
revision: file.revision ?? commitHash
|
|
3508
|
+
}))
|
|
3509
|
+
};
|
|
3510
|
+
groupOrder.push(fallbackGroup.id);
|
|
3511
|
+
}
|
|
3512
|
+
if (!existing && groupOrder.length === 0) return void 0;
|
|
3513
|
+
return LocalModelProfileManifestSchema.parse({
|
|
3514
|
+
modelId: input.modelId,
|
|
3515
|
+
source: "huggingface",
|
|
3516
|
+
endpoint: existing?.endpoint ?? "",
|
|
3517
|
+
revision: existing?.revision ?? "legacy",
|
|
3518
|
+
commitHash: existing?.commitHash ?? "legacy",
|
|
3519
|
+
shortCommitHash: existing?.shortCommitHash ?? "legacy",
|
|
3520
|
+
fetchedAt: existing?.fetchedAt ?? 0,
|
|
3521
|
+
updatedAt: existing?.updatedAt ?? 0,
|
|
3522
|
+
raw: existing?.raw,
|
|
3523
|
+
groups,
|
|
3524
|
+
groupOrder
|
|
3525
|
+
});
|
|
3526
|
+
}
|
|
3527
|
+
function filterConcreteProfileManifest(manifest) {
|
|
3528
|
+
if (!manifest || !isConcreteCommitHash(manifest.commitHash)) return void 0;
|
|
3529
|
+
const groups = Object.fromEntries(manifest.groupOrder.flatMap((groupId) => {
|
|
3530
|
+
const group = manifest.groups[groupId];
|
|
3531
|
+
if (!group || !isConcreteCommitHash(group.commitHash)) return [];
|
|
3532
|
+
return [[groupId, group]];
|
|
3533
|
+
}));
|
|
3534
|
+
const groupOrder = manifest.groupOrder.filter((groupId) => groups[groupId]);
|
|
3535
|
+
if (groupOrder.length === 0) return void 0;
|
|
3536
|
+
return LocalModelProfileManifestSchema.parse({
|
|
3537
|
+
...manifest,
|
|
3538
|
+
groups,
|
|
3539
|
+
groupOrder
|
|
3540
|
+
});
|
|
3541
|
+
}
|
|
3542
|
+
function isConcreteCommitHash(value) {
|
|
3543
|
+
return Boolean(value && value !== "legacy");
|
|
3544
|
+
}
|
|
3545
|
+
function formatManifestGroupChipLabel(manifest, group) {
|
|
3546
|
+
if (group.commitHash === manifest.commitHash) return group.label;
|
|
3547
|
+
return `${group.label} · ${group.shortCommitHash}`;
|
|
3548
|
+
}
|
|
3549
|
+
function buildPlanFromManifest(input) {
|
|
3550
|
+
const manifest = input.manifest;
|
|
3551
|
+
if (!manifest) return null;
|
|
3552
|
+
const selectedGroupId = input.selectedGroupId && manifest.groups[input.selectedGroupId]?.selectable ? input.selectedGroupId : selectFirstManifestGroupId(manifest);
|
|
3553
|
+
const groups = manifest.groupOrder.flatMap((groupId) => {
|
|
3554
|
+
const manifestGroup = manifest.groups[groupId];
|
|
3555
|
+
if (!manifestGroup) return [];
|
|
3556
|
+
const groupState = input.groupsState[groupId];
|
|
3557
|
+
return [{
|
|
3558
|
+
id: manifestGroup.id,
|
|
3559
|
+
label: formatManifestGroupChipLabel(manifest, manifestGroup),
|
|
3560
|
+
description: manifestGroup.description,
|
|
3561
|
+
profile: manifestGroup.profile,
|
|
3562
|
+
dtype: manifestGroup.dtype,
|
|
3563
|
+
estimatedTotalBytes: manifestGroup.estimatedTotalBytes,
|
|
3564
|
+
baseGroupId: manifestGroup.baseGroupId,
|
|
3565
|
+
commitHash: manifestGroup.commitHash,
|
|
3566
|
+
shortCommitHash: manifestGroup.shortCommitHash,
|
|
3567
|
+
rootDir: manifestGroup.rootDir,
|
|
3568
|
+
status: groupState?.status ?? "not-downloaded",
|
|
3569
|
+
progress: groupState?.progress,
|
|
3570
|
+
bytesDownloaded: groupState?.bytesDownloaded,
|
|
3571
|
+
totalBytes: groupState?.totalBytes ?? manifestGroup.estimatedTotalBytes,
|
|
3572
|
+
resumable: groupState?.resumable,
|
|
3573
|
+
error: groupState?.error,
|
|
3574
|
+
selectable: manifestGroup.selectable,
|
|
3575
|
+
selected: manifestGroup.id === selectedGroupId,
|
|
3576
|
+
files: manifestGroup.files.map((file) => ({
|
|
3577
|
+
...file,
|
|
3578
|
+
required: file.required
|
|
3579
|
+
}))
|
|
3580
|
+
}];
|
|
3581
|
+
});
|
|
3582
|
+
const selectedGroup = groups.find((group) => group.id === selectedGroupId) ?? groups[0];
|
|
3583
|
+
if (!selectedGroup) return null;
|
|
3584
|
+
return {
|
|
3585
|
+
modelId: input.modelId,
|
|
3586
|
+
estimatedTotalBytes: selectedGroup.estimatedTotalBytes,
|
|
3587
|
+
files: selectedGroup.files,
|
|
3588
|
+
selectedGroupId: selectedGroup.id,
|
|
3589
|
+
groups
|
|
3590
|
+
};
|
|
3591
|
+
}
|
|
3592
|
+
function reconcileGroupFiles(input) {
|
|
2930
3593
|
const currentFileByPath = new Map(input.currentFiles.map((file) => [file.path, file]));
|
|
2931
|
-
return input.
|
|
2932
|
-
const
|
|
2933
|
-
|
|
3594
|
+
return input.manifestGroup.files.map((file) => {
|
|
3595
|
+
const current = currentFileByPath.get(file.path);
|
|
3596
|
+
const downloadedBytes = current?.downloadedBytes === void 0 ? 0 : file.sizeBytes === void 0 ? current.downloadedBytes : Math.min(current.downloadedBytes, file.sizeBytes);
|
|
3597
|
+
const status = file.sizeBytes !== void 0 && downloadedBytes >= file.sizeBytes ? "downloaded" : current?.status ?? "not-downloaded";
|
|
3598
|
+
return LocalModelLifecycleFileStateSchema.parse({
|
|
2934
3599
|
path: file.path,
|
|
2935
3600
|
sizeBytes: file.sizeBytes,
|
|
2936
|
-
downloadedBytes
|
|
2937
|
-
|
|
3601
|
+
downloadedBytes,
|
|
3602
|
+
required: file.required,
|
|
3603
|
+
status,
|
|
3604
|
+
updatedAt: current?.updatedAt,
|
|
3605
|
+
error: current?.error
|
|
3606
|
+
});
|
|
3607
|
+
});
|
|
3608
|
+
}
|
|
3609
|
+
function reconcileGroupFilesFromSnapshot(input) {
|
|
3610
|
+
const currentFileByPath = new Map(input.currentFiles.map((file) => [file.path, file]));
|
|
3611
|
+
return input.manifestGroup.files.map((file) => {
|
|
3612
|
+
const current = currentFileByPath.get(file.path);
|
|
3613
|
+
const downloadedBytes = current?.downloadedBytes === void 0 ? input.currentStatus === "downloaded" ? file.sizeBytes : 0 : file.sizeBytes === void 0 ? current.downloadedBytes : Math.min(current.downloadedBytes, file.sizeBytes);
|
|
3614
|
+
const status = current?.status ?? (file.sizeBytes !== void 0 && downloadedBytes !== void 0 && downloadedBytes >= file.sizeBytes ? "downloaded" : input.currentStatus === "downloaded" ? "downloaded" : input.currentStatus === "paused" || input.currentStatus === "downloading" || input.currentStatus === "error" ? input.currentStatus : "not-downloaded");
|
|
3615
|
+
return LocalModelLifecycleFileStateSchema.parse({
|
|
3616
|
+
path: file.path,
|
|
3617
|
+
sizeBytes: file.sizeBytes,
|
|
3618
|
+
downloadedBytes,
|
|
3619
|
+
required: file.required,
|
|
3620
|
+
status,
|
|
3621
|
+
updatedAt: current?.updatedAt,
|
|
3622
|
+
error: current?.error
|
|
3623
|
+
});
|
|
2938
3624
|
});
|
|
2939
3625
|
}
|
|
3626
|
+
async function reconcileGroupFilesFromDisk(input) {
|
|
3627
|
+
const currentFileByPath = new Map(input.currentFiles.map((file) => [file.path, file]));
|
|
3628
|
+
return Promise.all(input.manifestGroup.files.map(async (file) => {
|
|
3629
|
+
const current = currentFileByPath.get(file.path);
|
|
3630
|
+
const diskBytes = await readPathSize(join(input.rootDir, file.path));
|
|
3631
|
+
const downloadedBytes = diskBytes === null ? current?.downloadedBytes ?? 0 : file.sizeBytes === void 0 ? diskBytes : Math.min(diskBytes, file.sizeBytes);
|
|
3632
|
+
const status = file.sizeBytes !== void 0 && downloadedBytes >= file.sizeBytes ? "downloaded" : downloadedBytes > 0 ? "paused" : "not-downloaded";
|
|
3633
|
+
return LocalModelLifecycleFileStateSchema.parse({
|
|
3634
|
+
path: file.path,
|
|
3635
|
+
sizeBytes: file.sizeBytes,
|
|
3636
|
+
downloadedBytes,
|
|
3637
|
+
required: file.required,
|
|
3638
|
+
status,
|
|
3639
|
+
updatedAt: current?.updatedAt,
|
|
3640
|
+
error: current?.error
|
|
3641
|
+
});
|
|
3642
|
+
}));
|
|
3643
|
+
}
|
|
3644
|
+
function isActiveDownloadStatus(status) {
|
|
3645
|
+
return status === "queued" || status === "downloading" || status === "deleting";
|
|
3646
|
+
}
|
|
3647
|
+
function normalizeLiveStatusForStoredState(status) {
|
|
3648
|
+
if (status === "queued" || status === "downloading") return "paused";
|
|
3649
|
+
return status;
|
|
3650
|
+
}
|
|
2940
3651
|
function sumDownloadedBytes(files) {
|
|
2941
3652
|
return files.reduce((total, file) => {
|
|
2942
3653
|
const downloadedBytes = file.downloadedBytes ?? 0;
|
|
@@ -2964,12 +3675,11 @@ function throwIfAborted(signal) {
|
|
|
2964
3675
|
if (signal.aborted) throw new Error("Local model download aborted.");
|
|
2965
3676
|
}
|
|
2966
3677
|
async function downloadHuggingFaceFileToCacheDirWithProgress(input) {
|
|
2967
|
-
const revision = "main";
|
|
2968
3678
|
let lastError;
|
|
2969
3679
|
const info = await readHuggingFaceFileDownloadInfoWithRetry({
|
|
2970
3680
|
repo: input.repo,
|
|
2971
3681
|
path: input.path,
|
|
2972
|
-
revision,
|
|
3682
|
+
revision: input.revision,
|
|
2973
3683
|
hubUrl: input.hubUrl,
|
|
2974
3684
|
retryPolicy: input.retryPolicy,
|
|
2975
3685
|
fetch: input.fetch,
|
|
@@ -2983,25 +3693,25 @@ async function downloadHuggingFaceFileToCacheDirWithProgress(input) {
|
|
|
2983
3693
|
cacheDir: input.cacheDir,
|
|
2984
3694
|
modelId: input.repo.name,
|
|
2985
3695
|
filePath: input.path,
|
|
2986
|
-
revision,
|
|
2987
|
-
etag: info.etag
|
|
3696
|
+
revision: input.revision,
|
|
3697
|
+
etag: input.etag ?? info.etag
|
|
2988
3698
|
});
|
|
2989
|
-
const
|
|
2990
|
-
if (
|
|
3699
|
+
const existingTargetSize = await readPathSize(input.targetPath);
|
|
3700
|
+
if (existingTargetSize !== null && existingTargetSize >= totalBytes) {
|
|
2991
3701
|
await input.onProgress(totalBytes);
|
|
2992
|
-
return
|
|
3702
|
+
return input.targetPath;
|
|
2993
3703
|
}
|
|
2994
3704
|
for (let attempt = 0; attempt <= input.retryPolicy.limit; attempt += 1) try {
|
|
2995
3705
|
throwIfAborted(input.signal);
|
|
2996
|
-
let resumeBytes = await readPathSize(
|
|
3706
|
+
let resumeBytes = await readPathSize(`${input.targetPath}.incomplete`);
|
|
2997
3707
|
if (resumeBytes !== null && resumeBytes > totalBytes) {
|
|
2998
|
-
await rm(
|
|
3708
|
+
await rm(`${input.targetPath}.incomplete`, { force: true });
|
|
2999
3709
|
resumeBytes = 0;
|
|
3000
3710
|
}
|
|
3001
3711
|
if (resumeBytes !== null && resumeBytes > 0) await input.onProgress(Math.min(resumeBytes, totalBytes));
|
|
3002
3712
|
if (!await streamDownloadToIncompleteFile({
|
|
3003
3713
|
url: info.url,
|
|
3004
|
-
incompletePath:
|
|
3714
|
+
incompletePath: `${input.targetPath}.incomplete`,
|
|
3005
3715
|
startBytes: resumeBytes ?? 0,
|
|
3006
3716
|
totalBytes,
|
|
3007
3717
|
accessToken: void 0,
|
|
@@ -3012,7 +3722,7 @@ async function downloadHuggingFaceFileToCacheDirWithProgress(input) {
|
|
|
3012
3722
|
const blob = await downloadFile({
|
|
3013
3723
|
repo: input.repo,
|
|
3014
3724
|
path: input.path,
|
|
3015
|
-
revision,
|
|
3725
|
+
revision: input.revision,
|
|
3016
3726
|
hubUrl: input.hubUrl,
|
|
3017
3727
|
fetch: input.fetch,
|
|
3018
3728
|
downloadInfo: info,
|
|
@@ -3021,15 +3731,24 @@ async function downloadHuggingFaceFileToCacheDirWithProgress(input) {
|
|
|
3021
3731
|
if (!blob) throw new Error(`Invalid response for file ${input.path}.`);
|
|
3022
3732
|
await appendBlobToIncompleteFile({
|
|
3023
3733
|
blob: resumeBytes && resumeBytes > 0 ? blob.slice(resumeBytes, totalBytes) : blob,
|
|
3024
|
-
incompletePath:
|
|
3734
|
+
incompletePath: `${input.targetPath}.incomplete`,
|
|
3025
3735
|
startBytes: resumeBytes ?? 0,
|
|
3026
3736
|
totalBytes,
|
|
3027
3737
|
onProgress: input.onProgress
|
|
3028
3738
|
});
|
|
3029
3739
|
}
|
|
3030
|
-
await
|
|
3740
|
+
const incompleteSize = await readPathSize(`${input.targetPath}.incomplete`);
|
|
3741
|
+
if (incompleteSize === null || incompleteSize < totalBytes) throw new Error(`Incomplete response for file ${input.path}: downloaded ${incompleteSize ?? 0} of ${totalBytes} bytes.`);
|
|
3742
|
+
await finalizeDownloadedFile({
|
|
3743
|
+
incompletePath: `${input.targetPath}.incomplete`,
|
|
3744
|
+
targetPath: input.targetPath
|
|
3745
|
+
});
|
|
3746
|
+
await mirrorDownloadedFileToHubCache({
|
|
3747
|
+
targetPath: input.targetPath,
|
|
3748
|
+
cachePaths
|
|
3749
|
+
});
|
|
3031
3750
|
await input.onProgress(totalBytes);
|
|
3032
|
-
return
|
|
3751
|
+
return input.targetPath;
|
|
3033
3752
|
} catch (error) {
|
|
3034
3753
|
lastError = error;
|
|
3035
3754
|
if (!isRetryableDownloadError(error) || attempt === input.retryPolicy.limit) throw error;
|
|
@@ -3112,13 +3831,18 @@ async function streamDownloadToIncompleteFile(input) {
|
|
|
3112
3831
|
}
|
|
3113
3832
|
return true;
|
|
3114
3833
|
}
|
|
3115
|
-
async function
|
|
3116
|
-
await mkdir(dirname(input.
|
|
3117
|
-
await
|
|
3118
|
-
await
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
await
|
|
3834
|
+
async function finalizeDownloadedFile(input) {
|
|
3835
|
+
await mkdir(dirname(input.targetPath), { recursive: true });
|
|
3836
|
+
await rm(input.targetPath, { force: true });
|
|
3837
|
+
await rename(input.incompletePath, input.targetPath);
|
|
3838
|
+
}
|
|
3839
|
+
async function mirrorDownloadedFileToHubCache(input) {
|
|
3840
|
+
await mkdir(dirname(input.cachePaths.blobPath), { recursive: true });
|
|
3841
|
+
await mkdir(dirname(input.cachePaths.pointerPath), { recursive: true });
|
|
3842
|
+
await rm(input.cachePaths.blobPath, { force: true });
|
|
3843
|
+
await copyFile(input.targetPath, input.cachePaths.blobPath);
|
|
3844
|
+
await unlink(input.cachePaths.pointerPath).catch(() => void 0);
|
|
3845
|
+
await symlink(input.cachePaths.blobPath, input.cachePaths.pointerPath);
|
|
3122
3846
|
}
|
|
3123
3847
|
function getHubCacheFilePaths(input) {
|
|
3124
3848
|
const repoPath = getHubCacheRepoPath(input.cacheDir, input.modelId);
|
|
@@ -3168,6 +3892,7 @@ function formatDuration(ms) {
|
|
|
3168
3892
|
}
|
|
3169
3893
|
async function mirrorHubCacheFileForTransformers(input) {
|
|
3170
3894
|
const sourcePath = await resolveRealCacheFile(input.cachedPath);
|
|
3895
|
+
await copyFileIfMissing(sourcePath, join(input.profileRoot, input.filePath));
|
|
3171
3896
|
await copyFileIfMissing(sourcePath, join(getTransformersLocalModelPath(input.cacheDir, input.modelId), input.filePath));
|
|
3172
3897
|
await copyFileIfMissing(sourcePath, join(getTransformersFileCacheModelPath(input.cacheDir, input.modelId), input.filePath));
|
|
3173
3898
|
}
|
|
@@ -3187,9 +3912,11 @@ function getHubCacheRepoPath(cacheDir, modelId) {
|
|
|
3187
3912
|
return join(cacheDir, `models--${modelId.split("/").join("--")}`);
|
|
3188
3913
|
}
|
|
3189
3914
|
function toCatalogItem(candidate, asset) {
|
|
3190
|
-
const
|
|
3915
|
+
const downloadGroups = asset.plan?.groups ?? candidate.downloadGroups;
|
|
3916
|
+
const hasSelectableGroup = downloadGroups?.some((group) => group.selectable) ?? false;
|
|
3191
3917
|
return {
|
|
3192
3918
|
...candidate,
|
|
3919
|
+
downloadGroups,
|
|
3193
3920
|
asset,
|
|
3194
3921
|
selectable: hasSelectableGroup || (candidate.size.estimatedTotalBytes ?? 0) > 0,
|
|
3195
3922
|
local: asset.status === "downloaded" || asset.status === "paused" || asset.status === "downloading" || (asset.progress ?? 0) > 0
|
|
@@ -4868,7 +5595,7 @@ const globalSettingsRouter = router({
|
|
|
4868
5595
|
translationCache: TranslationCacheSettingsSchema.partial().optional(),
|
|
4869
5596
|
translationEngines: z.object({
|
|
4870
5597
|
openai: TranslationOpenAISettingsSchema.partial().optional(),
|
|
4871
|
-
local: TranslationLocalSettingsSchema.partial().optional()
|
|
5598
|
+
local: TranslationLocalSettingsSchema.partial().extend({ selectedGroupId: z.string().min(1).nullable().optional() }).optional()
|
|
4872
5599
|
}).optional()
|
|
4873
5600
|
})).mutation(async ({ ctx, input }) => {
|
|
4874
5601
|
await ctx.globalSettingsManager.writeSettings(input);
|
|
@@ -4959,6 +5686,18 @@ const localModelsRouter = router({
|
|
|
4959
5686
|
})).query(({ ctx, input }) => {
|
|
4960
5687
|
return ctx.localModelAssetService.readSelectedModelState(input.modelId, input.selectedGroupId);
|
|
4961
5688
|
}),
|
|
5689
|
+
panelState: publicProcedure.input(z.object({
|
|
5690
|
+
modelId: z.string().min(1),
|
|
5691
|
+
selectedGroupId: z.string().min(1).optional()
|
|
5692
|
+
})).query(async ({ ctx, input }) => {
|
|
5693
|
+
const asset = await ctx.localModelAssetService.readSelectedModelState(input.modelId, input.selectedGroupId);
|
|
5694
|
+
return {
|
|
5695
|
+
modelId: input.modelId,
|
|
5696
|
+
selectedGroupId: input.selectedGroupId ?? asset.plan?.selectedGroupId,
|
|
5697
|
+
asset,
|
|
5698
|
+
downloadPlan: asset.plan ?? null
|
|
5699
|
+
};
|
|
5700
|
+
}),
|
|
4962
5701
|
subscribeLogs: publicProcedure.subscription(({ ctx }) => {
|
|
4963
5702
|
return ctx.localModelAssetService.subscribeLogs();
|
|
4964
5703
|
}),
|
|
@@ -4968,21 +5707,40 @@ const localModelsRouter = router({
|
|
|
4968
5707
|
}),
|
|
4969
5708
|
download: publicProcedure.input(z.object({
|
|
4970
5709
|
modelId: z.string().min(1),
|
|
5710
|
+
groupId: z.string().min(1).optional(),
|
|
4971
5711
|
selectedGroupId: z.string().min(1).optional()
|
|
4972
5712
|
})).mutation(async ({ ctx, input }) => {
|
|
4973
|
-
return ctx.localModelAssetService.startDownload(input.modelId, input.selectedGroupId);
|
|
5713
|
+
return ctx.localModelAssetService.startDownload(input.modelId, input.groupId ?? input.selectedGroupId);
|
|
4974
5714
|
}),
|
|
4975
|
-
pause: publicProcedure.input(z.object({
|
|
4976
|
-
|
|
5715
|
+
pause: publicProcedure.input(z.object({
|
|
5716
|
+
modelId: z.string().min(1),
|
|
5717
|
+
groupId: z.string().min(1).optional(),
|
|
5718
|
+
selectedGroupId: z.string().min(1).optional()
|
|
5719
|
+
})).mutation(({ ctx, input }) => {
|
|
5720
|
+
return ctx.localModelAssetService.pauseDownload(input.modelId, input.groupId ?? input.selectedGroupId);
|
|
4977
5721
|
}),
|
|
4978
5722
|
resume: publicProcedure.input(z.object({
|
|
4979
5723
|
modelId: z.string().min(1),
|
|
5724
|
+
groupId: z.string().min(1).optional(),
|
|
4980
5725
|
selectedGroupId: z.string().min(1).optional()
|
|
4981
5726
|
})).mutation(async ({ ctx, input }) => {
|
|
4982
|
-
return ctx.localModelAssetService.resumeDownload(input.modelId, input.selectedGroupId);
|
|
5727
|
+
return ctx.localModelAssetService.resumeDownload(input.modelId, input.groupId ?? input.selectedGroupId);
|
|
4983
5728
|
}),
|
|
4984
|
-
delete: publicProcedure.input(z.object({
|
|
4985
|
-
|
|
5729
|
+
delete: publicProcedure.input(z.object({
|
|
5730
|
+
modelId: z.string().min(1),
|
|
5731
|
+
groupId: z.string().min(1).optional(),
|
|
5732
|
+
selectedGroupId: z.string().min(1).optional()
|
|
5733
|
+
})).mutation(({ ctx, input }) => {
|
|
5734
|
+
return ctx.localModelAssetService.deleteModel(input.modelId, input.groupId ?? input.selectedGroupId);
|
|
5735
|
+
}),
|
|
5736
|
+
refreshProfiles: publicProcedure.input(z.object({ modelId: z.string().min(1).optional() })).mutation(async ({ ctx, input }) => {
|
|
5737
|
+
const asset = await ctx.localModelAssetService.refreshProfiles(input.modelId);
|
|
5738
|
+
return {
|
|
5739
|
+
modelId: asset.modelId,
|
|
5740
|
+
selectedGroupId: asset.selectedGroupId ?? asset.plan?.selectedGroupId,
|
|
5741
|
+
asset,
|
|
5742
|
+
downloadPlan: asset.plan ?? null
|
|
5743
|
+
};
|
|
4986
5744
|
})
|
|
4987
5745
|
});
|
|
4988
5746
|
const OPSX_CORE_PROFILE_WORKFLOWS = [
|
|
@@ -5290,6 +6048,31 @@ const changeRouter = router({
|
|
|
5290
6048
|
}),
|
|
5291
6049
|
subscribeFiles: publicProcedure.input(z.object({ id: z.string() })).subscription(({ ctx, input }) => {
|
|
5292
6050
|
return createReactiveSubscriptionWithInput((id) => ctx.adapter.readChangeFiles(id))(input.id);
|
|
6051
|
+
}),
|
|
6052
|
+
writeFile: publicProcedure.input(z.object({
|
|
6053
|
+
id: z.string(),
|
|
6054
|
+
path: z.string(),
|
|
6055
|
+
content: z.string()
|
|
6056
|
+
})).mutation(async ({ ctx, input }) => {
|
|
6057
|
+
const info = resolveEntityEntryPath({
|
|
6058
|
+
projectDir: ctx.projectDir,
|
|
6059
|
+
stage: "change",
|
|
6060
|
+
changeId: input.id,
|
|
6061
|
+
path: input.path
|
|
6062
|
+
});
|
|
6063
|
+
await mkdir(dirname(info.absolutePath), { recursive: true });
|
|
6064
|
+
await writeFile(info.absolutePath, input.content, "utf-8");
|
|
6065
|
+
return { success: true };
|
|
6066
|
+
}),
|
|
6067
|
+
prepareFilePreview: publicProcedure.input(z.object({
|
|
6068
|
+
id: z.string(),
|
|
6069
|
+
path: z.string()
|
|
6070
|
+
})).query(({ ctx, input }) => {
|
|
6071
|
+
return ctx.filePreviewService.prepareEntityFilePreview({
|
|
6072
|
+
stage: "change",
|
|
6073
|
+
changeId: input.id,
|
|
6074
|
+
path: input.path
|
|
6075
|
+
});
|
|
5293
6076
|
})
|
|
5294
6077
|
});
|
|
5295
6078
|
/**
|
|
@@ -5322,9 +6105,32 @@ const archiveRouter = router({
|
|
|
5322
6105
|
return createReactiveSubscriptionWithInput(async (id) => ctx.documentService.readEntityDetail("archive", id, "view", "processed", await buildEntityReadOptions(ctx, "archive", id)))(input.id);
|
|
5323
6106
|
}),
|
|
5324
6107
|
subscribeFiles: publicProcedure.input(z.object({ id: z.string() })).subscription(({ ctx, input }) => {
|
|
5325
|
-
return createReactiveSubscriptionWithInput(
|
|
5326
|
-
|
|
5327
|
-
|
|
6108
|
+
return createReactiveSubscriptionWithInput((id) => ctx.documentService.readArchivedChangeFiles(id, "view", "source"))(input.id);
|
|
6109
|
+
}),
|
|
6110
|
+
writeFile: publicProcedure.input(z.object({
|
|
6111
|
+
id: z.string(),
|
|
6112
|
+
path: z.string(),
|
|
6113
|
+
content: z.string()
|
|
6114
|
+
})).mutation(async ({ ctx, input }) => {
|
|
6115
|
+
const info = resolveEntityEntryPath({
|
|
6116
|
+
projectDir: ctx.projectDir,
|
|
6117
|
+
stage: "archive",
|
|
6118
|
+
changeId: input.id,
|
|
6119
|
+
path: input.path
|
|
6120
|
+
});
|
|
6121
|
+
await mkdir(dirname(info.absolutePath), { recursive: true });
|
|
6122
|
+
await writeFile(info.absolutePath, input.content, "utf-8");
|
|
6123
|
+
return { success: true };
|
|
6124
|
+
}),
|
|
6125
|
+
prepareFilePreview: publicProcedure.input(z.object({
|
|
6126
|
+
id: z.string(),
|
|
6127
|
+
path: z.string()
|
|
6128
|
+
})).query(({ ctx, input }) => {
|
|
6129
|
+
return ctx.filePreviewService.prepareEntityFilePreview({
|
|
6130
|
+
stage: "archive",
|
|
6131
|
+
changeId: input.id,
|
|
6132
|
+
path: input.path
|
|
6133
|
+
});
|
|
5328
6134
|
})
|
|
5329
6135
|
});
|
|
5330
6136
|
z.object({
|
|
@@ -5430,7 +6236,7 @@ const configRouter = router({
|
|
|
5430
6236
|
translation: DocumentTranslationConfigSchema.partial().extend({ engines: z.object({
|
|
5431
6237
|
local: z.object({
|
|
5432
6238
|
model: z.string().min(1).optional(),
|
|
5433
|
-
selectedGroupId: z.string().min(1).optional()
|
|
6239
|
+
selectedGroupId: z.string().min(1).nullable().optional()
|
|
5434
6240
|
}).optional(),
|
|
5435
6241
|
openai: z.object({ model: z.string().min(1).optional() }).optional()
|
|
5436
6242
|
}).optional() }).optional()
|
|
@@ -6465,22 +7271,19 @@ var TranslationCacheService = class {
|
|
|
6465
7271
|
//#endregion
|
|
6466
7272
|
//#region src/translation-engine-service.ts
|
|
6467
7273
|
var TranslationEngineService = class {
|
|
6468
|
-
projectDir;
|
|
6469
7274
|
configManager;
|
|
6470
7275
|
globalSettingsManager;
|
|
6471
7276
|
now;
|
|
6472
7277
|
localCacheDir;
|
|
6473
7278
|
localAssetStore;
|
|
6474
|
-
localFetchCacheStore;
|
|
6475
7279
|
constructor(options) {
|
|
6476
7280
|
ensureProxyAwareFetchDispatcher();
|
|
6477
|
-
this.projectDir = options.projectDir;
|
|
6478
7281
|
this.configManager = options.configManager;
|
|
6479
7282
|
this.globalSettingsManager = options.globalSettingsManager;
|
|
6480
7283
|
this.now = options.now ?? Date.now;
|
|
6481
7284
|
this.localCacheDir = options.localCacheDir ?? getDefaultLocalModelCacheDir();
|
|
6482
7285
|
this.localAssetStore = new LocalModelAssetStore({ indexPath: options.localAssetIndexPath ?? getDefaultLocalModelIndexPath() });
|
|
6483
|
-
|
|
7286
|
+
new LocalModelFetchCacheStore({
|
|
6484
7287
|
cachePath: options.localFetchCachePath ?? getDefaultLocalModelFetchCachePath(),
|
|
6485
7288
|
now: this.now
|
|
6486
7289
|
});
|
|
@@ -6500,20 +7303,7 @@ var TranslationEngineService = class {
|
|
|
6500
7303
|
}
|
|
6501
7304
|
async getModelDownloadPlan(input) {
|
|
6502
7305
|
if (input.engineId !== "local") return null;
|
|
6503
|
-
|
|
6504
|
-
const plan = await resolveLocalModelRuntimePlanFromProject({
|
|
6505
|
-
projectDir: this.projectDir,
|
|
6506
|
-
globalSettingsManager: this.globalSettingsManager,
|
|
6507
|
-
modelId: input.model,
|
|
6508
|
-
selectedGroupId: input.selectedGroupId,
|
|
6509
|
-
cacheDir: this.localCacheDir,
|
|
6510
|
-
fetchCacheStore: this.localFetchCacheStore,
|
|
6511
|
-
loadTransformersModule: this.loadLocalTransformersModuleForPlan.bind(this)
|
|
6512
|
-
}).catch(() => null);
|
|
6513
|
-
const fallbackPlan = selectPersistedLocalPlan(state, input.selectedGroupId);
|
|
6514
|
-
const effectivePlan = plan ?? fallbackPlan;
|
|
6515
|
-
if (!effectivePlan) return null;
|
|
6516
|
-
return enrichDownloadPlanWithAssetSnapshot(effectivePlan, state, input.selectedGroupId);
|
|
7306
|
+
return selectPersistedLocalPlan((await this.localAssetStore.readMap()).get(input.model), input.selectedGroupId);
|
|
6517
7307
|
}
|
|
6518
7308
|
async selectEngine(engineId) {
|
|
6519
7309
|
await this.configManager.writeConfig({ translation: { engineId } });
|
|
@@ -6529,14 +7319,26 @@ var TranslationEngineService = class {
|
|
|
6529
7319
|
(async () => {
|
|
6530
7320
|
try {
|
|
6531
7321
|
if (input.engineId === "browser") throw new Error("Browser translator runs in the browser runtime.");
|
|
6532
|
-
const
|
|
6533
|
-
|
|
6534
|
-
|
|
7322
|
+
const settingsSnapshot = await this.globalSettingsManager.readSettings();
|
|
7323
|
+
const effectiveModel = resolveBatchTranslateModel(input, settingsSnapshot);
|
|
7324
|
+
if (input.engineId === "local") {
|
|
7325
|
+
const directionCheck = checkLocalDirectionalModelLanguagePair({
|
|
7326
|
+
model: effectiveModel,
|
|
7327
|
+
sourceLanguage: input.sourceLanguage,
|
|
7328
|
+
targetLanguage: input.targetLanguage
|
|
7329
|
+
});
|
|
7330
|
+
if (!directionCheck.supported) throw new Error(directionCheck.message ?? "Selected local model does not support the requested translation direction.");
|
|
7331
|
+
}
|
|
7332
|
+
const effectiveSelectedGroupId = input.engineId === "local" ? input.selectedGroupId ?? settingsSnapshot.translationEngines.local.selectedGroupId : void 0;
|
|
7333
|
+
const dtype = await this.readLocalDtype(input.engineId, effectiveModel, effectiveSelectedGroupId);
|
|
7334
|
+
if (input.engineId === "local" && effectiveModel) await this.assertLocalModelReady(effectiveModel, effectiveSelectedGroupId);
|
|
7335
|
+
const runtimeConfig = input.engineId === "local" && effectiveModel ? await this.readLocalRuntimeConfig(effectiveModel, effectiveSelectedGroupId) : void 0;
|
|
7336
|
+
const translator = await (await this.loadFactory(input.engineId, effectiveModel, settingsSnapshot)).create({
|
|
6535
7337
|
sourceLanguage: input.sourceLanguage,
|
|
6536
7338
|
targetLanguage: input.targetLanguage,
|
|
6537
|
-
model:
|
|
7339
|
+
model: effectiveModel,
|
|
6538
7340
|
dtype,
|
|
6539
|
-
runtimeConfig
|
|
7341
|
+
runtimeConfig,
|
|
6540
7342
|
signal: controller.signal
|
|
6541
7343
|
});
|
|
6542
7344
|
try {
|
|
@@ -6562,11 +7364,11 @@ var TranslationEngineService = class {
|
|
|
6562
7364
|
if (engineId !== "local" || !model) return void 0;
|
|
6563
7365
|
const effectiveSelectedGroupId = selectedGroupId ?? (await this.globalSettingsManager.readSettings()).translationEngines.local.selectedGroupId;
|
|
6564
7366
|
if (!effectiveSelectedGroupId) return void 0;
|
|
6565
|
-
return (await this.getModelDownloadPlan({
|
|
7367
|
+
return selectLocalPlanGroup(await this.getModelDownloadPlan({
|
|
6566
7368
|
engineId: "local",
|
|
6567
7369
|
model,
|
|
6568
7370
|
selectedGroupId: effectiveSelectedGroupId
|
|
6569
|
-
})
|
|
7371
|
+
}), effectiveSelectedGroupId)?.dtype;
|
|
6570
7372
|
}
|
|
6571
7373
|
async assertLocalModelReady(model, selectedGroupId) {
|
|
6572
7374
|
const plan = await this.getModelDownloadPlan({
|
|
@@ -6574,89 +7376,57 @@ var TranslationEngineService = class {
|
|
|
6574
7376
|
model,
|
|
6575
7377
|
selectedGroupId
|
|
6576
7378
|
});
|
|
6577
|
-
const
|
|
6578
|
-
if (!plan || files.length === 0) throw new Error("No local runtime file plan is available for the selected model.");
|
|
6579
|
-
const
|
|
6580
|
-
|
|
6581
|
-
|
|
6582
|
-
|
|
6583
|
-
});
|
|
6584
|
-
if (cacheStatus.allCached) {
|
|
6585
|
-
const current = (await this.localAssetStore.readMap()).get(model);
|
|
6586
|
-
if (current) await this.localAssetStore.upsert({
|
|
6587
|
-
...current,
|
|
6588
|
-
status: "downloaded",
|
|
6589
|
-
progress: 1,
|
|
6590
|
-
bytesDownloaded: plan.estimatedTotalBytes ?? current.bytesDownloaded,
|
|
6591
|
-
totalBytes: plan.estimatedTotalBytes ?? current.totalBytes,
|
|
6592
|
-
resumable: false,
|
|
6593
|
-
error: void 0,
|
|
6594
|
-
plan,
|
|
6595
|
-
files: files.map((file) => ({
|
|
6596
|
-
path: file.path,
|
|
6597
|
-
sizeBytes: file.sizeBytes,
|
|
6598
|
-
downloadedBytes: file.sizeBytes
|
|
6599
|
-
})),
|
|
6600
|
-
installedAt: current.installedAt ?? this.now(),
|
|
6601
|
-
updatedAt: this.now()
|
|
6602
|
-
});
|
|
6603
|
-
return;
|
|
7379
|
+
const selectedGroup = selectLocalPlanGroup(plan, selectedGroupId);
|
|
7380
|
+
if (!plan || !selectedGroup || selectedGroup.files.length === 0) throw new Error("No local runtime file plan is available for the selected model.");
|
|
7381
|
+
const files = selectedGroup.files;
|
|
7382
|
+
const selectedGroupState = await this.readSelectedLocalGroupState(model, selectedGroup.id);
|
|
7383
|
+
if (selectedGroupState?.status === "downloaded" && selectedGroup.rootDir) {
|
|
7384
|
+
if ((await readMissingLocalGroupFiles(selectedGroup.rootDir, files)).length === 0) return;
|
|
6604
7385
|
}
|
|
6605
|
-
|
|
7386
|
+
if (selectedGroupState?.status === "downloaded") return;
|
|
7387
|
+
const allMissingFiles = selectedGroup.rootDir ? await readMissingLocalGroupFiles(selectedGroup.rootDir, files) : files.map((file) => file.path);
|
|
6606
7388
|
const missingFiles = allMissingFiles.slice(0, 3);
|
|
6607
7389
|
const suffix = allMissingFiles.length > missingFiles.length ? ` and ${allMissingFiles.length - missingFiles.length} more` : "";
|
|
6608
7390
|
throw new Error(`Selected local model files are not installed locally: ${missingFiles.join(", ")}${suffix}.`);
|
|
6609
7391
|
}
|
|
6610
|
-
async readLocalRuntimeConfig(model) {
|
|
7392
|
+
async readLocalRuntimeConfig(model, selectedGroupId) {
|
|
7393
|
+
const selectedGroup = selectLocalPlanGroup(await this.getModelDownloadPlan({
|
|
7394
|
+
engineId: "local",
|
|
7395
|
+
model,
|
|
7396
|
+
selectedGroupId
|
|
7397
|
+
}), selectedGroupId);
|
|
7398
|
+
const configPath = selectedGroup?.rootDir ? join(selectedGroup.rootDir, "config.json") : join(this.localCacheDir, "models", model, "config.json");
|
|
6611
7399
|
try {
|
|
6612
|
-
return JSON.parse(await readFile(
|
|
7400
|
+
return JSON.parse(await readFile(configPath, "utf8"));
|
|
6613
7401
|
} catch {
|
|
6614
7402
|
return;
|
|
6615
7403
|
}
|
|
6616
7404
|
}
|
|
6617
|
-
async
|
|
6618
|
-
|
|
6619
|
-
|
|
7405
|
+
async readSelectedLocalGroupState(model, selectedGroupId) {
|
|
7406
|
+
return (await this.localAssetStore.readMap()).get(model)?.groupsState[selectedGroupId];
|
|
7407
|
+
}
|
|
7408
|
+
async loadFactory(engineId, model, settingsSnapshot) {
|
|
7409
|
+
const globalSettings = settingsSnapshot ?? await this.globalSettingsManager.readSettings();
|
|
7410
|
+
if (engineId === "local") return (await import("./src-DPZ2-0Yn.mjs")).createLocalTranslatorFactory({
|
|
6620
7411
|
defaultModel: model ?? globalSettings.translationEngines.local.model,
|
|
6621
7412
|
cacheDir: this.localCacheDir,
|
|
6622
7413
|
localOnly: true
|
|
6623
7414
|
});
|
|
6624
|
-
return (await import("./src-
|
|
7415
|
+
return (await import("./src-CoUhFB25.mjs")).createOpenAICompletionTranslatorFactory({
|
|
6625
7416
|
baseUrl: globalSettings.translationEngines.openai.baseUrl,
|
|
6626
7417
|
token: globalSettings.translationEngines.openai.token,
|
|
6627
7418
|
model: model ?? globalSettings.translationEngines.openai.model
|
|
6628
7419
|
});
|
|
6629
7420
|
}
|
|
6630
|
-
async loadLocalTransformersModuleForPlan(_projectDir, _globalSettingsManager) {
|
|
6631
|
-
return await import("@huggingface/transformers");
|
|
6632
|
-
}
|
|
6633
7421
|
};
|
|
6634
|
-
function
|
|
6635
|
-
if (
|
|
6636
|
-
|
|
6637
|
-
|
|
6638
|
-
const matchingAssetGroup = state.plan?.groups?.find((asset) => asset.id === group.id);
|
|
6639
|
-
if (!matchingAssetGroup) return group;
|
|
6640
|
-
return {
|
|
6641
|
-
...group,
|
|
6642
|
-
estimatedTotalBytes: group.estimatedTotalBytes ?? matchingAssetGroup.estimatedTotalBytes,
|
|
6643
|
-
files: group.files.map((file) => {
|
|
6644
|
-
const matchingAssetFile = matchingAssetGroup.files.find((asset) => asset.path === file.path);
|
|
6645
|
-
return matchingAssetFile?.sizeBytes !== void 0 && file.sizeBytes === void 0 ? {
|
|
6646
|
-
...file,
|
|
6647
|
-
sizeBytes: matchingAssetFile.sizeBytes
|
|
6648
|
-
} : file;
|
|
6649
|
-
})
|
|
6650
|
-
};
|
|
6651
|
-
});
|
|
6652
|
-
return {
|
|
6653
|
-
...plan,
|
|
6654
|
-
estimatedTotalBytes: plan.estimatedTotalBytes ?? assetGroup?.estimatedTotalBytes ?? state.plan.estimatedTotalBytes,
|
|
6655
|
-
groups: mergedGroups
|
|
6656
|
-
};
|
|
7422
|
+
function resolveBatchTranslateModel(input, settings) {
|
|
7423
|
+
if (input.model) return input.model;
|
|
7424
|
+
if (input.engineId === "local") return settings.translationEngines.local.model;
|
|
7425
|
+
if (input.engineId === "openai") return settings.translationEngines.openai.model;
|
|
6657
7426
|
}
|
|
6658
7427
|
function selectPersistedLocalPlan(state, selectedGroupId) {
|
|
6659
|
-
|
|
7428
|
+
if (!state) return null;
|
|
7429
|
+
const plan = LocalModelAssetStateSchema.parse(state).plan;
|
|
6660
7430
|
if (!plan) return null;
|
|
6661
7431
|
if (!selectedGroupId || !plan.groups?.length) return {
|
|
6662
7432
|
...plan,
|
|
@@ -6666,7 +7436,7 @@ function selectPersistedLocalPlan(state, selectedGroupId) {
|
|
|
6666
7436
|
files: [...group.files]
|
|
6667
7437
|
}))
|
|
6668
7438
|
};
|
|
6669
|
-
const selectedGroup = plan
|
|
7439
|
+
const selectedGroup = selectLocalPlanGroup(plan, selectedGroupId);
|
|
6670
7440
|
if (!selectedGroup) return null;
|
|
6671
7441
|
return {
|
|
6672
7442
|
modelId: plan.modelId,
|
|
@@ -6680,6 +7450,22 @@ function selectPersistedLocalPlan(state, selectedGroupId) {
|
|
|
6680
7450
|
}))
|
|
6681
7451
|
};
|
|
6682
7452
|
}
|
|
7453
|
+
function selectLocalPlanGroup(plan, selectedGroupId) {
|
|
7454
|
+
if (!plan?.groups?.length) return void 0;
|
|
7455
|
+
const requestedGroupId = selectedGroupId ?? plan.selectedGroupId;
|
|
7456
|
+
return plan.groups.find((group) => group.id === requestedGroupId) ?? plan.groups.find((group) => group.baseGroupId === requestedGroupId) ?? plan.groups.find((group) => group.selected) ?? plan.groups[0];
|
|
7457
|
+
}
|
|
7458
|
+
async function readMissingLocalGroupFiles(rootDir, files) {
|
|
7459
|
+
return (await Promise.all(files.map(async (file) => {
|
|
7460
|
+
try {
|
|
7461
|
+
const entry = await stat(join(rootDir, file.path));
|
|
7462
|
+
if (file.sizeBytes !== void 0 && entry.size < file.sizeBytes) return file.path;
|
|
7463
|
+
return null;
|
|
7464
|
+
} catch {
|
|
7465
|
+
return file.path;
|
|
7466
|
+
}
|
|
7467
|
+
}))).filter((file) => file !== null);
|
|
7468
|
+
}
|
|
6683
7469
|
|
|
6684
7470
|
//#endregion
|
|
6685
7471
|
//#region src/workflow-invocation-service.ts
|
|
@@ -6940,6 +7726,7 @@ function createServer(config) {
|
|
|
6940
7726
|
const kernel = config.kernel;
|
|
6941
7727
|
const hookRuntime = createHookRuntime(config.projectDir);
|
|
6942
7728
|
const documentService = new DocumentService(config.projectDir, adapter, hookRuntime);
|
|
7729
|
+
const filePreviewService = new FilePreviewService(config.projectDir, config.previewAssetsDir ?? join(__dirname, "..", "..", "web", "dist"));
|
|
6943
7730
|
const workflowInvocationService = new WorkflowInvocationService({
|
|
6944
7731
|
projectDir: config.projectDir,
|
|
6945
7732
|
hookRuntime,
|
|
@@ -6975,6 +7762,7 @@ function createServer(config) {
|
|
|
6975
7762
|
});
|
|
6976
7763
|
const nmtModelCacheDir = config.runtimePaths?.localModelCacheDir ?? getDefaultLocalModelCacheDir();
|
|
6977
7764
|
const nmtModelIndexPath = config.runtimePaths?.localModelAssetIndexPath ?? getDefaultLocalModelIndexPath();
|
|
7765
|
+
const nmtModelProfileManifestPath = config.runtimePaths?.localModelProfileManifestPath ?? getDefaultLocalModelProfileManifestPath();
|
|
6978
7766
|
const nmtModelFetchCachePath = config.runtimePaths?.localModelFetchCachePath ?? getDefaultLocalModelFetchCachePath();
|
|
6979
7767
|
const translationEngineService = new TranslationEngineService({
|
|
6980
7768
|
projectDir: config.projectDir,
|
|
@@ -6990,6 +7778,7 @@ function createServer(config) {
|
|
|
6990
7778
|
globalSettingsManager,
|
|
6991
7779
|
cacheDir: nmtModelCacheDir,
|
|
6992
7780
|
indexPath: nmtModelIndexPath,
|
|
7781
|
+
profileManifestPath: nmtModelProfileManifestPath,
|
|
6993
7782
|
fetchCachePath: nmtModelFetchCachePath
|
|
6994
7783
|
});
|
|
6995
7784
|
const watcher = config.enableWatcher !== false ? new OpenSpecWatcher(config.projectDir) : void 0;
|
|
@@ -7054,6 +7843,21 @@ function createServer(config) {
|
|
|
7054
7843
|
"Cache-Control": "private, max-age=31536000, immutable"
|
|
7055
7844
|
} });
|
|
7056
7845
|
});
|
|
7846
|
+
app.get("/api/file-preview/:hash/*", async (c) => {
|
|
7847
|
+
const hash = c.req.param("hash");
|
|
7848
|
+
const prefix = `/api/file-preview/${hash}/`;
|
|
7849
|
+
const requestPath = c.req.path.startsWith(prefix) ? c.req.path.slice(prefix.length) : "";
|
|
7850
|
+
const asset = filePreviewService.readPreviewRequest(hash, requestPath);
|
|
7851
|
+
if (!asset) return c.notFound();
|
|
7852
|
+
const stream = new ReadableStream({ start(controller) {
|
|
7853
|
+
controller.enqueue(new Uint8Array(asset.content));
|
|
7854
|
+
controller.close();
|
|
7855
|
+
} });
|
|
7856
|
+
return new Response(stream, {
|
|
7857
|
+
status: 200,
|
|
7858
|
+
headers: { "Content-Type": asset.contentType }
|
|
7859
|
+
});
|
|
7860
|
+
});
|
|
7057
7861
|
app.use("/trpc/*", async (c) => {
|
|
7058
7862
|
return await fetchRequestHandler({
|
|
7059
7863
|
endpoint: "/trpc",
|
|
@@ -7073,6 +7877,7 @@ function createServer(config) {
|
|
|
7073
7877
|
customSoundService,
|
|
7074
7878
|
globalSettingsManager,
|
|
7075
7879
|
translationCacheService,
|
|
7880
|
+
filePreviewService,
|
|
7076
7881
|
translationEngineService,
|
|
7077
7882
|
localModelAssetService,
|
|
7078
7883
|
gitWorktreeHandoff: config.gitWorktreeHandoff,
|
|
@@ -7095,6 +7900,7 @@ function createServer(config) {
|
|
|
7095
7900
|
customSoundService,
|
|
7096
7901
|
globalSettingsManager,
|
|
7097
7902
|
translationCacheService,
|
|
7903
|
+
filePreviewService,
|
|
7098
7904
|
translationEngineService,
|
|
7099
7905
|
localModelAssetService,
|
|
7100
7906
|
gitWorktreeHandoff: config.gitWorktreeHandoff,
|
|
@@ -7116,7 +7922,9 @@ function createServer(config) {
|
|
|
7116
7922
|
customSoundService,
|
|
7117
7923
|
globalSettingsManager,
|
|
7118
7924
|
translationCacheService,
|
|
7925
|
+
filePreviewService,
|
|
7119
7926
|
translationEngineService,
|
|
7927
|
+
localModelAssetService,
|
|
7120
7928
|
hookRuntime,
|
|
7121
7929
|
watcher,
|
|
7122
7930
|
createContext,
|