@qvac/sdk 0.9.2 → 0.10.1

package/dist/server/rpc/handlers/load-model/registry-client-options.d.ts

@@ -0,0 +1,27 @@
+import type { QvacConfig } from "../../../../schemas/index";
+export declare const DEFAULT_REGISTRY_STREAM_TIMEOUT_MS = 60000;
+export interface RegistryClientDownloadOptions {
+    timeout: number;
+    outputFile: string;
+    maxRetries?: number;
+    onProgress?: (progress: {
+        downloaded: number;
+        total: number;
+    }) => void;
+    signal?: globalThis.AbortSignal;
+}
+/**
+ * Build the options passed to @qvac/registry-client download methods from the
+ * current SDK config. Pure helper (no bare-* imports) so the mapping can be
+ * unit-tested without standing up a real registry client.
+ */
+export declare function buildRegistryClientOptions(params: {
+    sdkConfig: Pick<QvacConfig, "registryStreamTimeoutMs" | "registryDownloadMaxRetries">;
+    outputFile: string;
+    onProgress?: ((progress: {
+        downloaded: number;
+        total: number;
+    }) => void) | undefined;
+    signal?: globalThis.AbortSignal | undefined;
+}): RegistryClientDownloadOptions;
+//# sourceMappingURL=registry-client-options.d.ts.map

package/dist/server/rpc/handlers/load-model/registry-client-options.js

@@ -0,0 +1,19 @@
+export const DEFAULT_REGISTRY_STREAM_TIMEOUT_MS = 60_000;
+/**
+ * Build the options passed to @qvac/registry-client download methods from the
+ * current SDK config. Pure helper (no bare-* imports) so the mapping can be
+ * unit-tested without standing up a real registry client.
+ */
+export function buildRegistryClientOptions(params) {
+    const { sdkConfig, outputFile, onProgress, signal } = params;
+    const timeout = sdkConfig.registryStreamTimeoutMs
+        ?? DEFAULT_REGISTRY_STREAM_TIMEOUT_MS;
+    const maxRetries = sdkConfig.registryDownloadMaxRetries;
+    return {
+        timeout,
+        outputFile,
+        ...(maxRetries !== undefined && { maxRetries }),
+        ...(onProgress && { onProgress }),
+        ...(signal && { signal: signal }),
+    };
+}

package/dist/server/rpc/handlers/load-model/registry-companion-set.d.ts

@@ -0,0 +1,22 @@
+import type { ModelProgressUpdate } from "../../../../schemas/index";
+import type { RegistryItem } from "../../../../models/registry/index";
+import type { AbortSignal } from "bare-abort-controller";
+import type { DownloadHooks } from "./types";
+type CompanionSetMetadata = NonNullable<RegistryItem["companionSet"]>;
+export interface DownloadCompanionSetOptions {
+    companionSet: CompanionSetMetadata;
+    downloadKey: string;
+    progressCallback?: ((progress: ModelProgressUpdate) => void) | undefined;
+    signal?: AbortSignal | undefined;
+    hooks?: DownloadHooks | undefined;
+    shouldClearCache?: (() => boolean) | undefined;
+}
+/**
+ * Download a companion set from the registry, placing files directly
+ * into their canonical layout at `sets/<setKey>/<targetName>`.
+ *
+ * Returns the path to the primary file.
+ */
+export declare function downloadCompanionSetFromRegistry(options: DownloadCompanionSetOptions): Promise<string>;
+export {};
+//# sourceMappingURL=registry-companion-set.d.ts.map

package/dist/server/rpc/handlers/load-model/registry-companion-set.js

@@ -0,0 +1,208 @@
+import { promises as fsPromises } from "bare-fs";
+import { getModelsCacheDir, getCompanionSetPath, getCompanionSetCacheDir, getSingleFileCachePath, generateShortHash, calculatePercentage, } from "../../../utils/index.js";
+import { measureChecksum } from "../../../utils/checksum.js";
+import { validateAndJoinPath } from "../../../utils/path-security.js";
+import { buildBlobBinding, downloadSingleFileFromRegistry, validateCachedFile, } from "./registry-download-utils.js";
+import { DownloadCancelledError, RegistryDownloadFailedError, } from "../../../../utils/errors-server.js";
+import { getServerLogger } from "../../../../logging/index.js";
+const logger = getServerLogger();
+/**
+ * Download a companion set from the registry, placing files directly
+ * into their canonical layout at `sets/<setKey>/<targetName>`.
+ *
+ * Returns the path to the primary file.
+ */
+export async function downloadCompanionSetFromRegistry(options) {
+    const { companionSet, downloadKey, progressCallback, signal, hooks, shouldClearCache, } = options;
+    const { files, setKey, primaryKey } = companionSet;
+    validateCompanionSetMetadata(files, primaryKey);
+    const primaryEntry = files.find((f) => f.key === primaryKey);
+    const primaryRegistryPath = primaryEntry.registryPath;
+    if (isLegacyOnnxSet(files)) {
+        const legacyPath = await checkLegacyOnnxCache(primaryRegistryPath, files, hooks);
+        if (legacyPath) {
+            logger.info(`✅ Using legacy ONNX cache for companion set: ${legacyPath}`);
+            hooks?.markCacheHit?.();
+            emitFinalProgress(progressCallback, downloadKey, setKey, files);
+            return legacyPath;
+        }
+    }
+    // Legacy flat-cache compatibility for Bergamot-style companion sets.
+    // This is valid only for families whose runtime still works with explicit
+    // per-file absolute paths; it is not a generic rule for all companion sets.
+    if (!isLegacyOnnxSet(files)) {
+        const flatPath = await checkLegacyFlatCache(files, primaryKey, hooks);
+        if (flatPath) {
+            logger.info(`✅ Using legacy flat cache for companion set: ${flatPath}`);
+            hooks?.markCacheHit?.();
+            emitFinalProgress(progressCallback, downloadKey, setKey, files);
+            return flatPath;
+        }
+    }
+    // Check canonical companion set cache
+    const validatedFiles = new Set();
+    for (const file of files) {
+        const filePath = getCompanionSetPath(setKey, file.targetName);
+        const cached = await validateCachedFile(filePath, file.targetName, file.expectedSize, file.sha256Checksum, hooks);
+        if (cached) {
+            validatedFiles.add(file.key);
+        }
+    }
+    if (validatedFiles.size === files.length) {
+        logger.info(`✅ All companion set files cached`);
+        hooks?.markCacheHit?.();
+        emitFinalProgress(progressCallback, downloadKey, setKey, files);
+        return getCompanionSetPath(setKey, primaryEntry.targetName);
+    }
+    // Download missing files
+    hooks?.markCacheMiss?.();
+    const overallTotal = files.reduce((sum, f) => sum + f.expectedSize, 0);
+    let overallDownloaded = 0;
+    try {
+        for (let i = 0; i < files.length; i++) {
+            const file = files[i];
+            const filePath = getCompanionSetPath(setKey, file.targetName);
+            if (validatedFiles.has(file.key)) {
+                overallDownloaded += file.expectedSize;
+                continue;
+            }
+            const fileProgressCallback = progressCallback
+                ? (progress) => {
+                    const currentOverall = overallDownloaded + progress.downloaded;
+                    progressCallback({
+                        ...progress,
+                        downloadKey,
+                        fileSetInfo: {
+                            setKey,
+                            currentFile: file.targetName,
+                            fileIndex: i + 1,
+                            totalFiles: files.length,
+                            overallDownloaded: currentOverall,
+                            overallTotal,
+                            overallPercentage: calculatePercentage(currentOverall, overallTotal),
+                        },
+                    });
+                }
+                : undefined;
+            const blobBinding = file.blobCoreKey
+                ? buildBlobBinding(file)
+                : undefined;
+            await downloadSingleFileFromRegistry(file.registryPath, file.registrySource, filePath, file.targetName, downloadKey, file.expectedSize, file.sha256Checksum, fileProgressCallback, signal, blobBinding, hooks);
+            overallDownloaded += file.expectedSize;
+        }
+    }
+    catch (error) {
+        if (error instanceof DownloadCancelledError && shouldClearCache?.()) {
+            try {
+                await fsPromises.rm(getCompanionSetCacheDir(setKey), {
+                    recursive: true,
+                    force: true,
+                });
+            }
+            catch (cleanupError) {
+                logger.debug("Failed to delete companion set cache during cleanup", {
+                    setKey,
+                    error: cleanupError,
+                });
+            }
+        }
+        throw error;
+    }
+    emitFinalProgress(progressCallback, downloadKey, setKey, files);
+    return getCompanionSetPath(setKey, primaryEntry.targetName);
+}
+function emitFinalProgress(progressCallback, downloadKey, setKey, files) {
+    if (!progressCallback)
+        return;
+    const overallTotal = files.reduce((sum, f) => sum + f.expectedSize, 0);
+    const lastFile = files[files.length - 1];
+    progressCallback({
+        type: "modelProgress",
+        downloaded: overallTotal,
+        total: overallTotal,
+        percentage: 100,
+        downloadKey,
+        fileSetInfo: {
+            setKey,
+            currentFile: lastFile.targetName,
+            fileIndex: files.length,
+            totalFiles: files.length,
+            overallDownloaded: overallTotal,
+            overallTotal,
+            overallPercentage: 100,
+        },
+    });
+}
+function validateCompanionSetMetadata(files, primaryKey) {
+    if (!files.some((f) => f.key === primaryKey)) {
+        throw new RegistryDownloadFailedError(`Companion set missing primary file (key: ${primaryKey})`);
+    }
+    const targetNames = new Set();
+    for (const file of files) {
+        if (targetNames.has(file.targetName)) {
+            throw new RegistryDownloadFailedError(`Companion set has duplicate targetName "${file.targetName}"`);
+        }
+        targetNames.add(file.targetName);
+    }
+}
+function isLegacyOnnxSet(files) {
+    return files.some((f) => f.primary === true && f.targetName.endsWith(".onnx"));
+}
+/**
+ * Check if a complete legacy ONNX cache exists for a companion set.
+ * Read-only probe: does not create legacy cache directories.
+ */
+async function checkLegacyOnnxCache(primaryRegistryPath, files, hooks) {
+    const cacheKey = generateShortHash(primaryRegistryPath);
+    let primaryPath = null;
+    for (const file of files) {
+        const filePath = getLegacyOnnxPath(cacheKey, file.targetName);
+        try {
+            const stats = await fsPromises.stat(filePath);
+            if (stats.size !== file.expectedSize)
+                return null;
+            if (file.sha256Checksum && file.sha256Checksum.length === 64) {
+                const checksum = await measureChecksum(filePath, hooks);
+                if (checksum !== file.sha256Checksum)
+                    return null;
+            }
+        }
+        catch {
+            return null;
+        }
+        if (file.primary === true) {
+            primaryPath = filePath;
+        }
+    }
+    return primaryPath;
+}
+function getLegacyOnnxPath(cacheKey, filename) {
+    const baseCache = getModelsCacheDir();
+    return validateAndJoinPath(baseCache, "onnx", cacheKey, filename);
+}
+/**
+ * Check if all companion files exist in the flat single-file cache.
+ */
+async function checkLegacyFlatCache(files, primaryKey, hooks) {
+    let primaryPath = null;
+    for (const file of files) {
+        const flatPath = getSingleFileCachePath(file.registryPath);
+        try {
+            const stats = await fsPromises.stat(flatPath);
+            if (stats.size !== file.expectedSize)
+                return null;
+            if (file.sha256Checksum && file.sha256Checksum.length === 64) {
+                const checksum = await measureChecksum(flatPath, hooks);
+                if (checksum !== file.sha256Checksum)
+                    return null;
+            }
+        }
+        catch {
+            return null;
+        }
+        if (file.key === primaryKey) {
+            primaryPath = flatPath;
+        }
+    }
+    return primaryPath;
+}

package/dist/server/rpc/handlers/load-model/registry-download-utils.d.ts

@@ -0,0 +1,24 @@
+import type { ModelProgressUpdate } from "../../../../schemas/index";
+import type { QVACBlobBinding } from "@qvac/registry-client";
+import type { AbortSignal } from "bare-abort-controller";
+import type { DownloadHooks } from "./types";
+export { DEFAULT_REGISTRY_STREAM_TIMEOUT_MS, buildRegistryClientOptions, } from "./registry-client-options";
+export type { RegistryClientDownloadOptions } from "./registry-client-options";
+export declare function buildBlobBinding(meta: {
+    blobCoreKey: string;
+    blobBlockOffset: number;
+    blobBlockLength: number;
+    blobByteOffset: number;
+    expectedSize: number;
+}): QVACBlobBinding;
+/**
+ * Validate a cached file against expected size and checksum.
+ */
+export declare function validateCachedFile(modelPath: string, modelFileName: string, expectedSize: number, expectedChecksum?: string, hooks?: DownloadHooks): Promise<string | null>;
+/**
+ * Download a single file from the registry to filesystem.
+ * When blobBinding is provided, uses direct blob download (skips metadata sync).
+ * Otherwise falls back to metadata-based downloadModel.
+ */
+export declare function downloadSingleFileFromRegistry(registryPath: string, registrySource: string, modelPath: string, modelFileName: string, downloadKey: string, expectedSize: number, expectedChecksum: string, progressCallback?: (progress: ModelProgressUpdate) => void, signal?: AbortSignal, blobBinding?: QVACBlobBinding, hooks?: DownloadHooks): Promise<void>;
+//# sourceMappingURL=registry-download-utils.d.ts.map

package/dist/server/rpc/handlers/load-model/registry-download-utils.js

@@ -0,0 +1,114 @@
+import { promises as fsPromises } from "bare-fs";
+import path from "bare-path";
+import { measureChecksum, calculatePercentage, } from "../../../utils/index.js";
+import { getRegistryClient } from "../../../bare/registry/registry-client.js";
+import { getSDKConfig } from "../../../bare/registry/config-registry.js";
+import { buildRegistryClientOptions } from "./registry-client-options.js";
+import { ChecksumValidationFailedError, DownloadCancelledError, } from "../../../../utils/errors-server.js";
+import { getServerLogger } from "../../../../logging/index.js";
+const logger = getServerLogger();
+export { DEFAULT_REGISTRY_STREAM_TIMEOUT_MS, buildRegistryClientOptions, } from "./registry-client-options.js";
+export function buildBlobBinding(meta) {
+    return {
+        coreKey: meta.blobCoreKey,
+        blockOffset: meta.blobBlockOffset,
+        blockLength: meta.blobBlockLength,
+        byteOffset: meta.blobByteOffset,
+        byteLength: meta.expectedSize,
+    };
+}
+/**
+ * Validate a cached file against expected size and checksum.
+ */
+export async function validateCachedFile(modelPath, modelFileName, expectedSize, expectedChecksum, hooks) {
+    try {
+        await fsPromises.access(modelPath);
+        const localStats = await fsPromises.stat(modelPath);
+        const localSize = localStats.size;
+        if (localSize === expectedSize) {
+            logger.info(`✅ Model cached with correct size: ${modelPath}`);
+            if (expectedChecksum && expectedChecksum.length === 64) {
+                const checksum = await measureChecksum(modelPath, hooks);
+                if (checksum !== expectedChecksum) {
+                    throw new ChecksumValidationFailedError(`${modelFileName}. Expected: ${expectedChecksum}. Actual: ${checksum}. File may be corrupted`);
+                }
+            }
+            logger.info(`✅ Model already cached and validated: ${modelPath}`);
+            return modelPath;
+        }
+        logger.warn(`🗑️ Removing incomplete cached file (expected ${expectedSize}, got ${localSize}): ${modelPath}`);
+        await fsPromises.unlink(modelPath);
+        return null;
+    }
+    catch (error) {
+        if (error instanceof ChecksumValidationFailedError) {
+            logger.warn(`🗑️ Removing corrupted cached file: ${modelPath}`);
+            await fsPromises.unlink(modelPath).catch(() => { });
+        }
+        return null;
+    }
+}
+/**
+ * Download a single file from the registry to filesystem.
+ * When blobBinding is provided, uses direct blob download (skips metadata sync).
+ * Otherwise falls back to metadata-based downloadModel.
+ */
+export async function downloadSingleFileFromRegistry(registryPath, registrySource, modelPath, modelFileName, downloadKey, expectedSize, expectedChecksum, progressCallback, signal, blobBinding, hooks) {
+    if (signal?.aborted) {
+        throw new DownloadCancelledError();
+    }
+    const client = await getRegistryClient();
+    const dir = path.dirname(modelPath);
+    await fsPromises.mkdir(dir, { recursive: true });
+    const onProgress = progressCallback
+        ? (progress) => {
+            const total = progress.total || expectedSize || progress.downloaded;
+            progressCallback({
+                type: "modelProgress",
+                downloaded: progress.downloaded,
+                total,
+                percentage: total > 0
+                    ? calculatePercentage(progress.downloaded, total)
+                    : 0,
+                downloadKey,
+            });
+        }
+        : undefined;
+    const clientOptions = buildRegistryClientOptions({
+        sdkConfig: getSDKConfig(),
+        outputFile: modelPath,
+        onProgress,
+        signal,
+    });
+    if (blobBinding) {
+        logger.info(`📥 Downloading blob directly: ${modelFileName}`);
+        await client.downloadBlob(blobBinding, clientOptions);
+    }
+    else {
+        logger.info(`📥 Downloading from registry: ${registryPath}`);
+        await client.downloadModel(registryPath, registrySource, clientOptions);
+    }
+    logger.info(`✅ Downloaded to ${modelPath}`);
+    const stats = await fsPromises.stat(modelPath);
+    if (expectedSize && stats.size !== expectedSize) {
+        await fsPromises.unlink(modelPath).catch(() => { });
+        throw new ChecksumValidationFailedError(`${modelFileName}. File size mismatch: expected ${expectedSize}, got ${stats.size}`);
+    }
+    if (expectedChecksum && expectedChecksum.length === 64) {
+        const checksum = await measureChecksum(modelPath, hooks);
+        if (checksum !== expectedChecksum) {
+            await fsPromises.unlink(modelPath);
+            throw new ChecksumValidationFailedError(`${modelFileName}. Expected: ${expectedChecksum}. Actual: ${checksum}`);
+        }
+        logger.info(`✅ Checksum validated for ${modelFileName}`);
+    }
+    if (progressCallback) {
+        progressCallback({
+            type: "modelProgress",
+            downloaded: stats.size,
+            total: stats.size,
+            percentage: 100,
+            downloadKey,
+        });
+    }
+}

package/dist/server/rpc/handlers/load-model/registry.d.ts

@@ -1,22 +1,4 @@
 import type { ModelProgressUpdate } from "../../../../schemas/index";
-import { type RegistryItem } from "../../../../models/registry/index";
-import type { DownloadMetricsHooks } from "./types";
-/**
- * Find companion ONNX data file in registry.
- * ONNX models with external data may use either .onnx_data or .onnx.data.
- */
-export declare function findOnnxCompanionDataFile(registryPath: string): RegistryItem | undefined;
-/**
- * Resolve the main ONNX registry path from a companion data registry path.
- * Supports both .onnx_data and .onnx.data naming conventions.
- */
-export declare function getPairedOnnxRegistryPath(registryPath: string): string | undefined;
-/**
- * Download a model from the QVAC Registry.
- *
- * For known models (present in models.ts), uses direct blob download which
- * skips the registry metadata sync entirely. Falls back to metadata-based
- * download for unknown models.
- */
-export declare function downloadModelFromRegistry(registryPath: string, registrySource: string, progressCallback?: (progress: ModelProgressUpdate) => void, expectedChecksum?: string, hooks?: DownloadMetricsHooks): Promise<string>;
+import type { DownloadHooks } from "./types";
+export declare function downloadModelFromRegistry(registryPath: string, registrySource: string, progressCallback?: (progress: ModelProgressUpdate) => void, expectedChecksum?: string, hooks?: DownloadHooks): Promise<string>;
 //# sourceMappingURL=registry.d.ts.map