@qvac/sdk 0.6.1 → 0.7.0

package/dist/models/update-models/processing.d.ts

@@ -0,0 +1,8 @@
+import type { QVACModelEntry } from "@qvac/registry-client";
+import type { ProcessedModel } from "./types";
+export declare function toHexString(value: Buffer | string | {
+    data: number[];
+} | undefined): string;
+export declare function extractModelName(registryPath: string): string;
+export declare function processRegistryModel(model: QVACModelEntry): ProcessedModel | null;
+//# sourceMappingURL=processing.d.ts.map

package/dist/models/update-models/processing.js

@@ -0,0 +1,67 @@
+import { getAddonFromEngine, resolveCanonicalEngine, } from "../../schemas/engine-addon-map";
+import { detectShardedModel } from "./shards.js";
+export function toHexString(value) {
+    if (!value)
+        return "";
+    if (Buffer.isBuffer(value))
+        return value.toString("hex");
+    if (typeof value === "string")
+        return value;
+    if (typeof value === "object" && "data" in value) {
+        return Buffer.from(value.data).toString("hex");
+    }
+    return "";
+}
+export function extractModelName(registryPath) {
+    const parts = registryPath.split("/");
+    if (parts.length >= 2) {
+        return parts[1] || parts[0] || "";
+    }
+    return (registryPath
+        .split("/")
+        .pop()
+        ?.replace(/\.\w+$/, "") || "");
+}
+export function processRegistryModel(model) {
+    const engine = resolveCanonicalEngine(model.engine);
+    if (!engine) {
+        console.warn(`⚠️  Skipping model with unknown engine "${model.engine}": ${model.path}`);
+        return null;
+    }
+    const filename = model.path.split("/").pop() || model.path;
+    const blobBinding = model.blobBinding;
+    const blobCoreKey = toHexString(blobBinding?.coreKey);
+    const blobBlockOffset = blobBinding?.blockOffset ?? 0;
+    const blobBlockLength = blobBinding?.blockLength ?? 0;
+    const blobByteOffset = blobBinding?.byteOffset ?? 0;
+    const expectedSize = blobBinding?.byteLength ?? 0;
+    // The sha256 lives on blobBinding at runtime (per the hyperschema),
+    // even though the TS types define it on QVACModelEntry. Try blobBinding first.
+    const sha256Checksum = blobBinding?.["sha256"] ||
+        model.sha256 ||
+        "";
+    const addon = getAddonFromEngine(engine);
+    const result = {
+        registryPath: model.path,
+        registrySource: model.source,
+        blobCoreKey,
+        blobBlockOffset,
+        blobBlockLength,
+        blobByteOffset,
+        modelId: filename,
+        addon,
+        expectedSize,
+        sha256Checksum,
+        engine,
+        modelName: extractModelName(model.path),
+        quantization: model.quantization || "",
+        params: model.params || "",
+        tags: model.tags || [],
+    };
+    const shardDetection = detectShardedModel(filename);
+    if (shardDetection.isSharded) {
+        result.isShardPart = true;
+        result.shardInfo = shardDetection;
+    }
+    return result;
+}

package/dist/models/update-models/registry.d.ts

@@ -0,0 +1,4 @@
+import type { CollectOptions, ProcessedModel } from "./types";
+export { processRegistryModel, extractModelName, toHexString, } from "./processing";
+export declare function collectModels(options?: CollectOptions): Promise<ProcessedModel[]>;
+//# sourceMappingURL=registry.d.ts.map

package/dist/models/update-models/registry.js

@@ -0,0 +1,67 @@
+import { QVACRegistryClient } from "@qvac/registry-client";
+import { groupShardedModels } from "./shards.js";
+import { processRegistryModel } from "./processing.js";
+import { DEFAULT_REGISTRY_CORE_KEY } from "../../constants/index.js";
+// Re-export for backward compat
+export { processRegistryModel, extractModelName, toHexString, } from "./processing.js";
+export async function collectModels(options = {}) {
+    const { showDuplicates = false, noDedup = false } = options;
+    const models = [];
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+    const registryCoreKey = process.env["QVAC_REGISTRY_CORE_KEY"] ?? DEFAULT_REGISTRY_CORE_KEY;
+    const client = new QVACRegistryClient({ registryCoreKey });
+    try {
+        await client.ready();
+        const registryModels = await client.findModels({});
+        console.log(`📦 Found ${registryModels.length} entries in registry`);
+        for (const registryModel of registryModels) {
+            const processed = processRegistryModel(registryModel);
+            if (processed)
+                models.push(processed);
+        }
+    }
+    finally {
+        await client.close();
+    }
+    const groupedModels = groupShardedModels(models);
+    if (noDedup) {
+        console.log(`\n⏭️  Skipping deduplication (--no-dedup flag set)`);
+        return groupedModels;
+    }
+    return deduplicateModels(groupedModels, showDuplicates);
+}
+function deduplicateModels(models, showDuplicates) {
+    const seenChecksums = new Map();
+    const dedupedModels = [];
+    const skipped = [];
+    for (const model of models) {
+        if (!model.sha256Checksum || model.sha256Checksum === "") {
+            dedupedModels.push(model);
+            continue;
+        }
+        if (seenChecksums.has(model.sha256Checksum)) {
+            skipped.push({
+                name: model.registryPath,
+                checksum: model.sha256Checksum,
+                reason: `Duplicate of ${seenChecksums.get(model.sha256Checksum)}`,
+            });
+            continue;
+        }
+        seenChecksums.set(model.sha256Checksum, model.registryPath);
+        dedupedModels.push(model);
+    }
+    if (skipped.length > 0) {
+        console.log(`\n🧹 Removed ${skipped.length} duplicate model(s)`);
+        if (showDuplicates) {
+            skipped.forEach(({ name, checksum, reason }) => {
+                console.log(`  - ${name}`);
+                console.log(`    Checksum: ${checksum}`);
+                console.log(`    ${reason}`);
+            });
+        }
+        else {
+            console.log(`   Use --show-duplicates to see details`);
+        }
+    }
+    return dedupedModels;
+}

package/dist/models/update-models/shards.d.ts

@@ -0,0 +1,4 @@
+import type { ProcessedModel, ShardDetection } from "./types";
+export declare function detectShardedModel(filename: string): ShardDetection;
+export declare function groupShardedModels(models: ProcessedModel[]): ProcessedModel[];
+//# sourceMappingURL=shards.d.ts.map

package/dist/models/update-models/shards.js

@@ -0,0 +1,57 @@
+export function detectShardedModel(filename) {
+    const shardPattern = /^(.+)-(\d{5})-of-(\d{5})(\.\w+)$/;
+    const match = filename.match(shardPattern);
+    if (match) {
+        return {
+            isSharded: true,
+            baseFilename: match[1],
+            currentShard: parseInt(match[2], 10),
+            totalShards: parseInt(match[3], 10),
+            extension: match[4],
+        };
+    }
+    return { isSharded: false };
+}
+export function groupShardedModels(models) {
+    const shardGroups = new Map();
+    const nonShardedModels = [];
+    for (const model of models) {
+        if (model.isShardPart && model.shardInfo) {
+            const baseKey = `${model.registrySource}:${model.shardInfo.baseFilename}`;
+            if (!shardGroups.has(baseKey)) {
+                shardGroups.set(baseKey, []);
+            }
+            shardGroups.get(baseKey).push(model);
+        }
+        else {
+            nonShardedModels.push(model);
+        }
+    }
+    const processedShards = [];
+    for (const [baseKey, shards] of shardGroups) {
+        shards.sort((a, b) => (a.shardInfo?.currentShard ?? 0) - (b.shardInfo?.currentShard ?? 0));
+        const firstShard = shards[0];
+        const totalExpectedShards = firstShard.shardInfo?.totalShards ?? 0;
+        if (shards.length !== totalExpectedShards) {
+            console.warn(`⚠️  Expected ${totalExpectedShards} shards but found ${shards.length} for ${baseKey}`);
+        }
+        const totalSize = shards.reduce((sum, s) => sum + s.expectedSize, 0);
+        const shardMetadata = shards.map((s) => ({
+            filename: s.modelId,
+            expectedSize: s.expectedSize,
+            sha256Checksum: s.sha256Checksum,
+            blobCoreKey: s.blobCoreKey,
+            blobBlockOffset: s.blobBlockOffset,
+            blobBlockLength: s.blobBlockLength,
+            blobByteOffset: s.blobByteOffset,
+        }));
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const { isShardPart: _shard, shardInfo: _info, ...rest } = firstShard;
+        processedShards.push({
+            ...rest,
+            expectedSize: totalSize,
+            shardMetadata,
+        });
+    }
+    return [...nonShardedModels, ...processedShards];
+}

package/dist/models/update-models/types.d.ts

@@ -0,0 +1,60 @@
+import type { ModelRegistryEntryAddon, ModelRegistryEngine } from "../../schemas/registry";
+export interface ShardInfo {
+    isSharded: true;
+    baseFilename: string;
+    currentShard: number;
+    totalShards: number;
+    extension: string;
+}
+export interface NotSharded {
+    isSharded: false;
+}
+export type ShardDetection = ShardInfo | NotSharded;
+export interface ShardMetadataEntry {
+    filename: string;
+    expectedSize: number;
+    sha256Checksum: string;
+    blobCoreKey: string;
+    blobBlockOffset: number;
+    blobBlockLength: number;
+    blobByteOffset: number;
+}
+export interface ProcessedModel {
+    registryPath: string;
+    registrySource: string;
+    blobCoreKey: string;
+    blobBlockOffset: number;
+    blobBlockLength: number;
+    blobByteOffset: number;
+    modelId: string;
+    addon: ModelRegistryEntryAddon;
+    expectedSize: number;
+    sha256Checksum: string;
+    engine: ModelRegistryEngine;
+    modelName: string;
+    quantization: string;
+    params: string;
+    tags: string[];
+    isShardPart?: boolean;
+    shardInfo?: ShardInfo;
+    shardMetadata?: ShardMetadataEntry[];
+    name?: string;
+}
+export interface CurrentModel {
+    name: string;
+    registryPath: string;
+}
+export interface CollectOptions {
+    showDuplicates?: boolean;
+    noDedup?: boolean;
+}
+export interface ExportNameInput {
+    path: string;
+    engine: ModelRegistryEngine;
+    name: string;
+    quantization: string;
+    params: string;
+    tags: string[];
+    usedNames: Set<string>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/models/update-models/types.js

@@ -0,0 +1 @@
+export {};

package/dist/models/update-models/utils.d.ts

@@ -0,0 +1,3 @@
+export declare function formatSize(bytes: number): string;
+export declare function getCommitHash(short?: boolean): string;
+//# sourceMappingURL=utils.d.ts.map

package/dist/models/update-models/utils.js

@@ -0,0 +1,21 @@
+import { execSync } from "child_process";
+export function formatSize(bytes) {
+    if (bytes < 1024)
+        return `${bytes}B`;
+    if (bytes < 1024 * 1024)
+        return `${(bytes / 1024).toFixed(1)}KB`;
+    if (bytes < 1024 * 1024 * 1024)
+        return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+    return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
+}
+export function getCommitHash(short = false) {
+    try {
+        const cmd = short ? "git rev-parse --short HEAD" : "git rev-parse HEAD";
+        return execSync(cmd, { encoding: "utf-8" }).trim();
+    }
+    catch (error) {
+        throw new Error("Git is required to generate history file", {
+            cause: error,
+        });
+    }
+}

package/dist/pear/pre.d.ts

@@ -0,0 +1,18 @@
+/**
+ * QVAC Pear Pre-Hook
+ *
+ * Invoked by Pear before `pear run` or `pear stage` to:
+ * 1. Generate `qvac/worker.pear.entry.mjs` with selected plugins
+ * 2. Persist `pear.stage.entrypoints` to package.json for routing
+ *
+ * @example package.json configuration
+ * ```json
+ * {
+ *   "pear": {
+ *     "pre": ["@qvac/sdk/pear-pre"]
+ *   }
+ * }
+ * ```
+ */
+export {};
+//# sourceMappingURL=pre.d.ts.map