@qvac/sdk 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -24
- package/NOTICE +632 -0
- package/README.md +2 -1
- package/dist/client/api/embed.js +1 -4
- package/dist/client/api/get-model-info.d.ts +10 -3
- package/dist/client/api/index.d.ts +2 -0
- package/dist/client/api/index.js +2 -0
- package/dist/client/api/invoke-plugin.d.ts +14 -0
- package/dist/client/api/invoke-plugin.js +38 -0
- package/dist/client/api/registry.d.ts +14 -0
- package/dist/client/api/registry.js +37 -0
- package/dist/client/api/transcribe.js +0 -4
- package/dist/client/api/unload-model.js +1 -1
- package/dist/client/config-loader/resolve-config.node.d.ts +3 -2
- package/dist/client/config-loader/resolve-config.node.js +31 -4
- package/dist/client/init-hooks.js +2 -1
- package/dist/client/rpc/bare-client.d.ts +2 -0
- package/dist/client/rpc/bare-client.js +48 -7
- package/dist/client/rpc/expo-device-info.d.ts +6 -0
- package/dist/client/rpc/expo-device-info.js +32 -0
- package/dist/client/rpc/expo-rpc-client.js +9 -32
- package/dist/client/rpc/node-rpc-client.d.ts +1 -1
- package/dist/client/rpc/node-rpc-client.js +201 -28
- package/dist/client/rpc/rpc-client.d.ts +1 -1
- package/dist/client/rpc/rpc-client.js +2 -2
- package/dist/constants/index.d.ts +1 -0
- package/dist/constants/index.js +1 -0
- package/dist/constants/registry.d.ts +2 -0
- package/dist/constants/registry.js +1 -0
- package/dist/examples/cache-management.js +2 -2
- package/dist/examples/config/plugins/plugins.config.d.ts +5 -0
- package/dist/examples/config/plugins/plugins.config.js +4 -0
- package/dist/examples/download-with-blind-relays.js +1 -1
- package/dist/examples/llamacpp-multimodal.js +3 -3
- package/dist/examples/llamacpp-native-tools.js +2 -2
- package/dist/examples/mcp-websearch.js +2 -2
- package/dist/examples/multi-model-demo.js +2 -2
- package/dist/examples/ocr-fasttext.js +4 -4
- package/dist/examples/parallel-download.js +2 -2
- package/dist/examples/plugins.d.ts +14 -0
- package/dist/examples/plugins.js +129 -0
- package/dist/examples/registry-query.d.ts +2 -0
- package/dist/examples/registry-query.js +87 -0
- package/dist/examples/translation/translation-bergamot-batch.js +2 -2
- package/dist/examples/translation/translation-bergamot.js +4 -4
- package/dist/examples/translation/translation-opus.js +2 -2
- package/dist/examples/translation/translation-stream.js +2 -2
- package/dist/examples/tts/chatterbox.d.ts +2 -0
- package/dist/examples/tts/chatterbox.js +59 -0
- package/dist/examples/tts/supertonic.d.ts +2 -0
- package/dist/examples/tts/supertonic.js +51 -0
- package/dist/examples/tts/utils.d.ts +17 -0
- package/dist/examples/tts/utils.js +93 -0
- package/dist/examples/whispercpp-filesystem.js +2 -3
- package/dist/examples/whispercpp-microphone-record.js +1 -2
- package/dist/examples/whispercpp-prompt.js +2 -3
- package/dist/expo/plugins/index.d.ts +3 -0
- package/dist/expo/plugins/index.js +3 -0
- package/dist/expo/plugins/resolve-sdk-package-dir.d.ts +15 -0
- package/dist/expo/plugins/resolve-sdk-package-dir.js +31 -0
- package/dist/expo/plugins/withAndroidArchitecture.d.ts +8 -0
- package/dist/expo/plugins/withAndroidArchitecture.js +52 -0
- package/dist/expo/plugins/withAndroidNdkVersion.d.ts +8 -0
- package/dist/expo/plugins/withAndroidNdkVersion.js +31 -0
- package/dist/expo/plugins/withDeviceInfo.d.ts +10 -0
- package/dist/expo/plugins/withDeviceInfo.js +48 -0
- package/dist/expo/plugins/withMobileBundle.d.ts +5 -2
- package/dist/expo/plugins/withMobileBundle.js +121 -65
- package/dist/expo/plugins/withQvacSDK.d.ts +4 -0
- package/dist/expo/plugins/withQvacSDK.js +26 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +5 -5
- package/dist/models/{hyperdrive → registry}/index.js +1 -1
- package/dist/models/registry/models.d.ts +9056 -0
- package/dist/models/registry/models.js +9351 -0
- package/dist/models/update-models/codegen.d.ts +3 -0
- package/dist/models/update-models/codegen.js +135 -0
- package/dist/models/update-models/history.d.ts +13 -0
- package/dist/models/update-models/history.js +88 -0
- package/dist/models/update-models/index.d.ts +2 -0
- package/dist/models/update-models/index.js +124 -0
- package/dist/models/update-models/naming.d.ts +4 -0
- package/dist/models/update-models/naming.js +314 -0
- package/dist/models/update-models/processing.d.ts +8 -0
- package/dist/models/update-models/processing.js +67 -0
- package/dist/models/update-models/registry.d.ts +4 -0
- package/dist/models/update-models/registry.js +67 -0
- package/dist/models/update-models/shards.d.ts +4 -0
- package/dist/models/update-models/shards.js +57 -0
- package/dist/models/update-models/types.d.ts +60 -0
- package/dist/models/update-models/types.js +1 -0
- package/dist/models/update-models/utils.d.ts +3 -0
- package/dist/models/update-models/utils.js +21 -0
- package/dist/pear/pre.d.ts +18 -0
- package/dist/pear/pre.js +433 -0
- package/dist/schemas/common.d.ts +286 -32
- package/dist/schemas/common.js +12 -0
- package/dist/schemas/download-asset.d.ts +35 -11
- package/dist/schemas/download-asset.js +5 -1
- package/dist/schemas/engine-addon-map.d.ts +5 -0
- package/dist/schemas/engine-addon-map.js +49 -0
- package/dist/schemas/get-model-info.d.ts +32 -16
- package/dist/schemas/get-model-info.js +17 -8
- package/dist/schemas/index.d.ts +4 -1
- package/dist/schemas/index.js +4 -4
- package/dist/schemas/load-model.d.ts +2224 -900
- package/dist/schemas/load-model.js +130 -25
- package/dist/schemas/model-src-utils.d.ts +30 -12
- package/dist/schemas/model-src-utils.js +5 -2
- package/dist/schemas/model-types.d.ts +15 -4
- package/dist/schemas/model-types.js +21 -4
- package/dist/schemas/path-validation.d.ts +10 -0
- package/dist/schemas/path-validation.js +22 -0
- package/dist/schemas/plugin.d.ts +180 -0
- package/dist/schemas/plugin.js +132 -0
- package/dist/schemas/rag.d.ts +2 -2
- package/dist/schemas/rag.js +7 -3
- package/dist/schemas/registry.d.ts +338 -0
- package/dist/schemas/registry.js +91 -0
- package/dist/schemas/sdk-errors-client.d.ts +6 -6
- package/dist/schemas/sdk-errors-client.js +15 -15
- package/dist/schemas/sdk-errors-registry.d.ts +6 -0
- package/dist/schemas/sdk-errors-registry.js +8 -0
- package/dist/schemas/sdk-errors-server.d.ts +16 -2
- package/dist/schemas/sdk-errors-server.js +90 -10
- package/dist/schemas/text-to-speech.d.ts +670 -1
- package/dist/schemas/text-to-speech.js +51 -2
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/scripts/run-security-tests-bare.d.ts +2 -0
- package/dist/scripts/run-security-tests-bare.js +78 -0
- package/dist/scripts/run-security-tests.d.ts +2 -0
- package/dist/scripts/run-security-tests.js +22 -0
- package/dist/server/bare/delegate-rpc-client.d.ts +6 -0
- package/dist/server/bare/delegate-rpc-client.js +136 -56
- package/dist/server/bare/{addons/llamacpp-embedding/handlers → ops}/embed.d.ts +1 -1
- package/dist/server/bare/{addons/llamacpp-embedding/handlers → ops}/embed.js +3 -3
- package/dist/server/bare/{addons/llamacpp-completion → ops}/kv-cache-utils.js +6 -6
- package/dist/server/bare/{addons → ops}/load-model.js +32 -42
- package/dist/server/bare/{addons/nmtcpp-translation/handlers → ops}/translate.d.ts +1 -1
- package/dist/server/bare/{addons/nmtcpp-translation/handlers → ops}/translate.js +5 -4
- package/dist/server/bare/plugins/index.d.ts +7 -0
- package/dist/server/bare/plugins/index.js +6 -0
- package/dist/server/bare/{addons/llamacpp-completion → plugins/llamacpp-completion/ops}/cache-logger.js +1 -1
- package/dist/server/bare/{addons/llamacpp-completion/handlers → plugins/llamacpp-completion/ops}/completion-stream.d.ts +1 -1
- package/dist/server/bare/{addons/llamacpp-completion/handlers → plugins/llamacpp-completion/ops}/completion-stream.js +2 -3
- package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts +115 -0
- package/dist/server/bare/plugins/llamacpp-completion/plugin.js +142 -0
- package/dist/server/bare/plugins/llamacpp-embedding/plugin.d.ts +24 -0
- package/dist/server/bare/plugins/llamacpp-embedding/plugin.js +73 -0
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.d.ts +45 -0
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.js +97 -0
- package/dist/server/bare/{addons/onnx-ocr/handlers → plugins/onnx-ocr/ops}/ocr-stream.js +2 -1
- package/dist/server/bare/plugins/onnx-ocr/plugin.d.ts +38 -0
- package/dist/server/bare/{addons/onnx-ocr/create-model.js → plugins/onnx-ocr/plugin.js} +39 -2
- package/dist/server/bare/{addons/onnx-tts/handlers → plugins/onnx-tts/ops}/text-to-speech.js +23 -1
- package/dist/server/bare/plugins/onnx-tts/plugin.d.ts +26 -0
- package/dist/server/bare/plugins/onnx-tts/plugin.js +162 -0
- package/dist/server/bare/plugins/onnx-tts/wav-helper.d.ts +14 -0
- package/dist/server/bare/plugins/onnx-tts/wav-helper.js +107 -0
- package/dist/server/bare/plugins/whispercpp-transcription/ops/transcribe-stream.d.ts +3 -0
- package/dist/server/bare/{addons/whispercpp-transcription/handlers → plugins/whispercpp-transcription/ops}/transcribe-stream.js +1 -2
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.d.ts +31 -0
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.js +74 -0
- package/dist/server/bare/rag-hyperdb/handlers/ingest.js +1 -1
- package/dist/server/bare/rag-hyperdb/handlers/search.js +1 -1
- package/dist/server/bare/rag-hyperdb/rag-workspace-manager.js +2 -1
- package/dist/server/bare/registry/config-registry.js +1 -1
- package/dist/server/bare/registry/registry-client.d.ts +5 -0
- package/dist/server/bare/registry/registry-client.js +37 -0
- package/dist/server/env.d.ts +22 -0
- package/dist/server/env.js +57 -0
- package/dist/server/plugins/index.d.ts +2 -0
- package/dist/server/plugins/index.js +1 -0
- package/dist/server/plugins/registry.d.ts +9 -0
- package/dist/server/plugins/registry.js +75 -0
- package/dist/server/rpc/delegate-transport.d.ts +11 -0
- package/dist/server/rpc/delegate-transport.js +60 -0
- package/dist/server/rpc/handle-request.d.ts +0 -9
- package/dist/server/rpc/handle-request.js +13 -321
- package/dist/server/rpc/handler-registry.d.ts +3 -0
- package/dist/server/rpc/handler-registry.js +84 -0
- package/dist/server/rpc/handler-utils.d.ts +22 -0
- package/dist/server/rpc/handler-utils.js +87 -0
- package/dist/server/rpc/handlers/cancelHandler.js +1 -1
- package/dist/server/rpc/handlers/completion-stream-delegated.js +1 -1
- package/dist/server/rpc/handlers/completion-stream.js +2 -43
- package/dist/server/rpc/handlers/delete-cache.js +1 -1
- package/dist/server/rpc/handlers/embed.js +2 -21
- package/dist/server/rpc/handlers/get-model-info.js +19 -15
- package/dist/server/rpc/handlers/index.d.ts +9 -0
- package/dist/server/rpc/handlers/index.js +9 -0
- package/dist/server/rpc/handlers/load-model/download-manager.d.ts +1 -0
- package/dist/server/rpc/handlers/load-model/download-manager.js +3 -0
- package/dist/server/rpc/handlers/load-model/handler.js +47 -24
- package/dist/server/rpc/handlers/load-model/http.js +3 -2
- package/dist/server/rpc/handlers/load-model/hyperdrive.js +14 -10
- package/dist/server/rpc/handlers/load-model/registry.d.ts +10 -0
- package/dist/server/rpc/handlers/load-model/registry.js +571 -0
- package/dist/server/rpc/handlers/load-model/resolve.js +18 -5
- package/dist/server/rpc/handlers/load-model-delegated.js +4 -2
- package/dist/server/rpc/handlers/ocr-stream.js +2 -37
- package/dist/server/rpc/handlers/ping.d.ts +3 -0
- package/dist/server/rpc/handlers/ping.js +3 -0
- package/dist/server/rpc/handlers/plugin-dispatch.d.ts +15 -0
- package/dist/server/rpc/handlers/plugin-dispatch.js +54 -0
- package/dist/server/rpc/handlers/plugin-invoke.d.ts +4 -0
- package/dist/server/rpc/handlers/plugin-invoke.js +87 -0
- package/dist/server/rpc/handlers/registry.d.ts +5 -0
- package/dist/server/rpc/handlers/registry.js +157 -0
- package/dist/server/rpc/handlers/text-to-speech.js +2 -5
- package/dist/server/rpc/handlers/transcribe-stream.js +2 -30
- package/dist/server/rpc/handlers/translate.js +2 -42
- package/dist/server/rpc/handlers/unload-model.js +1 -1
- package/dist/server/rpc/index.d.ts +1 -1
- package/dist/server/utils/archive.d.ts +1 -0
- package/dist/server/utils/archive.js +10 -2
- package/dist/server/utils/cache.d.ts +5 -0
- package/dist/server/utils/cache.js +26 -2
- package/dist/server/utils/gguf-tensor-extractor.js +3 -3
- package/dist/server/utils/index.d.ts +1 -1
- package/dist/server/utils/index.js +1 -1
- package/dist/server/utils/path-security.d.ts +12 -0
- package/dist/server/utils/path-security.js +25 -0
- package/dist/server/utils/shard-utils.js +3 -3
- package/dist/server/worker-core.d.ts +6 -0
- package/dist/server/worker-core.js +88 -0
- package/dist/server/worker.d.ts +4 -5
- package/dist/server/worker.js +18 -116
- package/dist/utils/errors-client.d.ts +9 -6
- package/dist/utils/errors-client.js +19 -11
- package/dist/utils/errors-server.d.ts +44 -5
- package/dist/utils/errors-server.js +82 -9
- package/dist/utils/path-sanitize.d.ts +17 -0
- package/dist/utils/path-sanitize.js +54 -0
- package/expo/plugins/patches/android-link.mjs +63 -0
- package/expo/plugins/patches/ios-link.mjs +63 -0
- package/package.json +72 -10
- package/dist/examples/text-to-speech.d.ts +0 -3
- package/dist/examples/text-to-speech.js +0 -137
- package/dist/models/hyperdrive/models.d.ts +0 -4003
- package/dist/models/hyperdrive/models.js +0 -4282
- package/dist/server/bare/addons/index.d.ts +0 -4
- package/dist/server/bare/addons/index.js +0 -3
- package/dist/server/bare/addons/llamacpp-completion/create-model.d.ts +0 -8
- package/dist/server/bare/addons/llamacpp-completion/create-model.js +0 -43
- package/dist/server/bare/addons/llamacpp-completion/index.d.ts +0 -4
- package/dist/server/bare/addons/llamacpp-completion/index.js +0 -3
- package/dist/server/bare/addons/llamacpp-embedding/create-model.d.ts +0 -8
- package/dist/server/bare/addons/llamacpp-embedding/create-model.js +0 -38
- package/dist/server/bare/addons/llamacpp-embedding/index.d.ts +0 -3
- package/dist/server/bare/addons/llamacpp-embedding/index.js +0 -2
- package/dist/server/bare/addons/nmtcpp-translation/create-model.d.ts +0 -8
- package/dist/server/bare/addons/nmtcpp-translation/create-model.js +0 -52
- package/dist/server/bare/addons/nmtcpp-translation/index.d.ts +0 -3
- package/dist/server/bare/addons/nmtcpp-translation/index.js +0 -2
- package/dist/server/bare/addons/onnx-ocr/create-model.d.ts +0 -10
- package/dist/server/bare/addons/onnx-ocr/index.d.ts +0 -3
- package/dist/server/bare/addons/onnx-ocr/index.js +0 -2
- package/dist/server/bare/addons/onnx-tts/create-model.d.ts +0 -10
- package/dist/server/bare/addons/onnx-tts/create-model.js +0 -25
- package/dist/server/bare/addons/onnx-tts/index.d.ts +0 -3
- package/dist/server/bare/addons/onnx-tts/index.js +0 -2
- package/dist/server/bare/addons/whispercpp-transcription/create-model.d.ts +0 -10
- package/dist/server/bare/addons/whispercpp-transcription/create-model.js +0 -37
- package/dist/server/bare/addons/whispercpp-transcription/handlers/transcribe-stream.d.ts +0 -3
- package/dist/server/bare/addons/whispercpp-transcription/index.d.ts +0 -3
- package/dist/server/bare/addons/whispercpp-transcription/index.js +0 -2
- /package/dist/models/{hyperdrive → registry}/index.d.ts +0 -0
- /package/dist/server/bare/{addons → ops}/cancel.d.ts +0 -0
- /package/dist/server/bare/{addons → ops}/cancel.js +0 -0
- /package/dist/server/bare/{addons/llamacpp-completion → ops}/kv-cache-utils.d.ts +0 -0
- /package/dist/server/bare/{addons → ops}/load-model.d.ts +0 -0
- /package/dist/server/bare/{addons → ops}/unload-model.d.ts +0 -0
- /package/dist/server/bare/{addons → ops}/unload-model.js +0 -0
- /package/dist/server/bare/{addons/llamacpp-completion → plugins/llamacpp-completion/ops}/cache-logger.d.ts +0 -0
- /package/dist/server/bare/{addons/onnx-ocr/handlers → plugins/onnx-ocr/ops}/ocr-stream.d.ts +0 -0
- /package/dist/server/bare/{addons/onnx-tts/handlers → plugins/onnx-tts/ops}/text-to-speech.d.ts +0 -0
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { generateExportName } from "./naming.js";
|
|
2
|
+
export function generateModelsFileContent(models) {
|
|
3
|
+
const usedNames = new Set();
|
|
4
|
+
const modelsWithNames = models.map((m) => ({
|
|
5
|
+
...m,
|
|
6
|
+
name: generateExportName({
|
|
7
|
+
path: m.registryPath,
|
|
8
|
+
engine: m.engine,
|
|
9
|
+
name: m.modelName,
|
|
10
|
+
quantization: m.quantization,
|
|
11
|
+
params: m.params,
|
|
12
|
+
tags: m.tags,
|
|
13
|
+
usedNames,
|
|
14
|
+
}),
|
|
15
|
+
}));
|
|
16
|
+
return generateFileContentWithNames(modelsWithNames);
|
|
17
|
+
}
|
|
18
|
+
function generateFileContentWithNames(modelsWithNames) {
|
|
19
|
+
const entries = modelsWithNames
|
|
20
|
+
.map((m) => {
|
|
21
|
+
const addonAlias = m.addon;
|
|
22
|
+
let entry = ` {
|
|
23
|
+
name: "${m.name}",
|
|
24
|
+
registryPath: "${m.registryPath}",
|
|
25
|
+
registrySource: "${m.registrySource}",
|
|
26
|
+
blobCoreKey: "${m.blobCoreKey}",
|
|
27
|
+
blobBlockOffset: ${m.blobBlockOffset},
|
|
28
|
+
blobBlockLength: ${m.blobBlockLength},
|
|
29
|
+
blobByteOffset: ${m.blobByteOffset},
|
|
30
|
+
modelId: "${m.modelId}",
|
|
31
|
+
addon: "${addonAlias}",
|
|
32
|
+
expectedSize: ${m.expectedSize},
|
|
33
|
+
sha256Checksum: "${m.sha256Checksum}",
|
|
34
|
+
engine: "${m.engine}",
|
|
35
|
+
quantization: "${m.quantization || ""}",
|
|
36
|
+
params: "${m.params || ""}"`;
|
|
37
|
+
if (m.shardMetadata) {
|
|
38
|
+
entry += `,\n shardMetadata: ${JSON.stringify(m.shardMetadata)}`;
|
|
39
|
+
}
|
|
40
|
+
entry += "\n }";
|
|
41
|
+
return entry;
|
|
42
|
+
})
|
|
43
|
+
.join(",\n");
|
|
44
|
+
const exports = modelsWithNames
|
|
45
|
+
.map((m, i) => {
|
|
46
|
+
return `export const ${m.name} = {
|
|
47
|
+
name: "${m.name}",
|
|
48
|
+
src: \`registry://\${models[${i}].registrySource}/\${models[${i}].registryPath}\`,
|
|
49
|
+
registryPath: models[${i}].registryPath,
|
|
50
|
+
registrySource: models[${i}].registrySource,
|
|
51
|
+
blobCoreKey: models[${i}].blobCoreKey,
|
|
52
|
+
blobBlockOffset: models[${i}].blobBlockOffset,
|
|
53
|
+
blobBlockLength: models[${i}].blobBlockLength,
|
|
54
|
+
blobByteOffset: models[${i}].blobByteOffset,
|
|
55
|
+
modelId: models[${i}].modelId,
|
|
56
|
+
expectedSize: models[${i}].expectedSize,
|
|
57
|
+
sha256Checksum: models[${i}].sha256Checksum,
|
|
58
|
+
addon: models[${i}].addon,
|
|
59
|
+
engine: models[${i}].engine,
|
|
60
|
+
quantization: models[${i}].quantization,
|
|
61
|
+
params: models[${i}].params,
|
|
62
|
+
} as const;`;
|
|
63
|
+
})
|
|
64
|
+
.join("\n\n");
|
|
65
|
+
return `// THIS FILE IS AUTO-GENERATED BY models/update-models
|
|
66
|
+
// DO NOT MODIFY MANUALLY
|
|
67
|
+
|
|
68
|
+
export type RegistryItem = {
|
|
69
|
+
name: string;
|
|
70
|
+
registryPath: string;
|
|
71
|
+
registrySource: string;
|
|
72
|
+
blobCoreKey: string;
|
|
73
|
+
blobBlockOffset: number;
|
|
74
|
+
blobBlockLength: number;
|
|
75
|
+
blobByteOffset: number;
|
|
76
|
+
modelId: string;
|
|
77
|
+
addon: "llm" | "whisper" | "embeddings" | "nmt" | "vad" | "tts" | "ocr" | "other";
|
|
78
|
+
expectedSize: number;
|
|
79
|
+
sha256Checksum: string;
|
|
80
|
+
engine: string;
|
|
81
|
+
quantization: string;
|
|
82
|
+
params: string;
|
|
83
|
+
shardMetadata?: readonly {
|
|
84
|
+
filename: string;
|
|
85
|
+
expectedSize: number;
|
|
86
|
+
sha256Checksum: string;
|
|
87
|
+
blobCoreKey: string;
|
|
88
|
+
blobBlockOffset: number;
|
|
89
|
+
blobBlockLength: number;
|
|
90
|
+
blobByteOffset: number;
|
|
91
|
+
}[];
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
export type ModelConstant = {
|
|
95
|
+
name: string;
|
|
96
|
+
src: string;
|
|
97
|
+
registryPath: string;
|
|
98
|
+
registrySource: string;
|
|
99
|
+
blobCoreKey: string;
|
|
100
|
+
blobBlockOffset: number;
|
|
101
|
+
blobBlockLength: number;
|
|
102
|
+
blobByteOffset: number;
|
|
103
|
+
modelId: string;
|
|
104
|
+
expectedSize: number;
|
|
105
|
+
sha256Checksum: string;
|
|
106
|
+
addon: "llm" | "whisper" | "embeddings" | "nmt" | "vad" | "tts" | "ocr" | "other";
|
|
107
|
+
engine: string;
|
|
108
|
+
quantization: string;
|
|
109
|
+
params: string;
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
export const models = [
|
|
113
|
+
${entries}
|
|
114
|
+
] as const satisfies readonly RegistryItem[];
|
|
115
|
+
|
|
116
|
+
// Individual model constants for direct import/use with loadModel
|
|
117
|
+
// These contain all metadata and can be used directly: loadModel({ modelSrc: WHISPER_TINY, ... })
|
|
118
|
+
${exports}
|
|
119
|
+
|
|
120
|
+
// Helper function to get model by name
|
|
121
|
+
export function getModelByName(name: string): RegistryItem | undefined {
|
|
122
|
+
return models.find((model) => model.name === name);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// Helper function to get model by registry path
|
|
126
|
+
export function getModelByPath(registryPath: string): RegistryItem | undefined {
|
|
127
|
+
return models.find((model) => model.registryPath === registryPath);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// Helper function for blob-based lookups
|
|
131
|
+
export function getModelBySrc(modelId: string, blobCoreKey: string): RegistryItem | undefined {
|
|
132
|
+
return models.find((model) => model.modelId === modelId && model.blobCoreKey === blobCoreKey);
|
|
133
|
+
}
|
|
134
|
+
`;
|
|
135
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { CurrentModel, ProcessedModel } from "./types";
|
|
2
|
+
export declare function loadCurrentModels(outputFile: string): CurrentModel[];
|
|
3
|
+
export declare function compareModels(remoteModels: ProcessedModel[], currentModels: CurrentModel[]): {
|
|
4
|
+
added: ProcessedModel[];
|
|
5
|
+
removed: CurrentModel[];
|
|
6
|
+
};
|
|
7
|
+
export declare function assignNames(models: ProcessedModel[]): (ProcessedModel & {
|
|
8
|
+
name: string;
|
|
9
|
+
})[];
|
|
10
|
+
export declare function createHistoryFile(added: (ProcessedModel & {
|
|
11
|
+
name: string;
|
|
12
|
+
})[], removed: CurrentModel[], currentModels: CurrentModel[], historyDir: string): string | null;
|
|
13
|
+
//# sourceMappingURL=history.d.ts.map
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import { generateExportName } from "./naming.js";
|
|
3
|
+
import { getCommitHash } from "./utils.js";
|
|
4
|
+
export function loadCurrentModels(outputFile) {
|
|
5
|
+
try {
|
|
6
|
+
if (!fs.existsSync(outputFile)) {
|
|
7
|
+
return [];
|
|
8
|
+
}
|
|
9
|
+
const content = fs.readFileSync(outputFile, "utf-8");
|
|
10
|
+
const modelsMatch = content.match(/export const models = \[([\s\S]*?)\] as const/);
|
|
11
|
+
if (!modelsMatch?.[1]) {
|
|
12
|
+
return [];
|
|
13
|
+
}
|
|
14
|
+
const modelsArrayContent = modelsMatch[1];
|
|
15
|
+
const currentModels = [];
|
|
16
|
+
const modelRegex = /\{[^}]+name:\s*"([^"]+)"[^}]+(?:registryPath|hyperbeeKey):\s*"([^"]+)"[^}]+\}/g;
|
|
17
|
+
let match;
|
|
18
|
+
while ((match = modelRegex.exec(modelsArrayContent)) !== null) {
|
|
19
|
+
if (match[1] && match[2]) {
|
|
20
|
+
currentModels.push({
|
|
21
|
+
name: match[1],
|
|
22
|
+
registryPath: match[2],
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return currentModels;
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
30
|
+
console.warn("⚠️ Could not load current models:", message);
|
|
31
|
+
return [];
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
export function compareModels(remoteModels, currentModels) {
|
|
35
|
+
const currentPaths = new Set(currentModels.map((m) => m.registryPath));
|
|
36
|
+
const remotePaths = new Set(remoteModels.map((m) => m.registryPath));
|
|
37
|
+
const added = remoteModels.filter((m) => !currentPaths.has(m.registryPath));
|
|
38
|
+
const removed = currentModels.filter((m) => !remotePaths.has(m.registryPath));
|
|
39
|
+
return { added, removed };
|
|
40
|
+
}
|
|
41
|
+
export function assignNames(models) {
|
|
42
|
+
const usedNames = new Set();
|
|
43
|
+
return models.map((m) => ({
|
|
44
|
+
...m,
|
|
45
|
+
name: generateExportName({
|
|
46
|
+
path: m.registryPath,
|
|
47
|
+
engine: m.engine,
|
|
48
|
+
name: m.modelName,
|
|
49
|
+
quantization: m.quantization,
|
|
50
|
+
params: m.params,
|
|
51
|
+
tags: m.tags,
|
|
52
|
+
usedNames,
|
|
53
|
+
}),
|
|
54
|
+
}));
|
|
55
|
+
}
|
|
56
|
+
export function createHistoryFile(added, removed, currentModels, historyDir) {
|
|
57
|
+
if (added.length === 0 && removed.length === 0) {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
if (!fs.existsSync(historyDir)) {
|
|
61
|
+
fs.mkdirSync(historyDir, { recursive: true });
|
|
62
|
+
}
|
|
63
|
+
const shortHash = getCommitHash(true);
|
|
64
|
+
const fullHash = getCommitHash(false);
|
|
65
|
+
const timestamp = new Date().toISOString();
|
|
66
|
+
const filename = `${shortHash}.txt`;
|
|
67
|
+
const filepath = `${historyDir}/${filename}`;
|
|
68
|
+
let content = `commit=${fullHash}\n`;
|
|
69
|
+
content += `timestamp=${timestamp}\n`;
|
|
70
|
+
content += `previous_count=${currentModels.length}\n`;
|
|
71
|
+
content += `new_count=${currentModels.length + added.length - removed.length}\n`;
|
|
72
|
+
content += `\n`;
|
|
73
|
+
if (added.length > 0) {
|
|
74
|
+
content += `[added]\n`;
|
|
75
|
+
added.forEach((m) => {
|
|
76
|
+
content += `${m.name}\n`;
|
|
77
|
+
});
|
|
78
|
+
content += `\n`;
|
|
79
|
+
}
|
|
80
|
+
if (removed.length > 0) {
|
|
81
|
+
content += `[removed]\n`;
|
|
82
|
+
removed.forEach((m) => {
|
|
83
|
+
content += `${m.name}\n`;
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
fs.writeFileSync(filepath, content);
|
|
87
|
+
return filepath;
|
|
88
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import { fileURLToPath } from "url";
|
|
3
|
+
import { execSync } from "child_process";
|
|
4
|
+
import { generateModelsFileContent } from "./codegen.js";
|
|
5
|
+
import { generateExportName } from "./naming.js";
|
|
6
|
+
import { assignNames, compareModels, createHistoryFile, loadCurrentModels, } from "./history.js";
|
|
7
|
+
import { collectModels } from "./registry.js";
|
|
8
|
+
import { formatSize } from "./utils.js";
|
|
9
|
+
const OUTPUT_FILE = fileURLToPath(new URL("../registry/models.ts", import.meta.url));
|
|
10
|
+
const HISTORY_DIR = fileURLToPath(new URL("../history", import.meta.url));
|
|
11
|
+
async function checkOnly(nonBlocking = false, showDuplicates = false) {
|
|
12
|
+
const timeoutMs = 30000;
|
|
13
|
+
let timedOut = false;
|
|
14
|
+
const timeoutPromise = new Promise((resolve) => {
|
|
15
|
+
setTimeout(() => {
|
|
16
|
+
timedOut = true;
|
|
17
|
+
console.log("⏱️ Model check timed out");
|
|
18
|
+
console.log(" Run 'bun check-models' manually to retry");
|
|
19
|
+
resolve(null);
|
|
20
|
+
}, timeoutMs);
|
|
21
|
+
});
|
|
22
|
+
try {
|
|
23
|
+
const result = await Promise.race([
|
|
24
|
+
(async () => {
|
|
25
|
+
const remoteModels = await collectModels({ showDuplicates });
|
|
26
|
+
const currentModels = loadCurrentModels(OUTPUT_FILE);
|
|
27
|
+
remoteModels.sort((a, b) => a.addon.localeCompare(b.addon) ||
|
|
28
|
+
a.registryPath.localeCompare(b.registryPath));
|
|
29
|
+
return { remoteModels, currentModels };
|
|
30
|
+
})(),
|
|
31
|
+
timeoutPromise,
|
|
32
|
+
]);
|
|
33
|
+
if (timedOut || !result) {
|
|
34
|
+
process.exit(nonBlocking ? 0 : 1);
|
|
35
|
+
}
|
|
36
|
+
const { remoteModels, currentModels } = result;
|
|
37
|
+
const { added, removed } = compareModels(remoteModels, currentModels);
|
|
38
|
+
if (added.length === 0 && removed.length === 0) {
|
|
39
|
+
console.log(`✅ Models are up to date (${remoteModels.length} models)`);
|
|
40
|
+
process.exit(0);
|
|
41
|
+
}
|
|
42
|
+
console.log("");
|
|
43
|
+
console.log("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
|
44
|
+
if (added.length > 0) {
|
|
45
|
+
console.log(`✨ ${added.length} new model${added.length === 1 ? "" : "s"} available:`);
|
|
46
|
+
const usedNames = new Set();
|
|
47
|
+
added.slice(0, 10).forEach((m) => {
|
|
48
|
+
const exportName = generateExportName({
|
|
49
|
+
path: m.registryPath,
|
|
50
|
+
engine: m.engine,
|
|
51
|
+
name: m.modelName,
|
|
52
|
+
quantization: m.quantization,
|
|
53
|
+
params: m.params,
|
|
54
|
+
tags: m.tags,
|
|
55
|
+
usedNames,
|
|
56
|
+
});
|
|
57
|
+
console.log(` + ${exportName} (${m.addon}, ${formatSize(m.expectedSize)})`);
|
|
58
|
+
});
|
|
59
|
+
if (added.length > 10) {
|
|
60
|
+
console.log(` ... and ${added.length - 10} more`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (removed.length > 0) {
|
|
64
|
+
console.log(`\n⚠️ ${removed.length} model${removed.length === 1 ? "" : "s"} removed:`);
|
|
65
|
+
removed.slice(0, 5).forEach((m) => {
|
|
66
|
+
console.log(` - ${m.name}`);
|
|
67
|
+
});
|
|
68
|
+
if (removed.length > 5) {
|
|
69
|
+
console.log(` ... and ${removed.length - 5} more`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
console.log("");
|
|
73
|
+
console.log(`💡 Run 'bun update-models' to sync changes`);
|
|
74
|
+
console.log("");
|
|
75
|
+
if (nonBlocking) {
|
|
76
|
+
console.log("💡 Commit will proceed - update models when ready");
|
|
77
|
+
}
|
|
78
|
+
console.log("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
|
79
|
+
console.log("");
|
|
80
|
+
process.exit(nonBlocking ? 0 : 1);
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
84
|
+
console.error("❌ Model check failed:", message);
|
|
85
|
+
process.exit(nonBlocking ? 0 : 1);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
async function updateModels(showDuplicates = false, noDedup = false) {
|
|
89
|
+
console.log("🔄 Fetching models from QVAC Registry...\n");
|
|
90
|
+
const currentModels = loadCurrentModels(OUTPUT_FILE);
|
|
91
|
+
const models = await collectModels({ showDuplicates, noDedup });
|
|
92
|
+
const { added, removed } = compareModels(models, currentModels);
|
|
93
|
+
models.sort((a, b) => a.addon.localeCompare(b.addon) ||
|
|
94
|
+
a.registryPath.localeCompare(b.registryPath));
|
|
95
|
+
fs.writeFileSync(OUTPUT_FILE, generateModelsFileContent(models));
|
|
96
|
+
try {
|
|
97
|
+
execSync(`npx prettier --write "${OUTPUT_FILE}"`, { stdio: "pipe" });
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
// prettier not available, skip formatting
|
|
101
|
+
}
|
|
102
|
+
console.log(`✅ Generated ${models.length} models → ${OUTPUT_FILE}`);
|
|
103
|
+
const addedWithNames = assignNames(added);
|
|
104
|
+
if (added.length > 0 || removed.length > 0) {
|
|
105
|
+
const historyFile = createHistoryFile(addedWithNames, removed, currentModels, HISTORY_DIR);
|
|
106
|
+
if (historyFile) {
|
|
107
|
+
console.log(`📜 Created history file → ${historyFile}`);
|
|
108
|
+
console.log(` Added: ${added.length}, Removed: ${removed.length}`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
async function main() {
|
|
113
|
+
const CHECK_ONLY = process.argv.includes("--check");
|
|
114
|
+
const NON_BLOCKING = process.argv.includes("--non-blocking");
|
|
115
|
+
const SHOW_DUPLICATES = process.argv.includes("--show-duplicates");
|
|
116
|
+
const NO_DEDUP = process.argv.includes("--no-dedup");
|
|
117
|
+
if (CHECK_ONLY) {
|
|
118
|
+
await checkOnly(NON_BLOCKING, SHOW_DUPLICATES);
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
await updateModels(SHOW_DUPLICATES, NO_DEDUP);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
main().catch(console.error);
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { ExportNameInput } from "./types";
|
|
2
|
+
export declare function cleanPart(p: string): string;
|
|
3
|
+
export declare function generateExportName({ path: registryPath, engine, name: modelName, quantization, params, tags, usedNames, }: ExportNameInput): string;
|
|
4
|
+
//# sourceMappingURL=naming.d.ts.map
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
import { getAddonFromEngine } from "../../schemas/engine-addon-map";
|
|
2
|
+
import { detectShardedModel } from "./shards.js";
|
|
3
|
+
// Normalizes a name part to a valid JS export name fragment.
|
|
4
|
+
// Strips ggml/gguf prefixes, shortens "instruct" to "inst",
|
|
5
|
+
// and uppercases with underscore separators.
|
|
6
|
+
export function cleanPart(p) {
|
|
7
|
+
if (!p)
|
|
8
|
+
return "";
|
|
9
|
+
return p
|
|
10
|
+
.replace(/ggml-?/gi, "")
|
|
11
|
+
.replace(/gguf-?/gi, "")
|
|
12
|
+
.replace(/instruct/gi, "inst")
|
|
13
|
+
.replace(/^-+|-+$/g, "")
|
|
14
|
+
.toUpperCase()
|
|
15
|
+
.replace(/[^A-Z0-9]+/g, "_");
|
|
16
|
+
}
|
|
17
|
+
// Generates a unique export constant name for a model based on its
|
|
18
|
+
// registry metadata, engine type, tags, and filename.
|
|
19
|
+
export function generateExportName({ path: registryPath, engine, name: modelName, quantization, params, tags, usedNames, }) {
|
|
20
|
+
let addon = getAddonFromEngine(engine);
|
|
21
|
+
// Override addon classification when tags clearly indicate a different type.
|
|
22
|
+
// e.g. SmolVLM2 f16 uses translation-llamacpp engine (→nmt) but is actually a multimodal LLM.
|
|
23
|
+
if (addon === "nmt" &&
|
|
24
|
+
tags.some((t) => t === "generation" || t === "multimodal")) {
|
|
25
|
+
addon = "llm";
|
|
26
|
+
}
|
|
27
|
+
let exportName = "";
|
|
28
|
+
const filename = decodeURIComponent(registryPath.split("/").pop() || registryPath);
|
|
29
|
+
const lowerFilename = filename.toLowerCase();
|
|
30
|
+
const lowerPath = registryPath.toLowerCase();
|
|
31
|
+
// Extract structured info from tags array.
|
|
32
|
+
// Tags typically contain: [function, type, modelName, langPair/other]
|
|
33
|
+
const tagType = tags[1] || "";
|
|
34
|
+
const tagName = tags[2] || "";
|
|
35
|
+
const tagExtra = tags[3] || "";
|
|
36
|
+
exportName = generateBaseName({
|
|
37
|
+
addon,
|
|
38
|
+
engine,
|
|
39
|
+
filename,
|
|
40
|
+
lowerFilename,
|
|
41
|
+
lowerPath,
|
|
42
|
+
modelName,
|
|
43
|
+
quantization,
|
|
44
|
+
params,
|
|
45
|
+
tagType,
|
|
46
|
+
tagName,
|
|
47
|
+
tagExtra,
|
|
48
|
+
tags,
|
|
49
|
+
});
|
|
50
|
+
exportName = exportName.replace(/^_+|_+$/g, "").replace(/_+/g, "_");
|
|
51
|
+
// Add suffix for metadata files (tensors.txt are shard metadata, not actual models)
|
|
52
|
+
if (lowerFilename.endsWith(".tensors.txt")) {
|
|
53
|
+
exportName = `${exportName}_TENSORS`;
|
|
54
|
+
}
|
|
55
|
+
// Add SHARD suffix for sharded models
|
|
56
|
+
if (detectShardedModel(filename).isSharded) {
|
|
57
|
+
exportName = `${exportName}_SHARD`;
|
|
58
|
+
}
|
|
59
|
+
return resolveCollision(exportName, quantization, usedNames);
|
|
60
|
+
}
|
|
61
|
+
// Resolves name collisions by appending quantization or a sequential counter.
|
|
62
|
+
function resolveCollision(exportName, quantization, usedNames) {
|
|
63
|
+
let finalName = exportName || "UNKNOWN_MODEL";
|
|
64
|
+
if (usedNames.has(finalName) && quantization) {
|
|
65
|
+
const cleanedQuant = cleanPart(quantization);
|
|
66
|
+
// Only append quantization if it's not already at the end of the name
|
|
67
|
+
if (cleanedQuant && !exportName.endsWith(cleanedQuant)) {
|
|
68
|
+
const withQuant = `${exportName}_${cleanedQuant}`;
|
|
69
|
+
if (!usedNames.has(withQuant)) {
|
|
70
|
+
finalName = withQuant;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
let counter = 1;
|
|
75
|
+
while (usedNames.has(finalName)) {
|
|
76
|
+
finalName = `${exportName}_${counter++}`;
|
|
77
|
+
}
|
|
78
|
+
usedNames.add(finalName);
|
|
79
|
+
return finalName;
|
|
80
|
+
}
|
|
81
|
+
function generateBaseName(input) {
|
|
82
|
+
const { addon, lowerFilename } = input;
|
|
83
|
+
// Detect VAD Silero before whisper (same engine, different model type)
|
|
84
|
+
if (lowerFilename.includes("silero")) {
|
|
85
|
+
return generateVadSileroName(input);
|
|
86
|
+
}
|
|
87
|
+
switch (addon) {
|
|
88
|
+
case "whisper":
|
|
89
|
+
return generateWhisperName(input);
|
|
90
|
+
case "vad":
|
|
91
|
+
return generateVadName(input);
|
|
92
|
+
case "nmt":
|
|
93
|
+
return generateNmtName(input);
|
|
94
|
+
case "llm":
|
|
95
|
+
return generateLlmName(input);
|
|
96
|
+
case "embeddings":
|
|
97
|
+
return generateEmbeddingsName(input);
|
|
98
|
+
case "tts":
|
|
99
|
+
return generateTtsName(input);
|
|
100
|
+
case "ocr":
|
|
101
|
+
return generateOcrName(input);
|
|
102
|
+
default:
|
|
103
|
+
return cleanPart(input.filename.replace(/\.\w+$/, ""));
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function generateVadSileroName({ filename }) {
|
|
107
|
+
const versionMatch = filename.match(/v(\d+\.\d+\.\d+)/i);
|
|
108
|
+
const version = versionMatch ? versionMatch[1] : "";
|
|
109
|
+
const nameParts = ["SILERO", version].filter((p) => p && p !== "");
|
|
110
|
+
return `VAD_${nameParts.map(cleanPart).join("_")}`;
|
|
111
|
+
}
|
|
112
|
+
function generateWhisperName({ filename, lowerFilename, quantization, }) {
|
|
113
|
+
// Extract language from filename prefix (e.g. "de-tiny-ggml-model-f16.bin")
|
|
114
|
+
const langPrefixMatch = filename.match(/^([a-z]{2})-/i);
|
|
115
|
+
// Detect English-only models from ".en" or ".en-" in filename
|
|
116
|
+
const isEnglishOnly = /\.en[-.]/.test(lowerFilename) || lowerFilename.endsWith(".en.bin");
|
|
117
|
+
// Extract size from filename
|
|
118
|
+
const sizeMatch = filename.match(/\b(tiny|base|small|medium|large(?:-v[0-9]+)?(?:-turbo)?)\b/i);
|
|
119
|
+
const modelSize = sizeMatch ? sizeMatch[1] : "";
|
|
120
|
+
// Extract quant from filename (f16, q8_0, etc.)
|
|
121
|
+
const quantMatch = filename.match(/[-_](f16|q[0-9]+[_][0-9]+|q8)\b/i);
|
|
122
|
+
const fileQuant = quantMatch ? quantMatch[1] : quantization;
|
|
123
|
+
const nameParts = [];
|
|
124
|
+
if (langPrefixMatch) {
|
|
125
|
+
const langCode = langPrefixMatch[1].toLowerCase();
|
|
126
|
+
const langMap = {
|
|
127
|
+
de: "GERMAN",
|
|
128
|
+
es: "SPANISH",
|
|
129
|
+
fr: "FRENCH",
|
|
130
|
+
it: "ITALIAN",
|
|
131
|
+
ja: "JAPANESE",
|
|
132
|
+
pt: "PORTUGUESE",
|
|
133
|
+
ru: "RUSSIAN",
|
|
134
|
+
nb: "NORWEGIAN",
|
|
135
|
+
en: "ENGLISH",
|
|
136
|
+
};
|
|
137
|
+
const langName = langMap[langCode] || langCode.toUpperCase();
|
|
138
|
+
nameParts.push(langName);
|
|
139
|
+
}
|
|
140
|
+
else if (isEnglishOnly) {
|
|
141
|
+
nameParts.push("EN");
|
|
142
|
+
}
|
|
143
|
+
if (modelSize)
|
|
144
|
+
nameParts.push(modelSize);
|
|
145
|
+
if (fileQuant)
|
|
146
|
+
nameParts.push(fileQuant);
|
|
147
|
+
return `WHISPER_${nameParts.map(cleanPart).join("_")}`;
|
|
148
|
+
}
|
|
149
|
+
function generateVadName({ filename, tagType, tagName, }) {
|
|
150
|
+
const versionMatch = filename.match(/v(\d+\.\d+\.\d+)/i);
|
|
151
|
+
const version = versionMatch ? versionMatch[1] : "";
|
|
152
|
+
const typeName = tagType || tagName || "SILERO";
|
|
153
|
+
const nameParts = [typeName, version].filter((p) => p && p !== "");
|
|
154
|
+
return `VAD_${nameParts.map(cleanPart).join("_")}`;
|
|
155
|
+
}
|
|
156
|
+
function generateNmtName(input) {
|
|
157
|
+
const { lowerPath, lowerFilename } = input;
|
|
158
|
+
if (lowerPath.includes("salamandra") ||
|
|
159
|
+
lowerFilename.includes("salamandra")) {
|
|
160
|
+
return generateNmtSalamandraName(input);
|
|
161
|
+
}
|
|
162
|
+
if (lowerPath.includes("indictrans") ||
|
|
163
|
+
lowerFilename.includes("indictrans")) {
|
|
164
|
+
return generateNmtIndictransName(input);
|
|
165
|
+
}
|
|
166
|
+
if (lowerPath.includes("opus") || lowerFilename.includes("opus")) {
|
|
167
|
+
return generateNmtOpusName(input);
|
|
168
|
+
}
|
|
169
|
+
if (lowerPath.includes("bergamot") || lowerFilename.includes("bergamot")) {
|
|
170
|
+
return generateNmtBergamotName(input);
|
|
171
|
+
}
|
|
172
|
+
const nameParts = [input.modelName, input.quantization].filter((p) => p && p !== "");
|
|
173
|
+
return `NMT_${nameParts.map(cleanPart).join("_")}`;
|
|
174
|
+
}
|
|
175
|
+
function generateNmtSalamandraName({ filename }) {
|
|
176
|
+
const base = filename.replace(/\.\w+$/, "").replace(/-\d{5}-of-\d{5}/, "");
|
|
177
|
+
return cleanPart(base);
|
|
178
|
+
}
|
|
179
|
+
function generateNmtIndictransName({ filename, quantization, }) {
|
|
180
|
+
const langMatch = filename.match(/(en-indic|indic-en|indic-indic)/i);
|
|
181
|
+
const langDir = langMatch ? langMatch[1] : "";
|
|
182
|
+
const langMap = {
|
|
183
|
+
"en-indic": "EN_HI",
|
|
184
|
+
"indic-en": "HI_EN",
|
|
185
|
+
"indic-indic": "HI_HI",
|
|
186
|
+
};
|
|
187
|
+
const langPair = langMap[langDir.toLowerCase()] || cleanPart(langDir);
|
|
188
|
+
const sizeMatch = filename.match(/(\d+[MB])/i);
|
|
189
|
+
const size = sizeMatch ? sizeMatch[1] : "";
|
|
190
|
+
const nameParts = [langPair, "INDIC", size, quantization].filter((p) => p && p !== "");
|
|
191
|
+
return `MARIAN_${nameParts.map(cleanPart).join("_")}`;
|
|
192
|
+
}
|
|
193
|
+
function generateNmtOpusName({ filename, quantization, tagExtra, }) {
|
|
194
|
+
const langMatch = filename.match(/-([a-z]{2,3})-([a-z]{2,3})\./i);
|
|
195
|
+
let langPair = "";
|
|
196
|
+
if (langMatch) {
|
|
197
|
+
langPair = `${langMatch[1].toUpperCase()}_${langMatch[2].toUpperCase()}`;
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
const tagLang = tagExtra || "";
|
|
201
|
+
if (tagLang.match(/^[a-z]{2}-[a-z]{2}$/i)) {
|
|
202
|
+
const [src, tgt] = tagLang.split("-");
|
|
203
|
+
langPair = `${src.toUpperCase()}_${tgt.toUpperCase()}`;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
if (!langPair) {
|
|
207
|
+
const roaMatch = filename.match(/-([a-z]{2,3})-([a-z]{2,3})-f16/i);
|
|
208
|
+
if (roaMatch) {
|
|
209
|
+
langPair = `${roaMatch[1].toUpperCase()}_${roaMatch[2].toUpperCase()}`;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
const nameParts = [langPair, quantization].filter((p) => p && p !== "");
|
|
213
|
+
return `MARIAN_OPUS_${nameParts.map(cleanPart).join("_")}`;
|
|
214
|
+
}
|
|
215
|
+
function generateNmtBergamotName({ filename }) {
|
|
216
|
+
const langMatch = filename.match(/[.]([a-z]{2,4})[.]/i);
|
|
217
|
+
let langPair = "";
|
|
218
|
+
if (langMatch) {
|
|
219
|
+
const code = langMatch[1];
|
|
220
|
+
if (code.length === 4) {
|
|
221
|
+
langPair = `${code.slice(0, 2).toUpperCase()}_${code.slice(2).toUpperCase()}`;
|
|
222
|
+
}
|
|
223
|
+
else {
|
|
224
|
+
langPair = code.toUpperCase();
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
let suffix = "";
|
|
228
|
+
if (filename.startsWith("model.")) {
|
|
229
|
+
suffix = "";
|
|
230
|
+
}
|
|
231
|
+
else if (filename.match(/^vocab\./)) {
|
|
232
|
+
suffix = "_VOCAB";
|
|
233
|
+
}
|
|
234
|
+
else if (filename.match(/^srcvocab\./)) {
|
|
235
|
+
suffix = "_SRCVOCAB";
|
|
236
|
+
}
|
|
237
|
+
else if (filename.match(/^trgvocab\./)) {
|
|
238
|
+
suffix = "_TRGVOCAB";
|
|
239
|
+
}
|
|
240
|
+
else if (filename.startsWith("lex.")) {
|
|
241
|
+
suffix = "_LEX";
|
|
242
|
+
}
|
|
243
|
+
else if (filename === "metadata.json") {
|
|
244
|
+
suffix = "_METADATA";
|
|
245
|
+
}
|
|
246
|
+
return `BERGAMOT_${langPair}${suffix}`;
|
|
247
|
+
}
|
|
248
|
+
function generateLlmName({ filename, tagType, tagName, modelName, quantization, params, }) {
|
|
249
|
+
const type = tagType || "";
|
|
250
|
+
const name = tagName || modelName || "";
|
|
251
|
+
const isMMProj = filename.includes("mmproj");
|
|
252
|
+
// Try to extract model family from filename to recover version info
|
|
253
|
+
// that tags may omit (e.g., tag "qwen" but filename "Qwen3-1.7B-...")
|
|
254
|
+
let familyName = name;
|
|
255
|
+
if (params) {
|
|
256
|
+
const parseFn = filename
|
|
257
|
+
.replace(/^mmproj[-_]/i, "")
|
|
258
|
+
.replace(/\.tensors\.txt$/, "")
|
|
259
|
+
.replace(/\.\w+$/, "")
|
|
260
|
+
.replace(/-\d{5}-of-\d{5}/, "");
|
|
261
|
+
const paramsEsc = params.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
262
|
+
const familyMatch = parseFn.match(new RegExp(`^(.+?)[-_]${paramsEsc}`, "i"));
|
|
263
|
+
if (familyMatch?.[1]) {
|
|
264
|
+
const candidate = familyMatch[1];
|
|
265
|
+
// Use filename family only if it extends the tag name
|
|
266
|
+
// Strip common engine/format suffixes from tag before comparing
|
|
267
|
+
const tagCore = name.replace(/[-_](ggml|cpp|onnx|llamacpp|metal)$/i, "");
|
|
268
|
+
const cleanedCandidate = candidate.replace(/[-_.]/g, "").toLowerCase();
|
|
269
|
+
const cleanedTagCore = tagCore.replace(/[-_.]/g, "").toLowerCase();
|
|
270
|
+
if (cleanedCandidate.startsWith(cleanedTagCore) &&
|
|
271
|
+
cleanedCandidate.length >= cleanedTagCore.length) {
|
|
272
|
+
familyName = candidate;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
const nameParts = [familyName, params, type, quantization].filter((p) => p && p !== "");
|
|
277
|
+
let exportName = nameParts.map(cleanPart).join("_");
|
|
278
|
+
if (isMMProj) {
|
|
279
|
+
exportName = "MMPROJ_" + exportName;
|
|
280
|
+
}
|
|
281
|
+
return exportName;
|
|
282
|
+
}
|
|
283
|
+
function generateEmbeddingsName({ tagName, modelName, params, quantization, }) {
|
|
284
|
+
const name = tagName || modelName || "";
|
|
285
|
+
const nameParts = [name, params, quantization].filter((p) => p && p !== "");
|
|
286
|
+
return nameParts.map(cleanPart).join("_");
|
|
287
|
+
}
|
|
288
|
+
function generateTtsName({ filename, tagName, modelName, tagExtra, tagType, quantization, }) {
|
|
289
|
+
const name = tagName || modelName || "";
|
|
290
|
+
const language = tagExtra || "";
|
|
291
|
+
const type = tagType || "";
|
|
292
|
+
const nameParts = [name, language, type, quantization].filter((p) => p && p !== "");
|
|
293
|
+
let exportName = `TTS_${nameParts.map(cleanPart).join("_")}`;
|
|
294
|
+
if (filename.endsWith(".onnx.json") || filename.includes("config.json")) {
|
|
295
|
+
exportName = exportName + "_CONFIG";
|
|
296
|
+
}
|
|
297
|
+
if (filename.endsWith(".onnx_data")) {
|
|
298
|
+
exportName = exportName + "_DATA";
|
|
299
|
+
}
|
|
300
|
+
return exportName;
|
|
301
|
+
}
|
|
302
|
+
function generateOcrName({ filename, tagName, modelName, tagExtra, }) {
|
|
303
|
+
const name = tagName || modelName || "";
|
|
304
|
+
const language = tagExtra || "";
|
|
305
|
+
let fileType = "";
|
|
306
|
+
if (filename.includes("detector")) {
|
|
307
|
+
fileType = "DETECTOR";
|
|
308
|
+
}
|
|
309
|
+
else if (filename.includes("recognizer")) {
|
|
310
|
+
fileType = "RECOGNIZER";
|
|
311
|
+
}
|
|
312
|
+
const nameParts = [name, language, fileType].filter((p) => p && p !== "");
|
|
313
|
+
return `OCR_${nameParts.map(cleanPart).join("_")}`;
|
|
314
|
+
}
|