@openspecui/server 3.11.3 → 3.11.4

package/dist/{src-awZ9aP1s.mjs → src-CZfiVL_-.mjs}

@@ -1,4 +1,5 @@
 import { join, posix } from "node:path";
+import { runControlledTranslationTask } from "@openspecui/core/translator";
 
 //#region ../local-llama-translator/src/factory.ts
 const DEFAULT_SYSTEM_PROMPT = "You are a translation engine. Return only the translated text, preserve Markdown structure, inline code, URLs, and file paths.";
@@ -9,23 +10,15 @@ var LocalLlamaTranslatorFactory = class {
 	async prepare(options) {
 		const model = options.model || this.options.defaultModel;
 		if (!model) throw new Error("A GGUF model id or runtime model path is required.");
-		const module = await (this.options.loadModule ?? loadLlamaRuntimeModule)();
-		const resolvedConfig = readRuntimeConfig(options.runtimeConfig);
-		const runtimeModel = await loadRuntimeModel({
-			module,
+		await probeLocalLlamaRuntimeModel({
 			model,
 			cacheDir: this.options.cacheDir,
-			runtimeConfig: resolvedConfig,
-			defaultGpuLayers: this.options.gpuLayers,
+			runtimeConfig: options.runtimeConfig,
+			loadModule: this.options.loadModule,
+			contextSize: this.options.contextSize,
+			gpuLayers: this.options.gpuLayers,
 			monitor: options.monitor
 		});
-		const context = await runtimeModel.createContext({ contextSize: resolvedConfig.contextSize ?? this.options.contextSize });
-		options.monitor?.setStatus({
-			message: `Llama model ${model} is ready.`,
-			progress: 1
-		});
-		await disposeRuntimeNode(context);
-		await disposeRuntimeNode(runtimeModel);
 	}
 	async create(options) {
 		const model = options.model || this.options.defaultModel;
@@ -56,6 +49,31 @@ var LocalLlamaTranslatorFactory = class {
 function createLocalLlamaTranslatorFactory(options = {}) {
 	return new LocalLlamaTranslatorFactory(options);
 }
+async function probeLocalLlamaRuntimeModel(input) {
+	const module = await (input.loadModule ?? loadLlamaRuntimeModule)();
+	const resolvedConfig = readRuntimeConfig(input.runtimeConfig);
+	const runtimeModel = await loadRuntimeModel({
+		module,
+		model: input.model,
+		cacheDir: input.cacheDir,
+		runtimeConfig: resolvedConfig,
+		defaultGpuLayers: input.gpuLayers,
+		monitor: input.monitor
+	});
+	try {
+		const context = await runtimeModel.createContext({ contextSize: resolvedConfig.contextSize ?? input.contextSize });
+		try {
+			input.monitor?.setStatus({
+				message: `Llama model ${input.model} is ready.`,
+				progress: 1
+			});
+		} finally {
+			await disposeRuntimeNode(context);
+		}
+	} finally {
+		await disposeRuntimeNode(runtimeModel);
+	}
+}
 var LocalLlamaTranslator = class {
 	constructor(module, model, options) {
 		this.module = module;
@@ -64,32 +82,46 @@ var LocalLlamaTranslator = class {
 	}
 	async *batchTranslate(inputs, options) {
 		for (const [index, input] of inputs.entries()) {
-			throwIfAborted(options?.signal);
-			const context = await this.model.createContext({ contextSize: this.options.runtimeConfig.contextSize ?? this.options.factoryOptions.contextSize });
-			try {
-				const session = new this.module.LlamaChatSession({
-					contextSequence: context.getSequence(),
-					systemPrompt: this.options.runtimeConfig.systemPrompt ?? this.options.factoryOptions.systemPrompt ?? DEFAULT_SYSTEM_PROMPT
+			const controlled = await runControlledTranslationTask(async (signal) => {
+				throwIfAborted(signal);
+				const context = await this.model.createContext({
+					contextSize: this.options.runtimeConfig.contextSize ?? this.options.factoryOptions.contextSize,
+					batchSize: this.options.runtimeConfig.batchSize ?? this.options.factoryOptions.batchSize,
+					flashAttention: this.options.runtimeConfig.flashAttention ?? this.options.factoryOptions.flashAttention
 				});
 				try {
-					const output = await session.prompt(buildTranslationPrompt({
-						sourceLanguage: this.options.sourceLanguage,
-						targetLanguage: this.options.targetLanguage,
-						text: input,
-						instructions: options?.instructions,
-						context: options?.context
-					}));
-					throwIfAborted(options?.signal);
-					yield {
-						index,
-						output: output.trim()
-					};
+					const session = new this.module.LlamaChatSession({
+						contextSequence: context.getSequence(),
+						systemPrompt: this.options.runtimeConfig.systemPrompt ?? this.options.factoryOptions.systemPrompt ?? DEFAULT_SYSTEM_PROMPT
+					});
+					try {
+						const output = await session.prompt(buildTranslationPrompt({
+							sourceLanguage: this.options.sourceLanguage,
+							targetLanguage: this.options.targetLanguage,
+							text: input,
+							instructions: options?.instructions,
+							context: options?.context
+						}));
+						throwIfAborted(signal);
+						return output.trim();
+					} finally {
+						await disposeRuntimeNode(session);
+					}
 				} finally {
-					await disposeRuntimeNode(session);
+					await disposeRuntimeNode(context);
 				}
-			} finally {
-				await disposeRuntimeNode(context);
+			}, options);
+			if (controlled.ok) {
+				yield {
+					index,
+					output: controlled.value
+				};
+				continue;
 			}
+			yield {
+				index,
+				error: controlled.error
+			};
 		}
 	}
 	destroy() {
@@ -104,7 +136,10 @@ async function loadRuntimeModel(input) {
 			cacheDir: input.cacheDir,
 			runtimeConfig: input.runtimeConfig
 		}),
-		gpuLayers: input.runtimeConfig.gpuLayers ?? input.defaultGpuLayers
+		gpuLayers: input.runtimeConfig.gpuLayers ?? input.defaultGpuLayers,
+		useMmap: input.runtimeConfig.useMmap,
+		useMlock: input.runtimeConfig.useMlock,
+		defaultContextFlashAttention: input.runtimeConfig.flashAttention
 	});
 }
 function buildTranslationPrompt(input) {
@@ -126,8 +161,12 @@ function readRuntimeConfig(runtimeConfig) {
 	return {
 		modelPath: readString(runtimeConfig, "modelPath"),
 		contextSize: readNumber(runtimeConfig, "contextSize"),
-		gpuLayers: readNumber(runtimeConfig, "gpuLayers"),
-		systemPrompt: readString(runtimeConfig, "systemPrompt")
+		gpuLayers: readGpuLayers(runtimeConfig?.gpuLayers),
+		systemPrompt: readString(runtimeConfig, "systemPrompt"),
+		batchSize: readNumber(runtimeConfig, "batchSize"),
+		flashAttention: readBoolean(runtimeConfig, "flashAttention"),
+		useMmap: readBoolean(runtimeConfig, "useMmap"),
+		useMlock: readBoolean(runtimeConfig, "useMlock")
 	};
 }
 function readString(record, key) {
@@ -138,6 +177,14 @@ function readNumber(record, key) {
 	const value = record?.[key];
 	return typeof value === "number" && Number.isFinite(value) ? value : void 0;
 }
+function readBoolean(record, key) {
+	const value = record?.[key];
+	return typeof value === "boolean" ? value : void 0;
+}
+function readGpuLayers(value) {
+	if (typeof value === "number" && Number.isFinite(value)) return value;
+	if (value === "auto" || value === "max") return value;
+}
 async function disposeRuntimeNode(value) {
 	await value?.dispose?.();
 }
@@ -229,4 +276,4 @@ function dedupeFiles(files) {
 }
 
 //#endregion
-export { LocalLlamaTranslatorFactory as n, createLocalLlamaTranslatorFactory as r, resolveGgufModelDownloadPlanFromRepositoryFiles as t };
+export { probeLocalLlamaRuntimeModel as i, LocalLlamaTranslatorFactory as n, createLocalLlamaTranslatorFactory as r, resolveGgufModelDownloadPlanFromRepositoryFiles as t };

package/dist/{src-5XpFsBo7.mjs → src-DTmyMpWU.mjs}

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { runControlledTranslationTask } from "@openspecui/core/translator";
 
 //#region ../../node_modules/.pnpm/@tanstack+devtools-event-client@0.4.3/node_modules/@tanstack/devtools-event-client/dist/esm/plugin.js
 var EventClient = class {
@@ -13383,31 +13384,42 @@ var OpenAICompletionTranslator = class {
 		this.options = options;
 	}
 	async *batchTranslate(inputs, options) {
-		const abortController = createAbortController(options?.signal);
-		const adapter = createConfiguredOpenAiAdapter({
-			model: this.options.model,
-			token: this.options.token,
-			baseUrl: this.options.baseUrl
-		});
-		for (const [index, source] of inputs.entries()) yield {
-			index,
-			output: (await chat({
-				adapter,
-				stream: false,
-				temperature: 0,
-				abortController,
-				systemPrompts: [[
-					"You are a translation engine.",
-					`Translate from ${this.options.sourceLanguage} to ${this.options.targetLanguage}.`,
-					options?.instructions ?? "Translate the source accurately.",
-					"Return only the translated source without commentary."
-				].filter(Boolean).join("\n")],
-				messages: [{
-					role: "user",
-					content: [options?.context ? `<context>\n${options.context}\n</context>` : "", `<source>\n${source}\n</source>`].filter(Boolean).join("\n\n")
-				}]
-			})).trim()
-		};
+		for (const [index, source] of inputs.entries()) {
+			const controlled = await runControlledTranslationTask(async (signal) => {
+				const abortController = createAbortController(signal);
+				return (await chat({
+					adapter: createConfiguredOpenAiAdapter({
+						model: this.options.model,
+						token: this.options.token,
+						baseUrl: this.options.baseUrl
+					}),
+					stream: false,
+					temperature: 0,
+					abortController,
+					systemPrompts: [[
+						"You are a translation engine.",
+						`Translate from ${this.options.sourceLanguage} to ${this.options.targetLanguage}.`,
+						options?.instructions ?? "Translate the source accurately.",
+						"Return only the translated source without commentary."
+					].filter(Boolean).join("\n")],
+					messages: [{
+						role: "user",
+						content: [options?.context ? `<context>\n${options.context}\n</context>` : "", `<source>\n${source}\n</source>`].filter(Boolean).join("\n\n")
+					}]
+				})).trim();
+			}, options);
+			if (controlled.ok) {
+				yield {
+					index,
+					output: controlled.value
+				};
+				continue;
+			}
+			yield {
+				index,
+				error: controlled.error
+			};
+		}
 	}
 };
 function createAbortController(signal) {

package/dist/src-DVJf9xbi.mjs

@@ -0,0 +1,3 @@
+import { i as probeLocalLlamaRuntimeModel, n as LocalLlamaTranslatorFactory, r as createLocalLlamaTranslatorFactory, t as resolveGgufModelDownloadPlanFromRepositoryFiles } from "./src-CZfiVL_-.mjs";
+
+export { createLocalLlamaTranslatorFactory, probeLocalLlamaRuntimeModel };

package/dist/{src-BHCDKXul.mjs → src-DfagK_w9.mjs}

@@ -1,5 +1,6 @@
 import { join } from "node:path";
 import { readFile } from "node:fs/promises";
+import { runControlledTranslationTask } from "@openspecui/core/translator";
 import "@openspecui/core/local-download-profiles";
 
 //#region ../local-translator/src/index.ts
@@ -27,18 +28,28 @@ var LocalTranslator = class {
 		this.languages = languages;
 	}
 	async *batchTranslate(inputs, options) {
-		throwIfAborted(options?.signal);
-		const result = await this.pipeline(inputs, {
-			src_lang: this.languages.sourceLanguage,
-			tgt_lang: this.languages.targetLanguage,
-			signal: options?.signal
-		});
-		throwIfAborted(options?.signal);
-		const outputs = readTranslatedOutputs(result, inputs.length);
-		for (const [index, output] of outputs.entries()) yield {
-			index,
-			output
-		};
+		for (const [index, input] of inputs.entries()) {
+			const controlled = await runControlledTranslationTask(async (signal) => {
+				const result = await this.pipeline(input, {
+					src_lang: this.languages.sourceLanguage,
+					tgt_lang: this.languages.targetLanguage,
+					signal
+				});
+				throwIfAborted(signal);
+				return readTranslatedOutputs(result, 1)[0] ?? "";
+			}, options);
+			if (controlled.ok) {
+				yield {
+					index,
+					output: controlled.value
+				};
+				continue;
+			}
+			yield {
+				index,
+				error: controlled.error
+			};
+		}
 	}
 	destroy() {
 		this.pipeline.dispose?.();

package/dist/{src-BJ-K9Dp2.mjs → src-Dh_UAz5C.mjs}

@@ -1,4 +1,5 @@
 import { join, posix } from "node:path";
+import { runControlledTranslationTask } from "@openspecui/core/translator";
 
 //#region ../local-ct2-translator/src/ct2-download-plan.ts
 const CT2_REQUIRED_FILE_NAMES = [
@@ -154,18 +155,30 @@ var LocalCt2Translator = class {
 		this.factoryOptions = factoryOptions;
 	}
 	async *batchTranslate(inputs, options) {
-		throwIfAborted(options?.signal);
-		const result = await this.translator.translateBatch(inputs, {
-			beamSize: this.runtimeConfig.beamSize ?? this.factoryOptions.beamSize,
-			maxBatchSize: this.runtimeConfig.maxBatchSize ?? this.factoryOptions.maxBatchSize,
-			returnScores: false
-		});
-		throwIfAborted(options?.signal);
-		if (result.length !== inputs.length) throw new Error(`CT2 translator returned ${result.length} outputs for ${inputs.length} inputs.`);
-		for (const [index, entry] of result.entries()) yield {
-			index,
-			output: entry.text
-		};
+		for (const [index, input] of inputs.entries()) {
+			const controlled = await runControlledTranslationTask(async (signal) => {
+				throwIfAborted(signal);
+				const result = await this.translator.translateBatch([input], {
+					beamSize: this.runtimeConfig.beamSize ?? this.factoryOptions.beamSize,
+					maxBatchSize: this.runtimeConfig.maxBatchSize ?? this.factoryOptions.maxBatchSize,
+					returnScores: false
+				});
+				throwIfAborted(signal);
+				if (result.length !== 1) throw new Error(`CT2 translator returned ${result.length} outputs for 1 input.`);
+				return result[0]?.text ?? "";
+			}, options);
+			if (controlled.ok) {
+				yield {
+					index,
+					output: controlled.value
+				};
+				continue;
+			}
+			yield {
+				index,
+				error: controlled.error
+			};
+		}
 	}
 };
 async function createRuntimeTranslator(options, modelPath, runtimeConfig) {

package/dist/{src-8bHfOTUF.mjs → src-cslnzFaN.mjs}

@@ -1,3 +1,3 @@
-import { a as resolveCt2ModelDownloadPlanFromRepositoryFiles, i as CT2_REQUIRED_FILE_NAMES, n as createLocalCt2TranslatorFactory, r as CT2_OPTIONAL_FILE_NAMES, t as LocalCt2TranslatorFactory } from "./src-BJ-K9Dp2.mjs";
+import { a as resolveCt2ModelDownloadPlanFromRepositoryFiles, i as CT2_REQUIRED_FILE_NAMES, n as createLocalCt2TranslatorFactory, r as CT2_OPTIONAL_FILE_NAMES, t as LocalCt2TranslatorFactory } from "./src-Dh_UAz5C.mjs";
 
 export { createLocalCt2TranslatorFactory };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openspecui/server",
-  "version": "3.11.3",
+  "version": "3.11.4",
   "type": "module",
   "main": "dist/index.mjs",
   "exports": {
@@ -24,8 +24,8 @@
     "@hono/node-server": "^1.14.1",
     "@huggingface/hub": "^2.12.0",
     "@lydell/node-pty": "^1.1.0",
-    "@openspecui/core": "3.11.3",
-    "@openspecui/search": "3.11.3",
+    "@openspecui/core": "3.11.4",
+    "@openspecui/search": "3.11.4",
     "@trpc/server": "^11.0.0",
     "better-sqlite3": "^12.5.0",
     "hono": "^4.7.3",

package/dist/src-Cc9NSywS.mjs

@@ -1,3 +0,0 @@
-import { n as LocalLlamaTranslatorFactory, r as createLocalLlamaTranslatorFactory, t as resolveGgufModelDownloadPlanFromRepositoryFiles } from "./src-awZ9aP1s.mjs";
-
-export { createLocalLlamaTranslatorFactory };