arcane-os 0.94.0 → 0.95.0

package/src/local-ai/magpie/native/main.cpp

@@ -0,0 +1,270 @@
+#include <nemo_speech/tts.h>
+
+#include <atomic>
+#include <condition_variable>
+#include <cstdint>
+#include <exception>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#ifdef _WIN32
+#include <fcntl.h>
+#include <io.h>
+#include <windows.h>
+#endif
+
+namespace {
+
+using Synthesizer = std::unique_ptr<nemo_speech_tts_synthesizer, decltype(&nemo_speech_tts_destroy)>;
+
+struct Utterance {
+    uint64_t id;
+    int32_t speaker;
+    std::string language;
+    std::string text;
+    std::atomic<bool> cancelled{false};
+};
+
+std::mutex output_mutex;
+
+// Lengths belong only to this binary transport. Every PCM and text frame is complete.
+void emit(const char* kind, uint64_t id, const char* content, size_t length) {
+    const std::lock_guard<std::mutex> lock(output_mutex);
+    std::cout << kind << '\t' << id << '\t' << length << '\n';
+    std::cout.write(content, static_cast<std::streamsize>(length));
+    std::cout.flush();
+    if (!std::cout) throw std::runtime_error("The Magpie output pipe closed.");
+}
+
+void emit(const char* kind, uint64_t id, const std::string& content = {}) {
+    emit(kind, id, content.data(), content.size());
+}
+
+struct Output {
+    Utterance& utterance;
+    std::string error;
+};
+
+bool deliver_audio(const uint8_t* pcm, size_t length, void* context) {
+    auto& output = *static_cast<Output*>(context);
+    if (output.utterance.cancelled.load()) return false;
+    try {
+        emit("audio", output.utterance.id, reinterpret_cast<const char*>(pcm), length);
+        return !output.utterance.cancelled.load();
+    } catch (const std::exception& error) {
+        output.error = error.what();
+        return false;
+    }
+}
+
+std::string read_text(size_t length) {
+    std::string text(length, '\0');
+    if (!std::cin.read(text.data(), static_cast<std::streamsize>(length))) {
+        throw std::runtime_error("The Magpie input pipe ended during a text frame.");
+    }
+    return text;
+}
+
+int serve(const std::vector<std::string>& arguments) {
+#ifdef _WIN32
+    _setmode(_fileno(stdin), _O_BINARY);
+    _setmode(_fileno(stdout), _O_BINARY);
+#endif
+    if (arguments.size() != 6) {
+        throw std::runtime_error("Usage: arcane-magpie MAGPIE.gguf CODEC.gguf TOKENIZER_DIR SPEAKER cpu|cuda");
+    }
+    const std::string& backend = arguments[5];
+    if (backend != "cpu" && backend != "cuda") {
+        throw std::runtime_error("Select the prepared CPU or CUDA TTS backend explicitly.");
+    }
+    nemo_speech_tts_model_config models = {};
+    models.size = sizeof(models);
+    models.magpie_model = arguments[1].c_str();
+    models.codec_model = arguments[2].c_str();
+    models.tokenizer_model_dir = arguments[3].c_str();
+    // Leave optional written-to-spoken normalization absent: the caller owns its text.
+    auto runtime = nemo_speech_tts_runtime_config_default();
+    runtime.speaker = std::stoi(arguments[4]);
+    runtime.lt_backend = backend == "cuda" ? NEMO_SPEECH_TTS_BACKEND_CUDA : NEMO_SPEECH_TTS_BACKEND_CPU;
+    runtime.sampling_backend = runtime.lt_backend;
+    runtime.codec_cpu = backend == "cpu";
+    nemo_speech_tts_synthesizer_config configuration = {};
+    configuration.size = sizeof(configuration);
+    configuration.model = &models;
+    configuration.runtime = &runtime;
+    nemo_speech_tts_synthesizer* raw = nullptr;
+    if (nemo_speech_tts_create(&configuration, &raw) != NEMO_SPEECH_TTS_OK) {
+        const char* error = nemo_speech_tts_last_error();
+        throw std::runtime_error(error && *error ? error : "The native TTS model creation failed without an error message.");
+    }
+    const Synthesizer synthesizer(raw, nemo_speech_tts_destroy);
+    emit("ready", 0, std::to_string(nemo_speech_tts_sample_rate(synthesizer.get())));
+
+    std::mutex work_mutex;
+    std::condition_variable work_ready;
+    std::shared_ptr<Utterance> pending;
+    std::shared_ptr<Utterance> active;
+    bool closing = false;
+    std::exception_ptr worker_error;
+
+    // Only this worker calls the synthesizer. The reader can flag the exact
+    // active utterance while inference or PCM backpressure occupies that worker.
+    std::thread worker([&]() {
+        try {
+            for (;;) {
+                std::shared_ptr<Utterance> utterance;
+                {
+                    std::unique_lock<std::mutex> lock(work_mutex);
+                    work_ready.wait(lock, [&]() { return closing || pending != nullptr; });
+                    if (!pending) return;
+                    utterance = std::move(pending);
+                    active = utterance;
+                }
+                auto options = nemo_speech_tts_synthesis_options_default();
+                const std::string request_id = std::to_string(utterance->id);
+                options.request_id = request_id.c_str();
+                options.language_code = utterance->language.c_str();
+                options.speaker = utterance->speaker;
+                Output output{*utterance, {}};
+                nemo_speech_tts_status status = NEMO_SPEECH_TTS_ERROR_CANCELLED;
+                if (!utterance->cancelled.load()) {
+                    emit("started", utterance->id);
+                    status = nemo_speech_tts_synthesize_text(
+                        synthesizer.get(), &options, utterance->text.c_str(), deliver_audio, &output, nullptr
+                    );
+                }
+                // Read the thread-local native error before any other native call.
+                std::string native_error;
+                if (status != NEMO_SPEECH_TTS_OK && status != NEMO_SPEECH_TTS_ERROR_CANCELLED) {
+                    const char* message = nemo_speech_tts_last_error();
+                    native_error = message && *message ? message
+                        : "The native TTS call failed with status " + std::to_string(status) + ".";
+                }
+                {
+                    const std::lock_guard<std::mutex> lock(work_mutex);
+                    active.reset();
+                }
+                if (!output.error.empty()) emit("error", utterance->id, output.error);
+                else if (!native_error.empty()) emit("error", utterance->id, native_error);
+                else if (utterance->cancelled.load() || status == NEMO_SPEECH_TTS_ERROR_CANCELLED) {
+                    emit("cancelled", utterance->id);
+                } else emit("done", utterance->id);
+            }
+        } catch (...) {
+            const auto failure = std::current_exception();
+            {
+                const std::lock_guard<std::mutex> lock(work_mutex);
+                worker_error = failure;
+                closing = true;
+            }
+            // A failed worker must wake the process owner, which closes input
+            // and joins this process instead of leaving an unanswered request.
+            try {
+                try { std::rethrow_exception(failure); }
+                catch (const std::exception& error) { emit("fatal", 0, error.what()); }
+                catch (...) { emit("fatal", 0, "The native synthesis worker raised an unknown exception."); }
+            } catch (...) {
+                // A closed output pipe cannot carry another diagnostic.
+            }
+        }
+    });
+
+    std::exception_ptr reader_error;
+    try {
+        std::string line;
+        while (std::getline(std::cin, line)) {
+            std::istringstream command(line);
+            std::string operation;
+            uint64_t id = 0;
+            if (!(command >> operation >> id)) throw std::runtime_error("Unreadable Magpie command.");
+            if (operation == "synthesize") {
+                int32_t speaker = 0;
+                size_t language_length = 0;
+                size_t text_length = 0;
+                if (!(command >> speaker >> language_length >> text_length)) {
+                    throw std::runtime_error("Unreadable Magpie synthesis framing.");
+                }
+                auto utterance = std::make_shared<Utterance>();
+                utterance->id = id;
+                utterance->speaker = speaker;
+                utterance->language = read_text(language_length);
+                utterance->text = read_text(text_length);
+                if (utterance->language.find('\0') != std::string::npos || utterance->text.find('\0') != std::string::npos) {
+                    emit("error", id, "The public TTS text interface cannot consume embedded NUL characters.");
+                    continue;
+                }
+                {
+                    const std::lock_guard<std::mutex> lock(work_mutex);
+                    if (worker_error) std::rethrow_exception(worker_error);
+                    if (closing || pending || active) {
+                        emit("error", id, "The native synthesizer already owns an utterance.");
+                        continue;
+                    }
+                    pending = utterance;
+                }
+                work_ready.notify_one();
+            } else if (operation == "stop") {
+                const std::lock_guard<std::mutex> lock(work_mutex);
+                if (pending && pending->id == id) pending->cancelled.store(true);
+                if (active && active->id == id) active->cancelled.store(true);
+            } else if (operation == "close") {
+                break;
+            } else {
+                emit("error", id, "Unknown Magpie command.");
+            }
+        }
+    } catch (...) {
+        reader_error = std::current_exception();
+    }
+    {
+        const std::lock_guard<std::mutex> lock(work_mutex);
+        closing = true;
+        if (pending) pending->cancelled.store(true);
+        if (active) active->cancelled.store(true);
+    }
+    work_ready.notify_one();
+    worker.join();
+    if (reader_error) std::rethrow_exception(reader_error);
+    if (worker_error) std::rethrow_exception(worker_error);
+    return 0;
+}
+
+int run(const std::vector<std::string>& arguments) {
+    try {
+        return serve(arguments);
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}
+
+} // namespace
+
+#ifdef _WIN32
+int wmain(int argc, wchar_t** argv) {
+    std::vector<std::string> arguments;
+    for (int index = 0; index < argc; ++index) {
+        const int length = WideCharToMultiByte(CP_UTF8, 0, argv[index], -1, nullptr, 0, nullptr, nullptr);
+        if (length == 0) {
+            std::cerr << "Windows could not encode the selected TTS argument as UTF-8.\n";
+            return 1;
+        }
+        std::string value(length, '\0');
+        WideCharToMultiByte(CP_UTF8, 0, argv[index], -1, value.data(), length, nullptr, nullptr);
+        value.pop_back();
+        arguments.push_back(std::move(value));
+    }
+    return run(arguments);
+}
+#else
+int main(int argc, char** argv) {
+    return run(std::vector<std::string>(argv, argv + argc));
+}
+#endif

package/src/local-ai/nemotron/build.mjs

@@ -0,0 +1,53 @@
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {mkdir} from 'node:fs/promises';
+import {runProcess} from '../../process.mjs';
+
+/** Build the first-party ASR helper against the caller's installed NeMo SDK. */
+export async function buildNemotronHelper({runtime, outputRoot, signal, onEvent} = {}) {
+    if (typeof runtime?.cmakeDirectory !== 'string' || typeof outputRoot !== 'string') {
+        throw new TypeError('Nemotron requires the selected runtime cmakeDirectory and helper outputRoot.');
+    }
+    const root = path.resolve(outputRoot);
+    const build = path.join(root, 'build');
+    const source = fileURLToPath(
+        new URL('./native/', import.meta.url)
+    );
+    await mkdir(
+        root,
+        {recursive: true}
+    );
+    const options = {signal, onEvent};
+    const configure = await runProcess(
+        'cmake',
+        [
+            '-S', source,
+            '-B', build,
+            `-DNeMoSpeech_DIR=${path.resolve(runtime.cmakeDirectory)}`,
+            `-DCMAKE_INSTALL_PREFIX=${root}`,
+            '-DCMAKE_BUILD_TYPE=Release'
+        ],
+        options
+    );
+    const compile = await runProcess(
+        'cmake',
+        ['--build', build, '--config', 'Release'],
+        options
+    );
+    const install = await runProcess(
+        'cmake',
+        ['--install', build, '--config', 'Release'],
+        options
+    );
+    const executable = path.join(root, 'bin', process.platform === 'win32' ? 'arcane-nemotron.exe' : 'arcane-nemotron');
+    return {
+        platform: process.platform,
+        architecture: process.arch,
+        root,
+        executable,
+        helperExecutable: executable,
+        diagnostics: {configure, compile, install}
+    };
+}
+
+export default buildNemotronHelper;

package/src/local-ai/nemotron/helper.mjs

@@ -0,0 +1,308 @@
+import {PassThrough} from 'node:stream';
+import path from 'node:path';
+import {runProcess} from '../../process.mjs';
+
+/** One owned process drives the public ASR C ABI on its single native thread. */
+export function createNemotronHelper(
+    {runtime, modelPath, diarizationModelPath, gpu, onEvent}
+) {
+    const input = new PassThrough();
+    const controller = new AbortController();
+    const readiness = deferred();
+    const pending = new Map();
+    const running = new Set();
+    let fragment = '';
+    let sequence = 0;
+    let writeTail = Promise.resolve();
+    let stopping = false;
+    let stopped = null;
+    let failure = null;
+    let metadata = null;
+    let closingStream = null;
+
+    function rejectPending(error) {
+        const results = [];
+        for (const [request, operation] of pending) {
+            if (operation.results.length) {
+                results.push(
+                    {request, results: operation.results}
+                );
+            }
+        }
+        const completeFailure = results.length || fragment !== ''
+            ? nemotronError(
+                error.code ?? 'NEMOTRON_HELPER_FAILED',
+                error.message ?? 'The Nemotron helper failed.',
+                {cause: error, pendingResults: results, nativeFragment: fragment}
+            )
+            : error;
+        readiness.reject(completeFailure);
+        for (const operation of pending.values()) operation.response.reject(completeFailure);
+        pending.clear();
+        running.clear();
+        return completeFailure;
+    }
+
+    function receive({stream, chunk}) {
+        if (stream !== 'stdout') return;
+        fragment += chunk;
+        let boundary = fragment.indexOf('\n');
+        while (boundary !== -1) {
+            const line = fragment.substring(0, boundary);
+            fragment = fragment.substring(boundary + 1);
+            if (line !== '') {
+                let message;
+                try {
+                    message = JSON.parse(line);
+                } catch (cause) {
+                    throw nemotronError(
+                        'NEMOTRON_PROTOCOL_UNREADABLE',
+                        'The Nemotron helper returned an unreadable record.',
+                        {line, cause}
+                    );
+                }
+                if (message.ready) {
+                    metadata = message;
+                    readiness.resolve(message);
+                } else if (message.request === 0 && message.error) {
+                    const error = nemotronError('NEMOTRON_MODEL_FAILED', message.error);
+                    rejectPending(error);
+                } else {
+                    const operation = pending.get(message.request);
+                    if (!operation) {
+                        throw nemotronError('NEMOTRON_PROTOCOL_UNKNOWN_REQUEST', 'The Nemotron helper returned an unknown request.', message);
+                    }
+                    if (message.running) {
+                        running.add(message.request);
+                        closingStream?.interrupt();
+                    } else if (message.update) {
+                        operation.results.push(message.update);
+                    } else {
+                        running.delete(message.request);
+                        pending.delete(message.request);
+                        if (message.error) {
+                            operation.response.reject(
+                                nemotronError(
+                                    'NEMOTRON_NATIVE_FAILED',
+                                    message.error,
+                                    {message, results: operation.results}
+                                )
+                            );
+                        } else {
+                            operation.response.resolve(operation.results);
+                        }
+                    }
+                }
+            }
+            boundary = fragment.indexOf('\n');
+        }
+    }
+
+    const completion = runProcess(
+        runtime.helperExecutable,
+        [modelPath, diarizationModelPath ?? '', String(gpu)],
+        {
+            env: runtimeEnvironment(runtime),
+            signal: controller.signal,
+            input: {
+                [Symbol.asyncIterator]: function helperInput() {
+                    const iterator = input[Symbol.asyncIterator]();
+                    return {
+                        next: function nextInput() { return iterator.next(); },
+                        return: function closeInput() {
+                            input.end();
+                            return iterator.return();
+                        }
+                    };
+                }
+            },
+            onOutput: receive,
+            captureOutput: {stdout: false, stderr: true},
+            emitOutputEvents: {stdout: false, stderr: true},
+            onEvent
+        }
+    ).then(
+        function helperExited(result) {
+            if (fragment !== '') {
+                throw nemotronError(
+                    'NEMOTRON_PROTOCOL_INCOMPLETE',
+                    'The Nemotron helper exited during a record.',
+                    {fragment, result}
+                );
+            }
+            if (!stopping || pending.size !== 0 || !metadata) {
+                throw nemotronError('NEMOTRON_HELPER_EXITED', 'The Nemotron helper exited before its operation completed.', result);
+            }
+            return result;
+        }
+    ).catch(
+        function helperFailed(error) {
+            failure = rejectPending(error);
+            throw failure;
+        }
+    ).finally(
+        function helperDrained() {
+            input.end();
+        }
+    );
+    completion.catch(
+        function observeHelperFailure() {}
+    );
+
+    function send(operation, parameters = '', payload) {
+        const response = deferred();
+        const id = ++sequence;
+        if (failure || stopping) {
+            response.reject(failure ?? nemotronError('NEMOTRON_HELPER_CLOSING', 'The Nemotron helper is closing.'));
+            return {response: response.promise, written: Promise.resolve()};
+        }
+        pending.set(
+            id,
+            {response, operation, results: []}
+        );
+        const written = writeTail.then(
+            async function writeCommand() {
+                if (failure) throw failure;
+                await writeInput(input, `${operation} ${id}${parameters ? ` ${parameters}` : ''}\n`);
+                if (payload !== undefined) await writeInput(input, payload);
+            }
+        );
+        writeTail = written.catch(
+            function commandWriteFailed(error) {
+                failure = rejectPending(error);
+                input.end();
+                controller.abort(failure);
+            }
+        );
+        return {response: response.promise, written};
+    }
+
+    function command(operation, parameters, payload) {
+        return send(operation, parameters, payload).response;
+    }
+
+    function stop(reason) {
+        if (stopped) return stopped;
+        stopping = true;
+        // EOF is the ordinary native shutdown path. Only a call already inside
+        // the noninterruptible ABI (including model creation) needs termination.
+        const noninterruptible = !metadata || running.size !== 0;
+        if (noninterruptible) {
+            input.end();
+            controller.abort(reason);
+        }
+        async function stopOwnedHelper() {
+            if (!noninterruptible) {
+                await writeTail;
+                input.end();
+            }
+            return completion;
+        }
+        stopped = stopOwnedHelper().catch(
+            function stoppedHelper(error) {
+                if (controller.signal.aborted && error.code === 'ARCANE_CANCELLED') {
+                    return {cancelled: true, resetRequired: true, diagnostics: error};
+                }
+                throw error;
+            }
+        );
+        return stopped;
+    }
+
+    async function closeStream(reason) {
+        if (failure) throw failure;
+        // Attempt the public stream close first. A running decode cannot read
+        // that command until it returns; terminate only this owned process then.
+        const closing = send('close');
+        let written = false;
+        let interrupted = false;
+        const cancellation = {
+            interrupt: function interruptActiveDecode() {
+                if (!written || running.size === 0 || interrupted) return;
+                interrupted = true;
+                stop(reason ?? nemotronError('NEMOTRON_CANCELLED', 'The active Nemotron decode was cancelled.')).catch(
+                    function observeOwnedStop() {}
+                );
+            }
+        };
+        closingStream = cancellation;
+        try {
+            await closing.written;
+            written = true;
+            cancellation.interrupt();
+            try {
+                await closing.response;
+            } catch (error) {
+                if (!interrupted) throw error;
+            }
+            if (interrupted) return {resetRequired: true, diagnostics: await stopped};
+            return {resetRequired: false};
+        } finally {
+            if (closingStream === cancellation) closingStream = null;
+        }
+    }
+
+    return {
+        ready: readiness.promise,
+        completion,
+        command,
+        closeStream,
+        stop,
+        current: function helperState() { return {metadata, failure, running: running.size !== 0, stopping}; }
+    };
+}
+
+export function encodeNemotronAudio(samples) {
+    // This is only the native transport's little-endian float32 framing.
+    const encoded = Buffer.allocUnsafe(samples.length * 4);
+    const bits = new Uint32Array(samples.buffer, samples.byteOffset, samples.length);
+    for (let index = 0; index < samples.length; index++) encoded.writeUInt32LE(bits[index], index * 4);
+    return encoded;
+}
+
+function runtimeEnvironment(runtime) {
+    const environment = {...process.env, ...runtime.env};
+    const key = process.platform === 'win32'
+        ? Object.keys(environment).find(
+            function pathKey(name) { return name.toLowerCase() === 'path'; }
+        ) ?? 'PATH'
+        : process.platform === 'darwin' ? 'DYLD_LIBRARY_PATH' : 'LD_LIBRARY_PATH';
+    const directories = [runtime.binaryDirectory, runtime.libraryDirectory].filter(Boolean);
+    if (directories.length) environment[key] = [...directories, environment[key]].filter(Boolean).join(path.delimiter);
+    return environment;
+}
+
+function writeInput(input, value) {
+    return new Promise(
+        function inputWrite(resolve, reject) {
+            input.write(
+                value,
+                function inputWritten(error) {
+                    if (error) reject(error);
+                    else resolve();
+                }
+            );
+        }
+    );
+}
+
+export function deferred() {
+    const result = {};
+    result.promise = new Promise(
+        function pending(resolve, reject) {
+            result.resolve = resolve;
+            result.reject = reject;
+        }
+    );
+    result.promise.catch(
+        function observePendingFailure() {}
+    );
+    return result;
+}
+
+export function nemotronError(code, message, details) {
+    const error = new Error(message);
+    error.code = code;
+    if (details !== undefined) error.details = details;
+    return error;
+}