@agentshouse/kit 0.2.1-alpha.8 → 0.2.1-c02dacffca87c

package/dist/transcription.js

@@ -0,0 +1,210 @@
+import { chmod, mkdir, mkdtemp, readdir, readFile, rename, rm, stat } from 'node:fs/promises';
+import { availableParallelism, tmpdir } from 'node:os';
+import { basename, join } from 'node:path';
+import { unlessAborted } from './abort.js';
+import { downloadServed } from './building.js';
+import { failureOf, run } from './clis.js';
+import { downloaded, fetched } from './files.js';
+import { kitHome } from './home.js';
+const RELEASE = 'https://github.com/agentshouse/kit/releases/download/engine-2';
+export const MODEL = {
+    name: 'ggml-parakeet-tdt-0.6b-v3-q4_k.bin',
+    from: 'https://huggingface.co/ggml-org/parakeet-GGUF/resolve/35156454d1a39de06863303dd209fd2bed6ee079/ggml-parakeet-tdt-0.6b-v3-q4_k.bin',
+    sha256: '8b205b8b39c6535e153de6fb11c51db46125d45c4f16ba496fe41a0fe71b885e',
+    bytes: 415_611_879,
+    executable: false,
+};
+export const ENGINES = {
+    'linux-x64': {
+        decoder: { name: 'ffmpeg-linux-x86_64', from: `${RELEASE}/ffmpeg-linux-x86_64`, sha256: '1f8235d206428305afb89c6e8ab99d100425a0ba61e22bb4c7aff017db8c8de5', bytes: 5_175_832, executable: true },
+        engine: { name: 'parakeet-cli-linux-x86_64', from: `${RELEASE}/parakeet-cli-linux-x86_64`, sha256: 'bb2585695992fe1927a33960002cdf51b8a90a96ad65593c3d0a9f1fbc0865c2', bytes: 3_801_616, executable: true },
+    },
+    'linux-arm64': {
+        decoder: { name: 'ffmpeg-linux-aarch64', from: `${RELEASE}/ffmpeg-linux-aarch64`, sha256: 'a7e08a0fae0179d1078312f6bb13d2a9aa4d4c71a1d82a022953f25dee452f5e', bytes: 4_590_208, executable: true },
+        engine: { name: 'parakeet-cli-linux-aarch64', from: `${RELEASE}/parakeet-cli-linux-aarch64`, sha256: 'cbc0bedc8daae97f1ef9dcee4200c06226218e300e0fd62ead1306d22742b7ed', bytes: 3_477_624, executable: true },
+    },
+    'darwin-arm64': {
+        decoder: { name: 'ffmpeg-darwin-arm64', from: `${RELEASE}/ffmpeg-darwin-arm64`, sha256: '7abe154aaed3d6b49f5167ca7221bb66bed51388925d99a7e4261a41fe52fba6', bytes: 3_696_768, executable: true },
+        engine: { name: 'parakeet-cli-darwin-arm64', from: `${RELEASE}/parakeet-cli-darwin-arm64`, sha256: '9a2aafc0922ad7c0b6db6eff1641e96572ba32ac40bf995a1df242ed2febf297', bytes: 1_939_728, executable: true },
+    },
+};
+const CHUNK_SECONDS = 20;
+const OFF = "transcription is off on the Agent's Environment, so this message's audio has no text; turn it on or write instead";
+const NOT_INSTALLED = "the speech engine is not installed on the Agent's Environment, so this message's audio has no text";
+const NOT_DOWNLOADED = 'an audio file this message links could not be downloaded; send the message again';
+const UNDECODABLE = 'an audio file this message links could not be decoded';
+const UNTRANSCRIBED = 'an audio file this message links could not be transcribed';
+let held = Promise.resolve();
+let holding = Promise.resolve();
+let setting = null;
+let downloading = null;
+let readiness = { installed: false, failure: null };
+function directory() {
+    return join(kitHome(), 'transcription');
+}
+function installed(artifact) {
+    return join(directory(), artifact.sha256);
+}
+async function executed(command, args, signal) {
+    // A decode or a transcription runs as long as its audio needs; the setting turning off or the turn's interrupt or end aborts it instead.
+    const ran = await run(command, args, 0, false, signal);
+    if (ran.status !== 0)
+        throw new Error(failureOf(ran, basename(command)));
+    return ran.stdout;
+}
+function present(path) {
+    return stat(path).then(() => true, () => false);
+}
+async function threads() {
+    const [quota, period] = (await readFile('/sys/fs/cgroup/cpu.max', 'utf8').catch(() => 'max')).split(' ');
+    const allowed = quota === 'max' ? Infinity : Math.ceil(Number(quota) / Number(period));
+    return Math.max(1, Math.floor(Math.min(availableParallelism(), allowed) / 2));
+}
+function causeOf(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+async function failing(reason, work) {
+    try {
+        return await work;
+    }
+    catch (error) {
+        throw new Error(reason, { cause: error });
+    }
+}
+function ready() {
+    if (setting !== true)
+        throw new Error(OFF);
+    if (!readiness.installed)
+        throw new Error(NOT_INSTALLED);
+}
+function pinnedHere() {
+    return ENGINES[`${process.platform}-${process.arch}`];
+}
+const DOWNLOADS = 3;
+async function downloadedAgain(artifact, signal) {
+    // An artifact arrives at a hundred kilobytes a second or its download ends, so a stalled Building cannot hold the install open.
+    const deadlineMs = Math.ceil(artifact.bytes / 100);
+    const deadline = AbortSignal.timeout(deadlineMs);
+    for (let attempt = 1;; attempt++) {
+        try {
+            return await downloadServed(artifact.name, artifact.sha256, `${installed(artifact)}.part`, AbortSignal.any([signal, deadline]), artifact.bytes);
+        }
+        catch (error) {
+            if (signal.aborted)
+                throw error;
+            if (deadline.aborted)
+                throw new Error(`${artifact.name} did not arrive within ${Math.ceil(deadlineMs / 1000)} seconds`);
+            if (attempt === DOWNLOADS)
+                throw error;
+        }
+    }
+}
+async function install(artifacts, signal) {
+    await mkdir(directory(), { recursive: true });
+    const pinned = new Set(artifacts.map((artifact) => artifact.sha256));
+    for (const name of await readdir(directory())) {
+        if (!pinned.has(name))
+            await rm(join(directory(), name), { recursive: true, force: true });
+    }
+    const missing = [];
+    for (const artifact of artifacts)
+        if (!(await present(installed(artifact))))
+            missing.push(artifact);
+    try {
+        for (const artifact of missing)
+            await downloadedAgain(artifact, signal);
+        for (const artifact of missing) {
+            if (artifact.executable)
+                await chmod(`${installed(artifact)}.part`, 0o755);
+            await rename(`${installed(artifact)}.part`, installed(artifact));
+        }
+    }
+    finally {
+        for (const artifact of missing)
+            await rm(`${installed(artifact)}.part`, { force: true });
+    }
+}
+function reported(next) {
+    const changed = next.installed !== readiness.installed || next.failure !== readiness.failure;
+    readiness = next;
+    return changed;
+}
+async function settle(transcribe) {
+    if (transcribe === setting)
+        return false;
+    setting = transcribe;
+    if (!transcribe) {
+        await rm(directory(), { recursive: true, force: true });
+        return reported({ installed: false, failure: null });
+    }
+    const download = new AbortController();
+    downloading = download;
+    try {
+        const pinned = pinnedHere();
+        await install([pinned.decoder, pinned.engine, MODEL], download.signal);
+        return reported({ installed: true, failure: null });
+    }
+    catch (error) {
+        await rm(directory(), { recursive: true, force: true });
+        return reported({ installed: false, failure: download.signal.aborted ? null : causeOf(error) });
+    }
+    finally {
+        downloading = null;
+    }
+}
+export function holdTranscription(transcribe) {
+    if (!transcribe)
+        downloading?.abort();
+    const settled = holding.then(() => settle(transcribe));
+    holding = settled.catch(() => undefined);
+    return settled;
+}
+export function transcriptionReport() {
+    return readiness;
+}
+async function transcribed(audio, parts, signal) {
+    const pinned = pinnedHere();
+    await mkdir(parts);
+    await failing(UNDECODABLE, executed(installed(pinned.decoder), ['-nostdin', '-hide_banner', '-loglevel', 'error', '-i', audio, '-ar', '16000', '-ac', '1', '-c:a', 'pcm_s16le',
+        '-f', 'segment', '-segment_time', String(CHUNK_SECONDS), join(parts, '%05d.wav')], signal));
+    const segments = (await readdir(parts)).sort().flatMap((part) => ['-f', join(parts, part)]);
+    const said = await failing(UNTRANSCRIBED, executed(installed(pinned.engine), ['-t', String(await threads()), '-np', '-m', installed(MODEL), ...segments], signal));
+    const text = said
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line !== '')
+        .join(' ');
+    if (text === '')
+        throw new Error(UNTRANSCRIBED);
+    return text;
+}
+async function alone(work, signal) {
+    const before = held;
+    let release;
+    const mine = new Promise((resolve) => {
+        release = resolve;
+    });
+    held = Promise.all([before, mine]);
+    try {
+        await unlessAborted(before, signal);
+        return await work();
+    }
+    finally {
+        release();
+    }
+}
+export async function transcribe(address, signal) {
+    ready();
+    const work = await mkdtemp(join(tmpdir(), 'kit-transcription-'));
+    try {
+        const audio = join(work, 'audio');
+        await failing(NOT_DOWNLOADED, downloaded(address, (url) => fetched(url, signal), audio, 'audio'));
+        return await alone(() => {
+            ready();
+            return transcribed(audio, join(work, 'parts'), signal);
+        }, signal);
+    }
+    finally {
+        await rm(work, { recursive: true, force: true });
+    }
+}

package/dist/translate.js

@@ -3,14 +3,8 @@ function same(left, right) {
     return JSON.stringify(left) === JSON.stringify(right);
 }
 function applied(document, changes) {
-    let current = document;
-    for (const change of changes) {
-        const result = apply(current, change);
-        if (result.outcome !== 'applied')
-            return null;
-        current = result.document;
-    }
-    return serialize(current);
+    const result = apply(document, changes);
+    return result.outcome === 'applied' ? serialize(result.document) : null;
 }
 function components(path, before, after, text) {
     const edits = [];

package/dist/update.js

@@ -0,0 +1,127 @@
+import { spawn } from 'node:child_process';
+import { mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { downloadServed, servedRelease } from './building.js';
+import { KIT_RELEASE } from './clis.js';
+import { restart, updateLine } from './end.js';
+import { kitHome, readEnrolment } from './home.js';
+import { placedPrefix } from './placement.js';
+import { childEnvironment, relayOrigin } from './relay.js';
+const INSTALLER_BYTES = 1_000_000;
+// A release's Node.js, Kit package and adapters are about a hundred megabytes, so an hour carries them at thirty kilobytes a second, and a placement still running then has hung.
+const PLACEMENT_MS = 60 * 60_000;
+// A refused Kit is offline until its placement succeeds, so it tries again within five minutes of its Building or this computer recovering, while a placement that fails only after its hundred-megabyte download runs at most twelve times an hour.
+export const PLACEMENT_RETRY_MS = 5 * 60_000;
+const REFUSED = 'kit_bootstrap_refused: ';
+const OUTPUT_CHARACTERS = 4000;
+function causeOf(error) {
+    if (error instanceof TypeError && error.cause instanceof Error)
+        return `the Building did not answer: ${error.cause.message}`;
+    return error instanceof Error ? error.message : String(error);
+}
+function placing(installer, house, prefix, signal) {
+    return new Promise((resolve) => {
+        const relay = relayOrigin();
+        const child = spawn('bash', [installer, '--house', house, '--place', prefix], {
+            stdio: ['ignore', 'pipe', 'pipe'],
+            env: { ...childEnvironment(), ...(relay === undefined ? {} : { HOUSE_KIT_RELAY: relay }) },
+            detached: true,
+        });
+        const stop = () => {
+            try {
+                process.kill(-child.pid, 'SIGKILL');
+            }
+            catch { }
+        };
+        signal.addEventListener('abort', stop, { once: true });
+        if (signal.aborted)
+            stop();
+        let output = '';
+        for (const stream of [child.stdout, child.stderr]) {
+            stream.setEncoding('utf8').on('data', (chunk) => {
+                output = (output + chunk).slice(-OUTPUT_CHARACTERS);
+            });
+        }
+        const ended = (failure) => {
+            signal.removeEventListener('abort', stop);
+            resolve(failure);
+        };
+        child.on('error', (error) => ended(error.message));
+        child.on('close', (status) => {
+            if (status === 0)
+                return ended(null);
+            if (signal.aborted)
+                return ended(`the placement did not end within ${PLACEMENT_MS / 1000} seconds`);
+            const lines = output.split('\n').filter((line) => line.trim() !== '');
+            const refused = lines.findLast((line) => line.startsWith(REFUSED));
+            ended(refused?.slice(REFUSED.length) ?? lines.at(-1) ?? `the placement ended with status ${status ?? 'a signal'}`);
+        });
+    });
+}
+export class KitUpdate {
+    calls;
+    queue = Promise.resolve();
+    placed = null;
+    constructor(calls) {
+        this.calls = calls;
+    }
+    take(call) {
+        void this.calls.carry(call.input_id, async () => ({ result: await this.queued(() => this.update(call.release)) }));
+    }
+    async takeRefused(release) {
+        for (;;) {
+            const failure = await this.queued(async () => (this.placed === null ? this.place(release, 'served') : null));
+            if (failure === null)
+                restart(`Kit ${this.placed} is placed; its supervisor starts it`);
+            process.stderr.write(`kit: ${updateLine(`Kit ${KIT_RELEASE} stays offline and tries again in ${PLACEMENT_RETRY_MS / 60_000} minutes`)}\n`);
+            await new Promise((resolve) => setTimeout(resolve, PLACEMENT_RETRY_MS));
+        }
+    }
+    placedRelease() {
+        return this.placed;
+    }
+    queued(work) {
+        const taken = this.queue.then(work);
+        this.queue = taken.catch(() => undefined);
+        return taken;
+    }
+    async update(release) {
+        if (release === (this.placed ?? KIT_RELEASE))
+            return {};
+        if (this.placed !== null)
+            return { refused: `Kit ${this.placed} is placed and starts first, once no turn is open` };
+        const failure = await this.place(release, 'named');
+        if (failure !== null)
+            return { refused: failure };
+        process.stderr.write(`kit: Kit ${release} is placed; it starts once no turn is open\n`);
+        return {};
+    }
+    async place(release, which) {
+        const signal = AbortSignal.timeout(PLACEMENT_MS);
+        let version = release;
+        let failure;
+        try {
+            const prefix = placedPrefix();
+            const served = await servedRelease(signal);
+            if (which === 'named' && served.version !== release)
+                throw new Error(`the Building serves Kit ${served.version}, not Kit ${release}`);
+            version = served.version;
+            if (version === KIT_RELEASE)
+                throw new Error(`the Building serves Kit ${KIT_RELEASE}, which this Kit runs`);
+            await mkdir(join(kitHome(), 'update'), { recursive: true, mode: 0o700 });
+            const installer = join(kitHome(), 'update', served.installer);
+            await downloadServed(served.installer, served.sha256, installer, signal, INSTALLER_BYTES);
+            failure = await placing(installer, (await readEnrolment()).house, prefix, signal);
+        }
+        catch (error) {
+            failure = signal.aborted ? `the placement did not end within ${PLACEMENT_MS / 1000} seconds` : causeOf(error);
+        }
+        if (failure !== null) {
+            const refused = `Kit ${version} could not be placed: ${failure}`;
+            process.stderr.write(`kit: ${refused}\n`);
+            return refused;
+        }
+        this.placed = version;
+        return null;
+    }
+}

package/dist/wire.js

@@ -0,0 +1,40 @@
+import { newerThanHouse, updateRequired } from './end.js';
+import { machine } from './placement.js';
+import { refusalOf } from './refusals.js';
+export const KIT_WIRE = 4;
+export const KIT_WIRE_HEADER = 'x-house-kit-wire';
+let take = null;
+let taken = false;
+function unservedBy(text) {
+    let body;
+    try {
+        body = JSON.parse(text);
+    }
+    catch {
+        return null;
+    }
+    const error = body?.error;
+    if (error?.code !== 'kit_update_required' && error?.code !== 'kit_newer_than_house')
+        return null;
+    return typeof error.release === 'string' ? error : null;
+}
+export function takeUnservedRelease(taking) {
+    take = taking;
+}
+export function unserved() {
+    return taken;
+}
+export function endUnserved(text) {
+    const refused = unservedBy(text);
+    if (refused !== null && take !== null) {
+        if (!taken) {
+            taken = true;
+            take(refused.release);
+        }
+        return;
+    }
+    if (refusalOf(text)?.code === 'kit_update_required')
+        updateRequired();
+    if (refused?.code === 'kit_newer_than_house')
+        newerThanHouse(refused.release, refused.commands[machine()]);
+}