@agentshouse/kit 0.2.1-ce145e9aadbf6 → 0.2.1-ceeaf32286734

package/dist/transcription.js

@@ -1,34 +1,44 @@
-import { createHash } from 'node:crypto';
-import { createReadStream } from 'node:fs';
 import { chmod, mkdir, mkdtemp, readdir, readFile, rename, rm, stat } from 'node:fs/promises';
 import { availableParallelism, tmpdir } from 'node:os';
 import { basename, join } from 'node:path';
-import { pipeline } from 'node:stream/promises';
 import { unlessAborted } from './abort.js';
+import { downloadServed } from './building.js';
 import { failureOf, run } from './clis.js';
+import { downloaded, fetched } from './files.js';
 import { kitHome } from './home.js';
 const RELEASE = 'https://github.com/agentshouse/kit/releases/download/engine-2';
 export const MODEL = {
-    url: 'https://huggingface.co/ggml-org/parakeet-GGUF/resolve/35156454d1a39de06863303dd209fd2bed6ee079/ggml-parakeet-tdt-0.6b-v3-q4_k.bin',
+    name: 'ggml-parakeet-tdt-0.6b-v3-q4_k.bin',
+    from: 'https://huggingface.co/ggml-org/parakeet-GGUF/resolve/35156454d1a39de06863303dd209fd2bed6ee079/ggml-parakeet-tdt-0.6b-v3-q4_k.bin',
     sha256: '8b205b8b39c6535e153de6fb11c51db46125d45c4f16ba496fe41a0fe71b885e',
+    bytes: 415_611_879,
     executable: false,
 };
 export const ENGINES = {
     'linux-x64': {
-        decoder: { url: `${RELEASE}/ffmpeg-linux-x86_64`, sha256: '1f8235d206428305afb89c6e8ab99d100425a0ba61e22bb4c7aff017db8c8de5', executable: true },
-        engine: { url: `${RELEASE}/parakeet-cli-linux-x86_64`, sha256: 'bb2585695992fe1927a33960002cdf51b8a90a96ad65593c3d0a9f1fbc0865c2', executable: true },
+        decoder: { name: 'ffmpeg-linux-x86_64', from: `${RELEASE}/ffmpeg-linux-x86_64`, sha256: '1f8235d206428305afb89c6e8ab99d100425a0ba61e22bb4c7aff017db8c8de5', bytes: 5_175_832, executable: true },
+        engine: { name: 'parakeet-cli-linux-x86_64', from: `${RELEASE}/parakeet-cli-linux-x86_64`, sha256: 'bb2585695992fe1927a33960002cdf51b8a90a96ad65593c3d0a9f1fbc0865c2', bytes: 3_801_616, executable: true },
     },
     'linux-arm64': {
-        decoder: { url: `${RELEASE}/ffmpeg-linux-aarch64`, sha256: 'a7e08a0fae0179d1078312f6bb13d2a9aa4d4c71a1d82a022953f25dee452f5e', executable: true },
-        engine: { url: `${RELEASE}/parakeet-cli-linux-aarch64`, sha256: 'cbc0bedc8daae97f1ef9dcee4200c06226218e300e0fd62ead1306d22742b7ed', executable: true },
+        decoder: { name: 'ffmpeg-linux-aarch64', from: `${RELEASE}/ffmpeg-linux-aarch64`, sha256: 'a7e08a0fae0179d1078312f6bb13d2a9aa4d4c71a1d82a022953f25dee452f5e', bytes: 4_590_208, executable: true },
+        engine: { name: 'parakeet-cli-linux-aarch64', from: `${RELEASE}/parakeet-cli-linux-aarch64`, sha256: 'cbc0bedc8daae97f1ef9dcee4200c06226218e300e0fd62ead1306d22742b7ed', bytes: 3_477_624, executable: true },
     },
     'darwin-arm64': {
-        decoder: { url: `${RELEASE}/ffmpeg-darwin-arm64`, sha256: '7abe154aaed3d6b49f5167ca7221bb66bed51388925d99a7e4261a41fe52fba6', executable: true },
-        engine: { url: `${RELEASE}/parakeet-cli-darwin-arm64`, sha256: '9a2aafc0922ad7c0b6db6eff1641e96572ba32ac40bf995a1df242ed2febf297', executable: true },
+        decoder: { name: 'ffmpeg-darwin-arm64', from: `${RELEASE}/ffmpeg-darwin-arm64`, sha256: '7abe154aaed3d6b49f5167ca7221bb66bed51388925d99a7e4261a41fe52fba6', bytes: 3_696_768, executable: true },
+        engine: { name: 'parakeet-cli-darwin-arm64', from: `${RELEASE}/parakeet-cli-darwin-arm64`, sha256: '9a2aafc0922ad7c0b6db6eff1641e96572ba32ac40bf995a1df242ed2febf297', bytes: 1_939_728, executable: true },
     },
 };
 const CHUNK_SECONDS = 20;
+const OFF = "transcription is off on the Agent's Environment, so this message's audio has no text; turn it on or write instead";
+const NOT_INSTALLED = "the speech engine is not installed on the Agent's Environment, so this message's audio has no text";
+const NOT_DOWNLOADED = 'an audio file this message links could not be downloaded; send the message again';
+const UNDECODABLE = 'an audio file this message links could not be decoded';
+const UNTRANSCRIBED = 'an audio file this message links could not be transcribed';
 let held = Promise.resolve();
+let holding = Promise.resolve();
+let setting = null;
+let downloading = null;
+let readiness = { installed: false, failure: null };
 function directory() {
     return join(kitHome(), 'transcription');
 }
@@ -36,6 +46,7 @@ function installed(artifact) {
     return join(directory(), artifact.sha256);
 }
 async function executed(command, args, signal) {
+    // A decode or a transcription runs as long as its audio needs; the setting turning off or the turn's interrupt or end aborts it instead.
     const ran = await run(command, args, 0, false, signal);
     if (ran.status !== 0)
         throw new Error(failureOf(ran, basename(command)));
@@ -44,15 +55,49 @@ async function executed(command, args, signal) {
 function present(path) {
     return stat(path).then(() => true, () => false);
 }
-async function cpus() {
+async function threads() {
     const [quota, period] = (await readFile('/sys/fs/cgroup/cpu.max', 'utf8').catch(() => 'max')).split(' ');
     const allowed = quota === 'max' ? Infinity : Math.ceil(Number(quota) / Number(period));
-    return Math.min(availableParallelism(), allowed);
+    return Math.max(1, Math.floor(Math.min(availableParallelism(), allowed) / 2));
 }
-async function sha256(path) {
-    const hash = createHash('sha256');
-    await pipeline(createReadStream(path), hash);
-    return hash.digest('hex');
+function causeOf(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+async function failing(reason, work) {
+    try {
+        return await work;
+    }
+    catch (error) {
+        throw new Error(reason, { cause: error });
+    }
+}
+function ready() {
+    if (setting !== true)
+        throw new Error(OFF);
+    if (!readiness.installed)
+        throw new Error(NOT_INSTALLED);
+}
+function pinnedHere() {
+    return ENGINES[`${process.platform}-${process.arch}`];
+}
+const DOWNLOADS = 3;
+async function downloadedAgain(artifact, signal) {
+    // An artifact arrives at a hundred kilobytes a second or its download ends, so a stalled Building cannot hold the install open.
+    const deadlineMs = Math.ceil(artifact.bytes / 100);
+    const deadline = AbortSignal.timeout(deadlineMs);
+    for (let attempt = 1;; attempt++) {
+        try {
+            return await downloadServed(artifact.name, artifact.sha256, `${installed(artifact)}.part`, AbortSignal.any([signal, deadline]), artifact.bytes);
+        }
+        catch (error) {
+            if (signal.aborted)
+                throw error;
+            if (deadline.aborted)
+                throw new Error(`${artifact.name} did not arrive within ${Math.ceil(deadlineMs / 1000)} seconds`);
+            if (attempt === DOWNLOADS)
+                throw error;
+        }
+    }
 }
 async function install(artifacts, signal) {
     await mkdir(directory(), { recursive: true });
@@ -66,12 +111,8 @@ async function install(artifacts, signal) {
         if (!(await present(installed(artifact))))
             missing.push(artifact);
     try {
-        for (const artifact of missing) {
-            const part = `${installed(artifact)}.part`;
-            await executed('curl', ['-fsSL', '--retry', '3', '-o', part, artifact.url], signal);
-            if ((await sha256(part)) !== artifact.sha256)
-                throw new Error(`${artifact.url} does not match its sha256`);
-        }
+        for (const artifact of missing)
+            await downloadedAgain(artifact, signal);
         for (const artifact of missing) {
             if (artifact.executable)
                 await chmod(`${installed(artifact)}.part`, 0o755);
@@ -83,29 +124,61 @@ async function install(artifacts, signal) {
             await rm(`${installed(artifact)}.part`, { force: true });
     }
 }
-async function transcribed(path, signal) {
-    const platform = `${process.platform}-${process.arch}`;
-    const pinned = ENGINES[platform];
-    if (pinned === undefined)
-        throw new Error(`no transcription engine is pinned for ${platform}`);
-    await install([pinned.decoder, pinned.engine, MODEL], signal);
-    const work = await mkdtemp(join(tmpdir(), 'kit-transcription-'));
+function reported(next) {
+    const changed = next.installed !== readiness.installed || next.failure !== readiness.failure;
+    readiness = next;
+    return changed;
+}
+async function settle(transcribe) {
+    if (transcribe === setting)
+        return false;
+    setting = transcribe;
+    if (!transcribe) {
+        await rm(directory(), { recursive: true, force: true });
+        return reported({ installed: false, failure: null });
+    }
+    const download = new AbortController();
+    downloading = download;
     try {
-        await executed(installed(pinned.decoder), ['-nostdin', '-hide_banner', '-loglevel', 'error', '-i', path, '-ar', '16000', '-ac', '1', '-c:a', 'pcm_s16le',
-            '-f', 'segment', '-segment_time', String(CHUNK_SECONDS), join(work, '%05d.wav')], signal);
-        const parts = (await readdir(work)).sort().flatMap((part) => ['-f', join(work, part)]);
-        const text = await executed(installed(pinned.engine), ['-t', String(await cpus()), '-np', '-m', installed(MODEL), ...parts], signal);
-        return text
-            .split('\n')
-            .map((line) => line.trim())
-            .filter((line) => line !== '')
-            .join(' ');
+        const pinned = pinnedHere();
+        await install([pinned.decoder, pinned.engine, MODEL], download.signal);
+        return reported({ installed: true, failure: null });
+    }
+    catch (error) {
+        await rm(directory(), { recursive: true, force: true });
+        return reported({ installed: false, failure: download.signal.aborted ? null : causeOf(error) });
     }
     finally {
-        await rm(work, { recursive: true, force: true });
+        downloading = null;
     }
 }
-export async function transcribe(path, signal) {
+export function holdTranscription(transcribe) {
+    if (!transcribe)
+        downloading?.abort();
+    const settled = holding.then(() => settle(transcribe));
+    holding = settled.catch(() => undefined);
+    return settled;
+}
+export function transcriptionReport() {
+    return readiness;
+}
+async function transcribed(audio, parts, signal) {
+    const pinned = pinnedHere();
+    await mkdir(parts);
+    await failing(UNDECODABLE, executed(installed(pinned.decoder), ['-nostdin', '-hide_banner', '-loglevel', 'error', '-i', audio, '-ar', '16000', '-ac', '1', '-c:a', 'pcm_s16le',
+        '-f', 'segment', '-segment_time', String(CHUNK_SECONDS), join(parts, '%05d.wav')], signal));
+    const segments = (await readdir(parts)).sort().flatMap((part) => ['-f', join(parts, part)]);
+    const said = await failing(UNTRANSCRIBED, executed(installed(pinned.engine), ['-t', String(await threads()), '-np', '-m', installed(MODEL), ...segments], signal));
+    const text = said
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line !== '')
+        .join(' ');
+    if (text === '')
+        throw new Error(UNTRANSCRIBED);
+    return text;
+}
+async function alone(work, signal) {
     const before = held;
     let release;
     const mine = new Promise((resolve) => {
@@ -114,9 +187,24 @@ export async function transcribe(path, signal) {
     held = Promise.all([before, mine]);
     try {
         await unlessAborted(before, signal);
-        return await transcribed(path, signal);
+        return await work();
     }
     finally {
         release();
     }
 }
+export async function transcribe(address, signal) {
+    ready();
+    const work = await mkdtemp(join(tmpdir(), 'kit-transcription-'));
+    try {
+        const audio = join(work, 'audio');
+        await failing(NOT_DOWNLOADED, downloaded(address, (url) => fetched(url, signal), audio, 'audio'));
+        return await alone(() => {
+            ready();
+            return transcribed(audio, join(work, 'parts'), signal);
+        }, signal);
+    }
+    finally {
+        await rm(work, { recursive: true, force: true });
+    }
+}

package/dist/translate.js

@@ -3,14 +3,8 @@ function same(left, right) {
     return JSON.stringify(left) === JSON.stringify(right);
 }
 function applied(document, changes) {
-    let current = document;
-    for (const change of changes) {
-        const result = apply(current, change);
-        if (result.outcome !== 'applied')
-            return null;
-        current = result.document;
-    }
-    return serialize(current);
+    const result = apply(document, changes);
+    return result.outcome === 'applied' ? serialize(result.document) : null;
 }
 function components(path, before, after, text) {
     const edits = [];

package/dist/update.js

@@ -4,11 +4,15 @@ import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 import { run } from './clis.js';
-import { kitHome } from './home.js';
-const RECORD = 'outcome=$1; shift; bash "$0" --linux "$@"; printf %s "$?" > "$outcome"; systemctl start house-kit.service';
-function logged(error) {
-    process.stderr.write(`kit: ${error instanceof Error ? error.message : String(error)}\n`);
-}
+import { kitHome, workspaceRoot } from './home.js';
+import { service, throughSudo } from './placement.js';
+import { relayed } from './relay.js';
+const BOOTSTRAP_BYTES = 1_000_000;
+// A bootstrap is at most a million bytes, so two minutes carry it over ten kilobytes a second and no stalled host holds the update queue.
+const BOOTSTRAP_MS = 120_000;
+// A bootstrap's downloads and CLI updates take minutes, so one still running after an hour has hung and is ended before it keeps the Kit offline.
+const RERUN_MS = 60 * 60_000;
+const TIMED_OUT = '124';
 function directory() {
     return join(kitHome(), 'update');
 }
@@ -18,10 +22,25 @@ function pending() {
 function outcome() {
     return join(directory(), 'outcome');
 }
-function unit(inputId) {
-    return `house-kit-update-${inputId}`;
+function unit(callId) {
+    return `house-kit-update-${callId}`;
+}
+function manager() {
+    return service() === 'user' ? ['--user'] : [];
+}
+function elevated() {
+    return throughSudo() ? ['sudo', '-n'] : [];
+}
+function account() {
+    return throughSudo() ? [`--uid=${process.getuid()}`, `--gid=${process.getgid()}`] : [];
+}
+function placement() {
+    return service() === 'user' ? ['--workspace', workspaceRoot()] : ['--linux'];
 }
-async function pendingInput() {
+function record() {
+    return `outcome=$1; shift; timeout ${RERUN_MS / 1000} bash "$0" "$@"; printf %s "$?" > "$outcome"; ${[...elevated(), 'systemctl', ...manager(), 'start', 'house-kit.service'].join(' ')}`;
+}
+async function pendingCall() {
     try {
         return JSON.parse(await readFile(pending(), 'utf8')).input_id;
     }
@@ -33,85 +52,130 @@ async function recorded() {
     const status = await readFile(outcome(), 'utf8').catch(() => null);
     if (status === null)
         return null;
+    if (status.trim() === TIMED_OUT)
+        return { refused: `the Kit bootstrap did not finish within ${RERUN_MS / 1000} seconds` };
     return status.trim() === '0' ? {} : { refused: `the Kit bootstrap exited with ${status.trim()}` };
 }
-async function finished(inputId) {
+async function forget(callId) {
+    if ((await pendingCall()) !== callId)
+        return;
+    await rm(pending(), { force: true });
+    await rm(outcome(), { force: true });
+}
+async function finished(callId) {
     for (;;) {
-        const ack = await recorded();
-        if (ack !== null)
-            return ack;
-        if ((await run('systemctl', ['is-active', '--quiet', unit(inputId)], 10_000)).status !== 0) {
+        const result = await recorded();
+        if (result !== null)
+            return result;
+        // systemctl answers at once, so ten seconds only bounds a stuck service manager.
+        if ((await run('systemctl', [...manager(), 'is-active', '--quiet', unit(callId)], 10_000)).status !== 0) {
             return (await recorded()) ?? { refused: 'the Kit bootstrap did not finish' };
         }
+        // The bootstrap runs for minutes, so checking each second sees its outcome within a second at little cost.
         await new Promise((resolve) => setTimeout(resolve, 1000));
     }
 }
-function rerun(script, inputId, flags) {
+async function bounded(body, most) {
+    const chunks = [];
+    let bytes = 0;
+    for await (const chunk of body ?? []) {
+        bytes += chunk.length;
+        if (bytes > most)
+            return null;
+        chunks.push(chunk);
+    }
+    return Buffer.concat(chunks, bytes);
+}
+function rerun(script, callId, flags) {
     return new Promise((resolve) => {
-        const child = spawn('systemd-run', [
+        const [command, ...args] = [
+            ...elevated(),
+            'systemd-run',
+            ...manager(),
+            ...account(),
             '--wait',
             '--collect',
             '--quiet',
-            `--unit=${unit(inputId)}`,
+            `--unit=${unit(callId)}`,
             `--setenv=HOME=${homedir()}`,
+            `--setenv=HOUSE_KIT_HOME=${kitHome()}`,
             '/bin/sh',
             '-c',
-            RECORD,
+            record(),
             script,
             outcome(),
+            ...placement(),
             ...flags,
-        ], { stdio: ['ignore', 'ignore', 'inherit'] });
-        child.on('error', () => resolve({ refused: 'no service manager runs this Kit' }));
+        ];
+        const child = spawn(command, args, { stdio: ['ignore', 'ignore', 'inherit'] });
+        child.on('error', () => resolve({ refused: 'no service manager runs this Kit process' }));
         child.on('close', () => resolve(null));
     });
 }
-export class Updates {
-    house;
-    held = new Set();
+export class KitUpdate {
+    calls;
+    changed;
+    waiting = new Map();
     queue = Promise.resolve();
-    constructor(house) {
-        this.house = house;
+    constructor(calls, changed) {
+        this.calls = calls;
+        this.changed = changed;
     }
     async resumed() {
-        const inputId = await pendingInput();
-        if (inputId === null)
+        const callId = await pendingCall();
+        if (callId === null)
             return;
-        await this.acknowledge(inputId, await finished(inputId));
+        await this.enqueue(callId, () => finished(callId));
     }
-    start(input) {
-        if (this.held.has(input.input_id))
-            return;
-        this.held.add(input.input_id);
-        this.queue = this.queue
-            .then(() => this.carry(input))
-            .then((ack) => this.acknowledge(input.input_id, ack))
-            .catch(logged)
-            .finally(() => this.held.delete(input.input_id));
+    hold(call) {
+        this.waiting.set(call.input_id, call);
+        this.changed();
+    }
+    held() {
+        return this.waiting.size > 0;
+    }
+    apply(resume) {
+        const held = [...this.waiting.values()];
+        this.waiting.clear();
+        void Promise.all(held.map((call) => this.enqueue(call.input_id, () => this.carry(call)))).then(resume);
     }
-    async carry(input) {
-        if ((await pendingInput()) === input.input_id)
-            return finished(input.input_id);
-        const answer = await fetch(input.bootstrap);
-        if (!answer.ok)
-            return { refused: `the Kit bootstrap could not be downloaded: ${answer.status}` };
-        const script = Buffer.from(await answer.arrayBuffer());
-        if (createHash('sha256').update(script).digest('hex') !== input.sha256) {
+    enqueue(callId, work) {
+        const before = this.queue;
+        const carried = this.calls.carry(callId, async () => {
+            await before;
+            return { result: await work(), returned: () => forget(callId) };
+        });
+        this.queue = before.then(() => carried);
+        return carried;
+    }
+    async carry(call) {
+        if ((await pendingCall()) === call.input_id)
+            return finished(call.input_id);
+        const deadline = AbortSignal.timeout(BOOTSTRAP_MS);
+        let script;
+        try {
+            const answer = await fetch(relayed(call.bootstrap), { signal: deadline });
+            if (!answer.ok)
+                return { refused: `the Kit bootstrap could not be downloaded: ${answer.status}` };
+            script = await bounded(answer.body, BOOTSTRAP_BYTES);
+        }
+        catch (error) {
+            if (!deadline.aborted)
+                throw error;
+            return { refused: `the Kit bootstrap did not arrive within ${BOOTSTRAP_MS / 1000} seconds` };
+        }
+        if (script === null)
+            return { refused: `the Kit bootstrap is larger than ${BOOTSTRAP_BYTES} bytes` };
+        if (createHash('sha256').update(script).digest('hex') !== call.sha256) {
             return { refused: 'the Kit bootstrap does not match its sha256' };
         }
         await mkdir(directory(), { recursive: true, mode: 0o700 });
         const path = join(directory(), 'connect-linux.sh');
         await writeFile(path, script, { mode: 0o700 });
         await rm(outcome(), { force: true });
-        await writeFile(pending(), `${JSON.stringify({ input_id: input.input_id })}\n`, { mode: 0o600 });
-        return (await rerun(path, input.input_id, input.update_clis ? ['--update-clis'] : [])) ?? (await recorded()) ?? {
+        await writeFile(pending(), `${JSON.stringify({ input_id: call.input_id })}\n`, { mode: 0o600 });
+        return (await rerun(path, call.input_id, call.update_clis ? ['--update-clis'] : [])) ?? (await recorded()) ?? {
             refused: 'the Kit bootstrap did not finish',
         };
     }
-    async acknowledge(inputId, ack) {
-        await this.house.deliver(`/kit/inputs/${inputId}/ack`, ack);
-        if ((await pendingInput()) !== inputId)
-            return;
-        await rm(pending(), { force: true });
-        await rm(outcome(), { force: true });
-    }
 }

package/dist/wire.js

@@ -0,0 +1,22 @@
+import { newerThanHouse, updateRequired } from './end.js';
+import { machine } from './placement.js';
+import { refusalOf } from './refusals.js';
+export const KIT_WIRE = 3;
+export const KIT_WIRE_HEADER = 'x-house-kit-wire';
+function newer(text) {
+    let body;
+    try {
+        body = JSON.parse(text);
+    }
+    catch {
+        return null;
+    }
+    return body?.error?.code === 'kit_newer_than_house' ? body.error : null;
+}
+export function endUnserved(text) {
+    if (refusalOf(text)?.code === 'kit_update_required')
+        updateRequired();
+    const refused = newer(text);
+    if (refused !== null)
+        newerThanHouse(refused.release, refused.commands[machine()]);
+}