@agentshouse/kit 0.2.1-alpha.19 → 0.2.1-alpha.20

package/README.md

@@ -48,6 +48,10 @@ workspace. Commit in one and run `house git push --owner` there to send the
 commit to House; in the container, run it through the install command from the
 copy's directory. Inside an Agent conversation the Agent runs `house git push`.
 
+House Kit turns each audio file of your message to an Agent into text on this
+computer before the Agent reads the message. The first audio file downloads the
+speech engine and model, about 425 MB, into `~/.house-kit`.
+
 ## License
 
 MIT

package/dist/abort.js

@@ -0,0 +1,9 @@
+export function unlessAborted(work, signal) {
+    return new Promise((resolve, reject) => {
+        const aborted = () => reject(signal.reason);
+        signal.addEventListener('abort', aborted, { once: true });
+        if (signal.aborted)
+            aborted();
+        work.then(resolve, reject).finally(() => signal.removeEventListener('abort', aborted));
+    });
+}

package/dist/bootstrap.json

@@ -1,12 +1,12 @@
 {
   "schemaVersion": 1,
-  "version": "0.2.1-alpha.19",
+  "version": "0.2.1-alpha.20",
   "hosts": {
     "linux": {
-      "command": "bash <(curl -fsSL https://github.com/agentshouse/kit/releases/download/v0.2.1-alpha.19/connect-linux.sh)",
+      "command": "bash <(curl -fsSL https://github.com/agentshouse/kit/releases/download/v0.2.1-alpha.20/connect-linux.sh)",
       "asset": {
         "name": "connect-linux.sh",
-        "url": "https://github.com/agentshouse/kit/releases/download/v0.2.1-alpha.19/connect-linux.sh"
+        "url": "https://github.com/agentshouse/kit/releases/download/v0.2.1-alpha.20/connect-linux.sh"
       },
       "workspace": {
         "argument": "--workspace",
@@ -17,10 +17,10 @@
       }
     },
     "windows": {
-      "command": "& ([scriptblock]::Create((irm https://github.com/agentshouse/kit/releases/download/v0.2.1-alpha.19/connect-windows.ps1)))",
+      "command": "& ([scriptblock]::Create((irm https://github.com/agentshouse/kit/releases/download/v0.2.1-alpha.20/connect-windows.ps1)))",
       "asset": {
         "name": "connect-windows.ps1",
-        "url": "https://github.com/agentshouse/kit/releases/download/v0.2.1-alpha.19/connect-windows.ps1"
+        "url": "https://github.com/agentshouse/kit/releases/download/v0.2.1-alpha.20/connect-windows.ps1"
       },
       "workspace": {
         "argument": "--workspace",

package/dist/clis.js

@@ -111,28 +111,29 @@ async function installedAdapter(kind) {
 export function withoutProxy() {
     return Object.fromEntries(Object.entries(process.env).filter(([name]) => !PROXY.test(name)));
 }
-export function run(command, args, timeoutMs, detached = false) {
+export function run(command, args, timeoutMs, detached = false, signal) {
     return new Promise((resolve) => {
         const child = spawn(command, args, {
             stdio: ['ignore', 'pipe', 'pipe'],
             timeout: timeoutMs,
             env: withoutProxy(),
             detached,
+            signal,
         });
         let stdout = '';
         let output = '';
-        child.stdout.on('data', (chunk) => {
-            stdout += chunk.toString('utf8');
-            output += chunk.toString('utf8');
+        child.stdout.setEncoding('utf8').on('data', (chunk) => {
+            stdout += chunk;
+            output += chunk;
         });
-        child.stderr.on('data', (chunk) => {
-            output += chunk.toString('utf8');
+        child.stderr.setEncoding('utf8').on('data', (chunk) => {
+            output += chunk;
         });
         child.on('error', (error) => resolve({ status: null, stdout, output: error.message }));
         child.on('close', (status) => resolve({ status, stdout, output }));
     });
 }
-function failureOf(ran, program) {
+export function failureOf(ran, program) {
     return ran.output.trim().slice(-CAUSE_CHARACTERS) || `${program} exited with ${ran.status ?? 'a signal'}`;
 }
 export async function installAdapter(kind) {

package/dist/conversations.js

@@ -2,6 +2,7 @@ import { client, } from '@agentclientprotocol/sdk';
 import { randomUUID } from 'node:crypto';
 import { mkdir } from 'node:fs/promises';
 import { join } from 'node:path';
+import { unlessAborted } from './abort.js';
 import { killMarked, startAdapter, TURN_ENDED, TURN_STARTED } from './acp.js';
 import { blocksOf, firstChange } from './blocks.js';
 import { openBridge } from './bridge.js';
@@ -11,6 +12,7 @@ import { agentBase } from './home.js';
 import { instructions } from './instructions.js';
 import { holdSecretInput } from './secret-input.js';
 import { createHowWeWork } from './skills.js';
+import { transcribe } from './transcription.js';
 const KILLED = 'the conversation was killed';
 function causeOf(error) {
     return error instanceof Error ? error.message : String(error);
@@ -21,15 +23,6 @@ function answerMessage(request, response) {
 function logged(error) {
     process.stderr.write(`kit: ${causeOf(error)}\n`);
 }
-function unlessKilled(work, signal) {
-    return new Promise((resolve, reject) => {
-        const killed = () => reject(signal.reason);
-        signal.addEventListener('abort', killed, { once: true });
-        if (signal.aborted)
-            killed();
-        work.then(resolve, reject).finally(() => signal.removeEventListener('abort', killed));
-    });
-}
 function stop(bridge) {
     killMarked(`HOUSE_BRIDGE=${bridge.env.HOUSE_BRIDGE}`);
 }
@@ -60,6 +53,8 @@ class Conversation {
     opening = null;
     kills = 0;
     killed = new AbortController();
+    interrupts = 0;
+    interrupted = new AbortController();
     open = 0;
     queue = Promise.resolve();
     reports = Promise.resolve();
@@ -95,6 +90,10 @@ export class Conversations {
                     conversation.killed.abort(new Error(KILLED));
                     this.halt(conversation);
                 }
+                if (input.kind === 'interrupt') {
+                    conversation.interrupts++;
+                    conversation.interrupted.abort();
+                }
                 const work = conversation.queue.then(() => this.carry(conversation, input));
                 conversation.queue = work.catch(() => undefined);
                 this.acks.set(input.input_id, work);
@@ -138,8 +137,11 @@ export class Conversations {
             return this.open(conversation, input);
         if (input.kind === 'message')
             return this.message(conversation, input);
-        if (input.kind === 'interrupt')
+        if (input.kind === 'interrupt') {
+            if (--conversation.interrupts === 0)
+                conversation.interrupted = new AbortController();
             await this.interrupt(conversation, String(input.turn_id));
+        }
         if (input.kind === 'kill') {
             await this.kill(conversation);
             if (--conversation.kills === 0)
@@ -171,7 +173,9 @@ export class Conversations {
         const prompt = [];
         try {
             const route = await this.route(conversation, input.agent_id);
-            const paths = await placeFiles(this.kit.house, route.working_directory, input.files, conversation.killed.signal);
+            const files = input.files;
+            const paths = await placeFiles(this.kit.house, route.working_directory, files, conversation.killed.signal);
+            const lines = await this.transcribed(conversation, files, paths);
             if (conversation.running === null) {
                 const opened = await this.start(conversation, route, input.provider_session_id);
                 if (input.provider_session_id === null)
@@ -184,8 +188,8 @@ export class Conversations {
                 });
             }
             prompt.push({ type: 'text', text: String(input.text) });
-            if (paths.length > 0)
-                prompt.push({ type: 'text', text: paths.join('\n') });
+            if (lines.length > 0)
+                prompt.push({ type: 'text', text: lines.join('\n') });
             if (conversation.kills > 0)
                 throw new Error(KILLED);
         }
@@ -197,6 +201,25 @@ export class Conversations {
         this.prompt(conversation, prompt);
         return ack;
     }
+    async transcribed(conversation, files, paths) {
+        const lines = [];
+        for (const [index, path] of paths.entries()) {
+            lines.push(path);
+            if (files[index].media_type?.startsWith('audio/') !== true)
+                continue;
+            try {
+                const text = await transcribe(path, AbortSignal.any([conversation.killed.signal, conversation.interrupted.signal]));
+                if (text !== '')
+                    lines.push(`Transcript: ${text}`);
+            }
+            catch (error) {
+                if (conversation.kills > 0)
+                    throw error;
+                logged(new Error(`${files[index].name} was not transcribed: ${causeOf(error)}`));
+            }
+        }
+        return lines;
+    }
     async interrupt(conversation, turnId) {
         if (conversation.running === null || conversation.turn?.id !== turnId)
             return;
@@ -235,9 +258,9 @@ export class Conversations {
     }
     async route(conversation, agentId) {
         const signal = conversation.killed.signal;
-        await unlessKilled(this.kit.agents.read(), signal);
+        await unlessAborted(this.kit.agents.read(), signal);
         if (this.kit.agents.route(agentId) === undefined)
-            await unlessKilled(this.kit.agents.refresh(), signal);
+            await unlessAborted(this.kit.agents.refresh(), signal);
         const route = this.kit.agents.route(agentId);
         if (route === undefined)
             throw new Error(`this Environment hosts no Agent ${agentId}`);
@@ -253,7 +276,7 @@ export class Conversations {
             .onNotification(TURN_ENDED, (params) => params, () => this.ended(conversation))
             .onRequest('session/request_permission', ({ params, signal }) => this.ask(conversation, 'session/request_permission', params, false, signal))
             .onRequest('elicitation/create', ({ params, signal }) => this.ask(conversation, 'elicitation/create', params, asksSecret(params), signal));
-        const cli = await unlessKilled(this.kit.agents.cli(route.kind), conversation.killed.signal);
+        const cli = await unlessAborted(this.kit.agents.cli(route.kind), conversation.killed.signal);
         const bridge = await openBridge(this.kit.house, this.kit.copies, conversation.id, conversation.killed.signal);
         if (conversation.kills > 0) {
             bridge.close();
@@ -261,7 +284,7 @@ export class Conversations {
         }
         conversation.opening = bridge;
         try {
-            await unlessKilled(createHowWeWork(bridge), conversation.killed.signal);
+            await unlessAborted(createHowWeWork(bridge), conversation.killed.signal);
             const adapter = await startAdapter(route.kind, cli, route.working_directory, app, (job) => this.job(conversation, job), bridge.env);
             void adapter.exited.then(() => bridge.close());
             const agent = adapter.connection.agent;

package/dist/transcription.js

@@ -0,0 +1,118 @@
+import { createHash } from 'node:crypto';
+import { createReadStream } from 'node:fs';
+import { chmod, mkdir, mkdtemp, readdir, readFile, rename, rm, stat } from 'node:fs/promises';
+import { availableParallelism, tmpdir } from 'node:os';
+import { basename, join } from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import { unlessAborted } from './abort.js';
+import { failureOf, run } from './clis.js';
+import { kitHome } from './home.js';
+const RELEASE = 'https://github.com/agentshouse/kit/releases/download/engine-1';
+export const MODEL = {
+    url: 'https://huggingface.co/ggml-org/parakeet-GGUF/resolve/35156454d1a39de06863303dd209fd2bed6ee079/ggml-parakeet-tdt-0.6b-v3-q4_k.bin',
+    sha256: '8b205b8b39c6535e153de6fb11c51db46125d45c4f16ba496fe41a0fe71b885e',
+    executable: false,
+};
+export const ENGINES = {
+    'linux-x64': {
+        decoder: { url: `${RELEASE}/ffmpeg-linux-x86_64`, sha256: '1f8235d206428305afb89c6e8ab99d100425a0ba61e22bb4c7aff017db8c8de5', executable: true },
+        engine: { url: `${RELEASE}/parakeet-cli-linux-x86_64`, sha256: '259fd7a1c83f9768be45d15ef88965f82ce86c481215cfe34f778f9890b3a833', executable: true },
+    },
+    'linux-arm64': {
+        decoder: { url: `${RELEASE}/ffmpeg-linux-aarch64`, sha256: 'a7e08a0fae0179d1078312f6bb13d2a9aa4d4c71a1d82a022953f25dee452f5e', executable: true },
+        engine: { url: `${RELEASE}/parakeet-cli-linux-aarch64`, sha256: 'e2585bc37a30baf48b16e19d18e56920cba78c05da8dc0dea2b0dab0b26caab9', executable: true },
+    },
+};
+const CHUNK_SECONDS = 20;
+let held = Promise.resolve();
+function directory() {
+    return join(kitHome(), 'transcription');
+}
+function installed(artifact) {
+    return join(directory(), artifact.sha256);
+}
+async function executed(command, args, signal) {
+    const ran = await run(command, args, 0, false, signal);
+    if (ran.status !== 0)
+        throw new Error(failureOf(ran, basename(command)));
+    return ran.stdout;
+}
+function present(path) {
+    return stat(path).then(() => true, () => false);
+}
+async function cpus() {
+    const [quota, period] = (await readFile('/sys/fs/cgroup/cpu.max', 'utf8').catch(() => 'max')).split(' ');
+    const allowed = quota === 'max' ? Infinity : Math.ceil(Number(quota) / Number(period));
+    return Math.min(availableParallelism(), allowed);
+}
+async function sha256(path) {
+    const hash = createHash('sha256');
+    await pipeline(createReadStream(path), hash);
+    return hash.digest('hex');
+}
+async function install(artifacts, signal) {
+    await mkdir(directory(), { recursive: true });
+    const pinned = new Set(artifacts.map((artifact) => artifact.sha256));
+    for (const name of await readdir(directory())) {
+        if (!pinned.has(name))
+            await rm(join(directory(), name), { recursive: true, force: true });
+    }
+    const missing = [];
+    for (const artifact of artifacts)
+        if (!(await present(installed(artifact))))
+            missing.push(artifact);
+    try {
+        for (const artifact of missing) {
+            const part = `${installed(artifact)}.part`;
+            await executed('curl', ['-fsSL', '--retry', '3', '-o', part, artifact.url], signal);
+            if ((await sha256(part)) !== artifact.sha256)
+                throw new Error(`${artifact.url} does not match its sha256`);
+        }
+        for (const artifact of missing) {
+            if (artifact.executable)
+                await chmod(`${installed(artifact)}.part`, 0o755);
+            await rename(`${installed(artifact)}.part`, installed(artifact));
+        }
+    }
+    finally {
+        for (const artifact of missing)
+            await rm(`${installed(artifact)}.part`, { force: true });
+    }
+}
+async function transcribed(path, signal) {
+    const platform = `${process.platform}-${process.arch}`;
+    const pinned = ENGINES[platform];
+    if (pinned === undefined)
+        throw new Error(`no transcription engine is pinned for ${platform}`);
+    await install([pinned.decoder, pinned.engine, MODEL], signal);
+    const work = await mkdtemp(join(tmpdir(), 'kit-transcription-'));
+    try {
+        await executed(installed(pinned.decoder), ['-nostdin', '-hide_banner', '-loglevel', 'error', '-i', path, '-ar', '16000', '-ac', '1', '-c:a', 'pcm_s16le',
+            '-f', 'segment', '-segment_time', String(CHUNK_SECONDS), join(work, '%05d.wav')], signal);
+        const parts = (await readdir(work)).sort().flatMap((part) => ['-f', join(work, part)]);
+        const text = await executed(installed(pinned.engine), ['-t', String(await cpus()), '-np', '-m', installed(MODEL), ...parts], signal);
+        return text
+            .split('\n')
+            .map((line) => line.trim())
+            .filter((line) => line !== '')
+            .join(' ');
+    }
+    finally {
+        await rm(work, { recursive: true, force: true });
+    }
+}
+export async function transcribe(path, signal) {
+    const before = held;
+    let release;
+    const mine = new Promise((resolve) => {
+        release = resolve;
+    });
+    held = Promise.all([before, mine]);
+    try {
+        await unlessAborted(before, signal);
+        return await transcribed(path, signal);
+    }
+    finally {
+        release();
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentshouse/kit",
-  "version": "0.2.1-alpha.19",
+  "version": "0.2.1-alpha.20",
   "description": "House Kit connects an Environment to agents.house.",
   "license": "MIT",
   "repository": {
@@ -22,6 +22,7 @@
   ],
   "scripts": {
     "build": "tsc -p tsconfig.build.json && node scripts/bootstrap-manifest.mjs",
+    "engine": "mkdir -p tmp/engine && docker run --rm -v \"$PWD:/kit\" -w /kit alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 sh scripts/engine.sh tmp/engine",
     "test": "tsc -p tsconfig.json && vitest run"
   },
   "dependencies": {