@jskit-ai/agent-runtime 0.0.0-stage → 0.1.0

package/test/planning.postgres.test.js

@@ -0,0 +1,63 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { PLANNING_TABLES, currentTodos, planningNotes, planningTools } from "../src/server/index.js";
+import { DATABASE, testDatabase } from "./database.js";
+
+// A to-do list the person sees, as one card updated in place, and notes only the model sees; the
+// open list and the notes are in the state note at every turn.
+async function chat(context) {
+  const knex = await testDatabase(context);
+  await knex.schema.createTable(PLANNING_TABLES.conversations, (table) => {
+    table.bigIncrements("id").primary();
+    table.jsonb("job_notes");
+  });
+  await knex.schema.createTable(PLANNING_TABLES.messages, (table) => {
+    table.bigIncrements("id").primary();
+    table.string("kind", 20).notNullable();
+    table.string("role", 20).notNullable();
+    table.jsonb("body").notNullable();
+  });
+  const [{ id }] = await knex(PLANNING_TABLES.conversations).insert({}).returning("id");
+  const told = [];
+  const post = async (messages) => {
+    told.push(messages.length);
+    for (const message of messages) await knex(PLANNING_TABLES.messages).insert(message);
+  };
+  return { knex, conversationId: id, post, told };
+}
+
+test("the to-do card is made, ticked off in place, remembered, and replaced for the next job", { skip: DATABASE }, async (context) => {
+  const { knex, conversationId, post, told } = await chat(context);
+  const tools = planningTools({ knex, conversationId, post });
+  const cards = () => knex(PLANNING_TABLES.messages).where({ kind: "todos" }).orderBy("id");
+  const made = await tools.updateTodos.execute({ items: [{ text: "Read the CRM", status: "doing" }, { text: "Add the field", status: "todo" }, { text: "Test it", status: "todo" }] });
+  assert.deepEqual(made, { status: "saved" });
+  assert.match((await tools.updateTodos.execute({ items: [{ text: "A", status: "doing" }, { text: "B", status: "doing" }] })).error, /Only one item can be doing at a time; these are all doing: A; B\./u);
+  await tools.updateTodos.execute({ items: [{ text: "Read the CRM", status: "done" }, { text: "Add the field", status: "doing" }, { text: "Test it", status: "todo" }] });
+  let list = await cards();
+  assert.equal(list.length, 1, "one card, updated in place");
+  assert.deepEqual(list[0].body.items.map((item) => item.status), ["done", "doing", "todo"]);
+  assert.deepEqual(told, [1, 0], "open pages are told each time");
+  const [state] = await planningNotes(knex, conversationId);
+  assert.equal(state, "Your to-do list for this job (keep it up to date with update_todos):\n[x] Read the CRM\n[doing] Add the field\n[ ] Test it");
+
+  const last = await tools.updateTodos.execute({ items: [{ text: "Read the CRM", status: "done" }, { text: "Add the field", status: "done" }, { text: "Test it", status: "done" }] });
+  assert.match(last.note, /Everything on the list is done/u);
+  assert.deepEqual(await planningNotes(knex, conversationId), [], "a finished list isn't carried");
+  await tools.updateTodos.execute({ items: [{ text: "Plan the trip", status: "doing" }] });
+  list = await cards();
+  assert.equal(list.length, 2, "the next job gets its own card");
+  assert.equal((await currentTodos(knex, conversationId)).body.items[0].text, "Plan the trip");
+  assert.equal(planningTools({ knex, conversationId }).updateTodos, undefined, "no card without a chat to show it");
+});
+
+test("job notes are kept for the model, shown at the next turn, and cleared", { skip: DATABASE }, async (context) => {
+  const { knex, conversationId } = await chat(context);
+  const tools = planningTools({ knex, conversationId });
+  assert.deepEqual(await planningNotes(knex, conversationId), []);
+  assert.equal((await tools.keepNotes.execute({ notes: ["The CRM table is contacts", "Due date goes after Stage"] })).status, "saved");
+  assert.deepEqual(await planningNotes(knex, conversationId), ["Your notes for the job in hand (keep them up to date with keep_notes):\n- The CRM table is contacts\n- Due date goes after Stage"]);
+  assert.equal((await tools.keepNotes.execute({ notes: [] })).status, "cleared");
+  assert.deepEqual(await planningNotes(knex, conversationId), [], "and gone once cleared");
+  assert.equal(await knex(PLANNING_TABLES.messages).count("* as n").first().then((row) => Number(row.n)), 0, "nothing shows in the chat");
+});

package/test/runs.postgres.test.js

@@ -0,0 +1,59 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { createRuns, startResumer } from "../src/server/index.js";
+import { DATABASE, testDatabase } from "./database.js";
+
+// A reply saves its steps as each one finishes; one a stopped server left part-way is taken over
+// by another server, once, and carried on from them.
+const call = { type: "tool-call", toolCallId: "c1", toolName: "list_files", input: {} };
+const result = { type: "tool-result", toolCallId: "c1", toolName: "list_files", output: { type: "json", value: { entries: ["notes.txt"] } } };
+const steps = [{ role: "assistant", content: [call] }, { role: "tool", content: [result] }];
+const quiet = (knex) => knex.raw("now() - interval '5 minutes'");
+
+test("a reply claims its conversation, saves its steps while it runs, and is gone once it ends", { skip: DATABASE }, async (context) => {
+  const knex = await testDatabase(context);
+  await knex.schema.alterTable("agent_runs", (table) => table.bigInteger("bot_id"));
+  const runs = createRuns({ knex });
+  assert.equal(await runs.claim(1, { bot_id: 9 }), true);
+  assert.equal(await runs.claim(1), false, "one reply per conversation at a time");
+  assert.equal((await knex("agent_runs").where({ conversation_id: 1 }).first()).bot_id, "9", "with the app's own columns");
+  await runs.save(1, steps, { human: true, answers: 5 });
+  const saved = await runs.saved(1);
+  assert.deepEqual(saved.steps.map((message) => message.role), ["assistant", "tool"]);
+  assert.deepEqual(saved.input, { human: true, answers: 5 });
+  await knex("agent_runs").where({ conversation_id: 1 }).update({ heartbeat_at: quiet(knex) });
+  await runs.heartbeat(1);
+  assert.equal(await runs.claim(1), false, "a beating run stays its server's");
+  await runs.release(1);
+  assert.equal(await runs.saved(1), undefined);
+  assert.equal(await runs.claim(1), true, "free again");
+});
+
+test("a run whose server stopped beating is taken over once, with its steps", { skip: DATABASE }, async (context) => {
+  const knex = await testDatabase(context);
+  const runs = createRuns({ knex });
+  await knex("agent_runs").insert([
+    { conversation_id: 2, heartbeat_at: quiet(knex), steps: JSON.stringify(steps), input: JSON.stringify({ human: true }) },
+    { conversation_id: 3, heartbeat_at: quiet(knex), steps: JSON.stringify(steps), input: JSON.stringify({ human: false }) },
+    { conversation_id: 4, heartbeat_at: quiet(knex) },
+    { conversation_id: 5, steps: JSON.stringify(steps), input: JSON.stringify({ human: true }) }
+  ]);
+  const human = (query) => query.whereRaw("input->>'human' = 'true'");
+  const taken = await runs.takeOverStalled({ resumable: human });
+  assert.deepEqual(taken.map((run) => Number(run.conversation_id)), [2], "only stale runs with steps, of the kinds that count");
+  assert.deepEqual(taken[0].steps.map((message) => message.role), ["assistant", "tool"]);
+  assert.deepEqual(await runs.takeOverStalled({ resumable: human }), [], "taken over once");
+  assert.equal(await runs.claim(4), true, "a stale run without steps can be claimed afresh");
+  assert.equal(await runs.claim(5), false);
+});
+
+test("the resumer carries on what stopped servers left, then drains", { skip: DATABASE }, async (context) => {
+  const knex = await testDatabase(context);
+  const runs = createRuns({ knex });
+  await knex("agent_runs").insert({ conversation_id: 6, heartbeat_at: quiet(knex), steps: JSON.stringify(steps), input: JSON.stringify({ human: true }) });
+  const resumed = [];
+  const resumer = startResumer({ runs, resume: async (run) => resumed.push(Number(run.conversation_id)), everyMs: 50 });
+  for (let tries = 0; tries < 40 && !resumed.length; tries += 1) await new Promise((resolve) => setTimeout(resolve, 50));
+  await resumer.drain(1000);
+  assert.deepEqual(resumed, [6]);
+});

package/test/summaryStore.postgres.test.js

@@ -0,0 +1,188 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { SUMMARY_LIMITS } from "@jskit-ai/agent/server";
+import { createSummaries } from "../src/server/index.js";
+import { DATABASE, testDatabase } from "./database.js";
+
+// Summarising conversations in the background: past a size, a conversation is queued; one server
+// summarises all but its latest turns, folding in the previous summary; replies read the summary
+// plus every message after it word for word; messages that arrive while it runs stay as they are;
+// failures wait and try again; a first summary takes in what the conversation inherited.
+const limits = { ...SUMMARY_LIMITS, compactAt: 500, keepTurns: 2, keepTokens: 400, minTokens: 50 };
+
+async function conversations(context) {
+  const knex = await testDatabase(context);
+  await knex.schema.createTable("chat_messages", (table) => {
+    table.bigIncrements("id").primary();
+    table.bigInteger("conversation_id").notNullable();
+    table.string("kind", 20).notNullable().defaultTo("text");
+    table.string("role", 20).notNullable();
+    table.jsonb("body").notNullable();
+  });
+  const inherited = new Map();
+  const say = (conversationId, role, text) => knex("chat_messages").insert({ conversation_id: conversationId, role, body: { text } });
+  // The app's side: its stored messages after a summary, what a conversation inherits, and how it
+  // turns messages into model messages.
+  const history = async (conversationId, { after }) => ({
+    messages: await knex("chat_messages").where({ conversation_id: conversationId }).where("id", ">", after).orderBy("id"),
+    inherited: inherited.get(conversationId) || [],
+    toModel: async (messages) => messages.map((message) => ({ role: message.role === "user" ? "user" : "assistant", content: message.body.text })),
+    named: false
+  });
+  return { knex, say, history, inherited };
+}
+
+test("long conversations are summarised in the background, once, and replies read the summary", { skip: DATABASE }, async (context) => {
+  const { knex, say, history } = await conversations(context);
+  const summaries = [];
+  let summarizer = async ({ previous, text }) => {
+    summaries.push({ previous, text });
+    return `SUMMARY ${summaries.length}`;
+  };
+  const tracked = [];
+  const store = createSummaries({ knex, history, limits, summarize: (input) => summarizer(input), track: (conversationId, work) => (tracked.push(conversationId), work()) });
+  const queued = () => knex("agent_summary_queue").where({ conversation_id: 1 }).first();
+  const exchange = async (n, size) => {
+    await say(1, "user", `${n === 1 ? "My name is Ada and my budget is 4,200 euros. " : `Message ${n}. `}${"More detail. ".repeat(size)}`);
+    await say(1, "bot", `Answer ${n}`);
+  };
+
+  await exchange(1, 20);
+  assert.equal(await store.consider(1), false);
+  assert.equal(await queued(), undefined, "below the size nothing is queued");
+  for (let n = 2; n <= 6; n += 1) await exchange(n, 30);
+  assert.equal(await store.consider(1), true);
+  assert.ok(await queued(), "queued once the conversation grew past the limit");
+  assert.deepEqual(await store.current(1), { summary: "", after: 0 }, "no summary yet: a reply reads everything");
+
+  // Two servers try at once: one summarises.
+  let release;
+  const gate = new Promise((resolve) => (release = resolve));
+  summarizer = async ({ previous, text }) => {
+    await gate;
+    summaries.push({ previous, text });
+    return `SUMMARY ${summaries.length}`;
+  };
+  const first = store.runOne();
+  await new Promise((resolve) => setTimeout(resolve, 100));
+  assert.equal(await store.runOne(), false, "the other server finds nothing to claim");
+  // A message arriving while the summary is written stays word for word after it.
+  await say(1, "user", "Arrived during the summary.");
+  release();
+  assert.equal(await first, true);
+  const [summary] = await knex("agent_summaries").where({ conversation_id: 1 });
+  assert.equal(summaries.length, 1);
+  assert.match(summaries[0].text, /^Person: My name is Ada and my budget is 4,200 euros\./mu);
+  assert.match(summaries[0].text, /^Bot: Answer 1$/mu);
+  assert.ok(!summaries[0].text.includes("Arrived during the summary."));
+  assert.deepEqual(tracked.map(Number), [1], "metered to the conversation");
+  assert.equal(await queued(), undefined, "done, and nothing more queued while the rest is small");
+
+  // A reply reads the summary and only the messages after it.
+  const { summary: text, after } = await store.current(1);
+  assert.equal(text, "SUMMARY 1");
+  assert.equal(after, Number(summary.covers_up_to));
+  const { messages } = await history(1, { after });
+  assert.equal(messages[0].role, "user", "it starts at a turn");
+  assert.ok(messages.some((message) => message.body.text === "Arrived during the summary."));
+  assert.ok(!messages.some((message) => /My name is Ada/u.test(message.body.text)), "summarised messages aren't read again");
+
+  // A failure waits and tries again later.
+  for (let n = 7; n <= 12; n += 1) await exchange(n, 30);
+  assert.equal(await store.consider(1), true);
+  summarizer = async () => {
+    throw new Error("model unavailable");
+  };
+  assert.equal(await store.runOne(), true);
+  const failed = await queued();
+  assert.equal(failed.attempts, 1);
+  assert.match(failed.last_error, /model unavailable/u);
+  assert.ok(new Date(failed.run_after) > new Date(), "tried again later");
+  assert.equal(await store.runOne(), false, "not before its time");
+
+  // The next summary folds in the previous one.
+  await knex("agent_summary_queue").where({ conversation_id: 1 }).update({ run_after: knex.fn.now() });
+  summarizer = async ({ previous, text: folded }) => {
+    summaries.push({ previous, text: folded });
+    return `SUMMARY ${summaries.length}`;
+  };
+  assert.equal(await store.runOne(), true);
+  assert.equal(summaries.at(-1).previous, "SUMMARY 1");
+  assert.ok(!summaries.at(-1).text.includes("My name is Ada"), "only what came after the previous summary");
+  assert.equal((await store.current(1)).summary, "SUMMARY 2");
+  assert.equal(await knex("chat_messages").where({ conversation_id: 1 }).count("* as n").first().then((row) => Number(row.n)), 25, "no message is deleted");
+});
+
+test("a first summary takes in what the conversation inherited", { skip: DATABASE }, async (context) => {
+  const { knex, say, history, inherited } = await conversations(context);
+  const captured = [];
+  const store = createSummaries({ knex, history, limits: { ...limits, minTokens: 1 }, summarize: async ({ text }) => (captured.push(text), "ROOM SUMMARY") });
+  inherited.set(2, [{ id: "0", role: "user", kind: "text", body: { text: "Inherited: the launch is on 3 March." } }]);
+  for (let n = 0; n < 4; n += 1) {
+    await say(2, "user", `Room message ${n}`);
+    await say(2, "bot", `Room answer ${n}`);
+  }
+  const done = await store.compact(2);
+  assert.equal(done.messages, 4, "all but the latest two turns");
+  assert.match(captured[0], /^Person: Inherited: the launch is on 3 March\.$/mu);
+  assert.match(captured[0], /^Person: Room message 0$/mu);
+  assert.match(captured[0], /^Bot: Room answer 0$/mu);
+  assert.equal((await knex("agent_summaries").where({ conversation_id: 2 }).first()).summary, "ROOM SUMMARY");
+  assert.deepEqual(await createSummaries({ knex, history, limits }).compact(3), { skipped: "nothing to summarise yet" });
+  await say(4, "user", "a");
+  await say(4, "bot", "b");
+  await say(4, "user", "c");
+  assert.deepEqual(await createSummaries({ knex, history, limits: { ...limits, keepTurns: 1 } }).compact(4), { skipped: "too little to summarise" });
+});
+
+test("a reply summarises first, rather than dropping messages unseen, when summarising fell behind", { skip: DATABASE }, async (context) => {
+  const { knex, say, history } = await conversations(context);
+  const filler = "We talked about the garden, the weather and the week ahead. ".repeat(14);
+  for (let n = 0; n < 12; n += 1) {
+    await say(5, "user", `${n === 0 ? "My sister Lucia's birthday is 14 March. " : ""}${filler}`);
+    await say(5, "bot", `Noted. ${filler}`);
+  }
+  const summaries = [];
+  const store = createSummaries({
+    knex,
+    history,
+    limits: { ...SUMMARY_LIMITS, compactAt: 1000000, summariseNowAt: 1500, keepTurns: 2, keepTokens: 400, minTokens: 50 },
+    summarize: async ({ text }) => {
+      summaries.push(text);
+      return `SUMMARY: ${/Lucia's birthday is 14 March/u.test(text) ? "Lucia's birthday is 14 March." : "nothing"}`;
+    }
+  });
+  await store.ensureWithin(5);
+  assert.ok(summaries.length >= 1, "it summarised before the reply");
+  const { summary, after } = await store.current(5);
+  assert.match(summary, /Lucia's birthday is 14 March/u, "so the oldest fact isn't lost");
+  assert.ok((await history(5, { after })).messages.length < 24, "while fewer messages are read word for word");
+});
+
+test("a notice wakes every server's loop, which summarises what's queued", { skip: DATABASE }, async (context) => {
+  const { knex, say, history } = await conversations(context);
+  const listeners = [];
+  const published = [];
+  const notices = {
+    publish: async (channel, key) => {
+      published.push(`${channel}:${key}`);
+      for (const listener of listeners) listener();
+    },
+    subscribe: (_channel, _key, listener) => {
+      listeners.push(listener);
+      return () => listeners.splice(listeners.indexOf(listener), 1);
+    }
+  };
+  const store = createSummaries({ knex, history, notices, limits, summarize: async () => "WOKEN SUMMARY" });
+  const loop = store.start({ everyMs: 60000 });
+  for (let n = 0; n < 6; n += 1) {
+    await say(6, "user", `Message ${n}. ${"Detail. ".repeat(60)}`);
+    await say(6, "bot", `Answer ${n}`);
+  }
+  assert.equal(await store.consider(6), true);
+  assert.deepEqual(published, ["agent-summaries:all"]);
+  for (let tries = 0; tries < 40 && !(await knex("agent_summaries").where({ conversation_id: 6 }).first()); tries += 1) await new Promise((resolve) => setTimeout(resolve, 50));
+  await loop.drain(1000);
+  assert.equal((await store.current(6)).summary, "WOKEN SUMMARY");
+  assert.equal(listeners.length, 0, "drained, it stops listening");
+});