eve-hub 0.6.62 → 0.6.64

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # eve-hub
 
+## 0.6.64
+
+### Patch Changes
+
+- Hosted hub: who takes part in a thread is read with one index seek per bot, not by reading every message in it, so a send or a fold costs the bots in the thread; the reads the feed, sweep and runner polls repeat now use indexes and cost what is waiting, not what the workspace holds.
+
+## 0.6.63
+
+### Patch Changes
+
+- Hosted hub: the state snapshot reads about a fifth of the rows (the last-turn, cache-gap, visibility and queue reads were rewritten), with the same state.
+
 ## 0.6.62
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eve-hub",
-  "version": "0.6.62",
+  "version": "0.6.64",
   "license": "Apache-2.0",
   "description": "Bot-side code shared by every hub agent: registry tools, the roster hook, the eve channel, the host sandbox and browser, and model helpers.",
   "type": "module",

package/world.mjs

@@ -20178,14 +20178,14 @@ function eventStatements(threadId2, event, text, at, id = eventId(), only) {
 }
 
 // ../../hosted-hub/src/store/routing.ts
-var THREAD_REFS = 3;
-var threadArgs = (threadId2, ...rest) => [...Array(THREAD_REFS).fill(threadId2), ...rest];
-var PARTICIPANTS = (thread, except = "''") => `SELECT bot FROM (
-    SELECT bot, created_at AS at, 0 AS leave, NULL AS mid FROM sessions WHERE thread_id = ${thread}
-    UNION ALL SELECT author, created_at, 0, id FROM messages WHERE thread_id = ${thread} AND author_kind = 'agent'
-    UNION ALL SELECT json_extract(event, '$.agent'), created_at, json_extract(event, '$.kind') = 'leave', id FROM messages
-      WHERE thread_id = ${thread} AND author_kind = 'event' AND json_extract(event, '$.kind') IN ('join', 'leave'))
-  GROUP BY bot HAVING MAX(CASE WHEN NOT leave AND mid IS NOT ${except} THEN at END) > COALESCE(MAX(CASE WHEN leave THEN at END), -1)`;
+var PARTICIPANTS = (thread, except = "''") => `SELECT b.name AS bot FROM bots b, (SELECT ${thread} AS t, ${except} AS x) p WHERE MAX(
+    COALESCE((SELECT created_at FROM sessions WHERE thread_id = p.t AND bot = b.name ORDER BY created_at DESC LIMIT 1), -2),
+    COALESCE((SELECT created_at FROM messages WHERE thread_id = p.t AND author = b.name AND author_kind = 'agent' AND id IS NOT p.x ORDER BY created_at DESC LIMIT 1), -2),
+    COALESCE((SELECT created_at FROM messages WHERE thread_id = p.t AND author_kind = 'event' AND json_extract(event, '$.agent') = +b.name AND json_extract(event, '$.kind') = 'join'
+      AND id IS NOT p.x ORDER BY created_at DESC LIMIT 1), -2))
+  > COALESCE((SELECT created_at FROM messages WHERE thread_id = p.t AND author_kind = 'event' AND json_extract(event, '$.agent') = +b.name AND json_extract(event, '$.kind') = 'leave'
+      ORDER BY created_at DESC LIMIT 1), -1)`;
+var NOT_IN_BEFORE = `NOT EXISTS (SELECT 1 FROM (${PARTICIPANTS("?", "?")}) WHERE bot = ?)`;
 var hasJoins = (threadId2) => !threadId2.startsWith("dm-") && !threadId2.startsWith("sch-");
 function routingStatements(messageId2, threadId2, bots, now, { by, joins }, held = {}, steer = false) {
   const quiet2 = !joins || !hasJoins(threadId2);
@@ -20198,7 +20198,7 @@ function routingStatements(messageId2, threadId2, bots, now, { by, joins }, held
         `@${bot} joined${by !== bot ? ` (added by @${by})` : ""}`,
         now,
         eventId(),
-        s(`${fresh} AND NOT EXISTS (SELECT 1 FROM (${PARTICIPANTS("?")}) WHERE bot = ?)`, messageId2, ...threadArgs(threadId2), bot)
+        s(`${fresh} AND NOT EXISTS (SELECT 1 FROM (${PARTICIPANTS("?")}) WHERE bot = ?)`, messageId2, threadId2, bot)
       ),
       s(`INSERT OR IGNORE INTO sessions (id,thread_id,bot,created_at) SELECT ?,?,?,? WHERE ${fresh}`, sessionId(), threadId2, bot, now, messageId2),
       s(
@@ -20243,6 +20243,7 @@ var LEASE_LEFT = `SELECT l.setup_failed_until - ${NOW_MS} AS ms FROM local_runne
 
 // ../../hosted-hub/src/store/schema.ts
 var OPEN_TURN_STATUSES = "('streaming','cancelling')";
+var UNSETTLED_DELIVERIES = "unsettled AS MATERIALIZED (SELECT rowid AS rid, * FROM deliveries WHERE accepted_at IS NULL AND failed_at IS NULL AND revoked_at IS NULL)";
 
 // mention-tokens.mjs
 function codeRanges(text) {
@@ -27221,11 +27222,11 @@ var SESSION_WORK_DELIVERY = `busy.accepted_at IS NOT NULL AND busy.failed_at IS
     WHERE td.delivery_id = busy.id AND t.status NOT IN ('streaming','cancelling'))`;
 var SESSION_BUSY = `s.ended_at IS NULL AND EXISTS (SELECT 1 FROM deliveries busy
   WHERE busy.session_id = s.id AND ${SESSION_WORK_DELIVERY})`;
-var PENDING_FROM = `SELECT d.id, d.message_id, d.session_id, d.bot, m.thread_id, t.title, m.author, m.author_kind, m.text, m.status, m.status_from, m.metadata, d.steer,
+var PENDING_FROM = `WITH ${UNSETTLED_DELIVERIES} SELECT d.id, d.message_id, d.session_id, d.bot, m.thread_id, t.title, m.author, m.author_kind, m.text, m.status, m.status_from, m.metadata, d.steer,
       EXISTS (SELECT 1 FROM deliveries a WHERE a.session_id = d.session_id AND a.accepted_at IS NOT NULL) AS session_started,
       s.ended_at IS NOT NULL AS session_ended, ${onLocal("d.bot")} AS on_local,
       (SELECT json_group_array(json_object('id', a.upload_id, 'name', a.name, 'size', a.size)) FROM (SELECT * FROM message_attachments WHERE message_id = m.id ORDER BY position) a) AS attachments
-    FROM deliveries d JOIN messages m ON m.id = d.message_id JOIN threads t ON t.id = m.thread_id JOIN sessions s ON s.id = d.session_id`;
+    FROM unsettled d JOIN messages m ON m.id = d.message_id JOIN threads t ON t.id = m.thread_id JOIN sessions s ON s.id = d.session_id`;
 var UNSETTLED = `d.accepted_at IS NULL AND d.failed_at IS NULL AND d.revoked_at IS NULL AND d.held_at IS NULL AND NOT ${heldByHopLimit("m")}`;
 var TASK_CARD = `(SELECT CASE WHEN json_array_length(json_extract(t.activity, '$.taskCard.tasks')) > 0 THEN json_extract(t.activity, '$.taskCard') END
   FROM turns t WHERE t.id = m.turn_id AND m.author_kind = 'agent' AND t.status NOT IN ${OPEN_TURN_STATUSES}
@@ -27258,29 +27259,40 @@ function handedBack({ session, roster, asked = [], posted = [], notify }, text,
   if ([text, ...posted].some((t) => mentionTokens(t).some((name) => name !== me && known.has(name)))) return [];
   return [...new Set(asked.filter((a) => (a.linked || named.has(a.id)) && a.author !== session.bot && roster.includes(a.author) && mentionTokens(a.text).includes(me)).map((a) => a.author))];
 }
-var FOLD_STATE_SQL = `SELECT s.id, s.thread_id, s.bot, COALESCE(f.folded_index, -1) AS folded_index, (SELECT json_group_array(name) FROM (SELECT name FROM bots WHERE ${inRoster} ORDER BY position)) AS roster,
-    (SELECT json_group_array(bot) FROM (${PARTICIPANTS("s.thread_id")})) AS participants,
-    (SELECT d.message_id FROM deliveries d WHERE d.bot = s.bot AND d.held_at IS NOT NULL AND d.held_paused IS NOT 1 AND d.accepted_at IS NULL AND d.failed_at IS NULL AND d.revoked_at IS NULL ORDER BY d.rowid LIMIT 1) AS held,
-    (SELECT json_group_array(json_object('id', dd.id, 'author', m.author, 'text', m.text, 'linked', EXISTS (SELECT 1 FROM turn_deliveries td JOIN turns t ON t.id = td.turn_id WHERE td.delivery_id = dd.id AND t.status IN ${OPEN_TURN_STATUSES})))
-      FROM deliveries dd JOIN messages m ON m.id = dd.message_id WHERE dd.session_id = s.id AND m.author_kind = 'agent'
-        AND (EXISTS (SELECT 1 FROM turn_deliveries td JOIN turns t ON t.id = td.turn_id WHERE td.delivery_id = dd.id AND t.status IN ${OPEN_TURN_STATUSES}) OR NOT EXISTS (SELECT 1 FROM turn_deliveries td WHERE td.delivery_id = dd.id))) AS asked,
-    (SELECT json_group_array(p.text) FROM messages p WHERE p.turn_id = (SELECT t.id FROM turns t WHERE t.session_id = s.id AND t.status IN ${OPEN_TURN_STATUSES} ORDER BY t.started_at DESC LIMIT 1)
-      AND p.author = s.bot AND p.author_kind = 'agent' AND p.status IS NULL) AS posted,
-    (SELECT p.id FROM messages p WHERE p.turn_id = (SELECT t.id FROM turns t WHERE t.session_id = s.id AND t.status IN ${OPEN_TURN_STATUSES} ORDER BY t.started_at DESC LIMIT 1)
-      AND p.author = s.bot AND p.author_kind = 'agent' AND p.status IS NULL ORDER BY p.created_at DESC, p.rowid DESC LIMIT 1) AS last_post,
+var FOLD_STATE_SQL = `SELECT * FROM (WITH sess AS (SELECT * FROM sessions WHERE id = ?), ${UNSETTLED_DELIVERIES}
+  SELECT s.id, s.thread_id, s.bot, COALESCE(f.folded_index, -1) AS folded_index, (SELECT json_group_array(name) FROM (SELECT name FROM bots WHERE ${inRoster} ORDER BY position)) AS roster,
+    (SELECT d.message_id FROM unsettled d WHERE d.bot = s.bot AND d.held_at IS NOT NULL AND d.held_paused IS NOT 1 ORDER BY d.rid LIMIT 1) AS held,
     (SELECT json_object('person', person, 'names', json(names)) FROM human_notify) AS notify,
     (SELECT json_object('status', t.status, 'id', t.id, 'text', t.reply_text, 'lastStep', json(t.last_step), 'activity', json(t.activity),
       'trigger', ${triggerOf("t")}) FROM turns t WHERE t.session_id = s.id AND t.status IN ${OPEN_TURN_STATUSES}
       ORDER BY t.started_at DESC LIMIT 1) AS open_turn,
     (SELECT json_object('to', c.report_to, 'trigger', c.report_trigger, 'before', c.report_before, 'title', (SELECT title FROM threads WHERE id = s.thread_id))
       FROM compactions c WHERE c.session_id = s.id AND c.report_to IS NOT NULL) AS report
-  FROM sessions s LEFT JOIN session_folds f ON f.session_id = s.id WHERE s.id = ?`;
+  FROM sess s LEFT JOIN session_folds f ON f.session_id = s.id)`;
+var TURN_END_SQL = `SELECT * FROM (WITH sess AS (SELECT * FROM sessions WHERE id = ?),
+  open_turn AS (SELECT t.id FROM turns t WHERE t.session_id = (SELECT id FROM sess) AND t.status IN ${OPEN_TURN_STATUSES} ORDER BY t.started_at DESC LIMIT 1),
+  answered AS (SELECT delivery_id AS id FROM turn_deliveries WHERE turn_id = (SELECT id FROM open_turn) UNION SELECT value FROM json_each(?))
+  SELECT (SELECT json_group_array(bot) FROM (${PARTICIPANTS("(SELECT thread_id FROM sess)")})) AS participants,
+    (SELECT json_group_array(json_object('id', dd.id, 'author', m.author, 'text', m.text, 'linked', EXISTS (SELECT 1 FROM turn_deliveries td WHERE td.delivery_id = dd.id AND td.turn_id = (SELECT id FROM open_turn))))
+      FROM answered a CROSS JOIN deliveries dd ON dd.id = a.id CROSS JOIN messages m ON m.id = dd.message_id WHERE dd.session_id = (SELECT id FROM sess) AND m.author_kind = 'agent'
+        AND (EXISTS (SELECT 1 FROM turn_deliveries td WHERE td.delivery_id = dd.id AND td.turn_id = (SELECT id FROM open_turn)) OR NOT EXISTS (SELECT 1 FROM turn_deliveries td WHERE td.delivery_id = dd.id))) AS asked,
+    (SELECT json_group_array(p.text) FROM messages p WHERE p.turn_id = (SELECT id FROM open_turn)
+      AND p.author = (SELECT bot FROM sess) AND p.author_kind = 'agent' AND p.status IS NULL) AS posted,
+    (SELECT p.id FROM messages p WHERE p.turn_id = (SELECT id FROM open_turn)
+      AND p.author = (SELECT bot FROM sess) AND p.author_kind = 'agent' AND p.status IS NULL ORDER BY p.created_at DESC, p.rowid DESC LIMIT 1) AS last_post)`;
+var namedDeliveries = (chunks) => [...new Set(chunks.flatMap((c) => c.events.flatMap((e) => e.meta?.deliveryIds ?? [])))];
+var completes = (chunks) => chunks.some((c) => c.events.some((e) => e.type === "turn.completed"));
+async function readFoldState(get, sessionId2, chunks) {
+  const row = await get(FOLD_STATE_SQL, sessionId2);
+  if (!row) return void 0;
+  return foldState({ ...row, ...completes(chunks) ? await get(TURN_END_SQL, sessionId2, JSON.stringify(namedDeliveries(chunks))) : {} });
+}
 function foldState(row) {
   if (!row) return void 0;
   return {
     session: { id: String(row.id), thread: String(row.thread_id), bot: String(row.bot) },
     roster: JSON.parse(String(row.roster)),
-    participants: JSON.parse(String(row.participants)),
+    participants: JSON.parse(String(row.participants ?? "[]")),
     held: row.held == null ? null : String(row.held),
     asked: JSON.parse(String(row.asked ?? "[]")),
     posted: JSON.parse(String(row.posted ?? "[]")),
@@ -27530,7 +27542,7 @@ function fold(state, chunks, now = Date.now()) {
   return { statements: stmts, deliver };
 }
 async function foldInTx(tx, sessionId2, chunks) {
-  const state = foldState(await tx.get(FOLD_STATE_SQL, sessionId2));
+  const state = await readFoldState((sql, ...args) => tx.get(sql, ...args), sessionId2, chunks);
   const result = state ? fold(state, chunks) : foldSubagent(subagentState(sessionId2, await tx.get(SUBAGENT_STATE_SQL, sessionId2)), chunks);
   for (const stmt of result.statements) await tx.write(stmt.sql, ...stmt.args);
   return result.deliver;