@agentshouse/kit 0.1.0-alpha.4 → 0.1.0-alpha.40

package/src/sync.js

@@ -13,6 +13,7 @@ import {
   assertWritableChanges,
   authorityFor,
   deriveChanges,
+  housePath,
   OPERATIONS,
   relativeTo,
   resultingFiles,
@@ -41,12 +42,12 @@ function sessionDependencies(options, dependencies) {
   };
 }
 
-async function withSession(options, dependencies, operation) {
+export async function withSession(options, dependencies, operation) {
   const session = await openSession(options, sessionDependencies(options, dependencies));
   return maintainControl(session, (signal) => operation(session, signal), options.signal);
 }
 
-async function door(session, path, body, signal) {
+export async function door(session, path, body, signal) {
   return houseRequest(
     session.house,
     await heldControl(session, signal),
@@ -83,7 +84,7 @@ function changePosition(value, declaration) {
   return { position: value.position, log_epoch: value.log_epoch };
 }
 
-function discoveryOf(value) {
+export function discoveryOf(value) {
   if (typeof value?.md_model_version !== "string" || !Array.isArray(value.authorities)) {
     throw new KitRefusal("kit_sync_rejected", "discover");
   }
@@ -93,7 +94,7 @@ function discoveryOf(value) {
   return value;
 }
 
-async function bootstrapBundle(session, prefixes, signal) {
+export async function bootstrapBundle(session, prefixes, signal) {
   const answer = await door(session, KIT_DOOR_BOOTSTRAP_PATH, { prefixes }, signal);
   if (typeof answer?.transfer?.url === "string") {
     const granted = await houseGetJson(answer.transfer.url, session.dependencies, signal);
@@ -110,7 +111,7 @@ function matchedDiscovery(discovery, scope) {
   );
 }
 
-async function writeScope(replica, scope, signal) {
+function scopeContents(replica, scope) {
   if (
     typeof scope?.manifest?.path !== "string" ||
     typeof scope.manifest.position !== "string" ||
@@ -121,36 +122,54 @@ async function writeScope(replica, scope, signal) {
   if (!Array.isArray(scope.contents) || !Array.isArray(scope.manifest.files)) {
     throw new KitRefusal("kit_bootstrap_rejected");
   }
-  const files = [];
   for (const entry of scope.contents) {
     if (typeof entry?.path !== "string" || typeof entry.content !== "string") {
       throw new KitRefusal("kit_bootstrap_rejected");
     }
-    const meta = scope.manifest.files.find((file) => file.path === entry.path);
+    locate(replica, scope.manifest.path, entry.path);
+  }
+  return scope.contents;
+}
+
+function servedFile(scope, path) {
+  return scope.manifest.files.find((file) => file?.path === path);
+}
+
+async function writeScope(replica, scope, signal) {
+  const recorded = new Map();
+  for (const entry of scopeContents(replica, scope)) {
+    const meta = servedFile(scope, entry.path) ?? {};
     await writeReplicaFile(locate(replica, scope.manifest.path, entry.path), entry.content, signal);
-    files.push({
-      path: entry.path,
-      revision: typeof meta?.revision === "string" ? meta.revision : "",
-      sha256: typeof meta?.sha256 === "string" ? meta.sha256 : "",
-      content: entry.content,
-    });
+    recorded.set(entry.path, enumeratedRecord(entry.path, entry.content, meta));
+    if (typeof meta.provenance === "string") {
+      await writeSidecar({ replica }, scope.manifest, recorded, entry.path, meta, signal);
+    }
   }
-  return files;
+  return [...recorded.values()];
 }
 
 async function replaceScope(replica, previous, scope, signal) {
-  const next = await writeScope(replica, scope, signal);
-  const kept = new Set(next.map((file) => file.path));
+  const kept = new Set(
+    scopeContents(replica, scope).flatMap((entry) =>
+      typeof servedFile(scope, entry.path)?.provenance === "string"
+        ? [entry.path, sidecarPath(entry.path)]
+        : [entry.path],
+    ),
+  );
   const root = locate(replica, scope.manifest.path);
   for (const path of await listFiles(root)) {
-    if (!kept.has(path)) await removeReplicaFile(locate(replica, scope.manifest.path, path));
+    if (!kept.has(path)) await removeReplicaFile(replica, scope.manifest.path, path);
   }
   if (previous !== undefined) {
     for (const file of previous.files) {
-      if (!kept.has(file.path)) await removeReplicaFile(locate(replica, previous.path, file.path));
+      if (!kept.has(file.path)) await removeReplicaFile(replica, previous.path, file.path);
     }
   }
-  return next;
+  return writeScope(replica, scope, signal);
+}
+
+function removalsFirst(entries, removal) {
+  return [...entries.filter(removal), ...entries.filter((entry) => !removal(entry))];
 }
 
 function stringList(value) {
@@ -238,6 +257,18 @@ function enumeratedRecord(path, content, change) {
   };
 }
 
+async function writeSidecar(options, authority, recorded, path, change, signal) {
+  const sidecar = sidecarPath(path);
+  await writeReplicaFile(locate(options.replica, authority.path, sidecar), change.provenance, signal);
+  recorded.set(sidecar, enumeratedRecord(sidecar, change.provenance, change));
+}
+
+async function removeSidecar(options, authority, recorded, path) {
+  const sidecar = sidecarPath(path);
+  await removeReplicaFile(options.replica, authority.path, sidecar);
+  recorded.delete(sidecar);
+}
+
 async function applyEnumeration(options, state, enumeration, signal) {
   if (!Array.isArray(enumeration?.authorities)) throw new KitRefusal("kit_sync_rejected", "enumerate");
   for (const view of enumeration.authorities) {
@@ -245,11 +276,12 @@ async function applyEnumeration(options, state, enumeration, signal) {
     if (authority === undefined) throw new KitRefusal("kit_sync_rejected", view.authority);
     const reached = changePosition(view, "enumerate");
     const recorded = new Map(authority.files.map((file) => [file.path, file]));
-    for (const change of view.changes ?? []) {
+    for (const change of removalsFirst(view.changes ?? [], (change) => change?.removed === true)) {
       if (typeof change?.path !== "string") throw new KitRefusal("kit_sync_rejected", "enumerate");
       if (change.removed === true) {
-        await removeReplicaFile(locate(options.replica, authority.path, change.path));
+        await removeReplicaFile(options.replica, authority.path, change.path);
         recorded.delete(change.path);
+        await removeSidecar(options, authority, recorded, change.path);
         continue;
       }
       if (typeof change.content === "string") {
@@ -261,13 +293,7 @@ async function applyEnumeration(options, state, enumeration, signal) {
         recorded.set(change.path, enumeratedRecord(change.path, change.content, change));
       }
       if (typeof change.provenance === "string") {
-        const sidecar = sidecarPath(change.path);
-        await writeReplicaFile(
-          locate(options.replica, authority.path, sidecar),
-          change.provenance,
-          signal,
-        );
-        recorded.set(sidecar, enumeratedRecord(sidecar, change.provenance, change));
+        await writeSidecar(options, authority, recorded, change.path, change, signal);
       }
     }
     authority.files = [...recorded.values()];
@@ -299,13 +325,41 @@ function uncertainTransport(error) {
   );
 }
 
-async function acceptBatch(options, state, outcome, signal, pending = null) {
+async function servedProvenance(session, options, state, authority, signal) {
+  const enumeration = await replicaDoor(
+    session,
+    options,
+    state,
+    KIT_DOOR_ENUMERATE_PATH,
+    {
+      prefixes: state.prefixes.filter((prefix) => authorityFor(state, prefix) === authority),
+      positions: [
+        { authority: authority.authority, position: authority.position, log_epoch: authority.log_epoch },
+      ],
+    },
+    signal,
+  );
+  const view = Array.isArray(enumeration?.authorities)
+    ? enumeration.authorities.find((entry) => entry?.authority === authority.authority)
+    : undefined;
+  if (!Array.isArray(view?.changes)) throw new KitRefusal("kit_sync_rejected", "enumerate");
+  return new Map(
+    view.changes
+      .filter((change) => typeof change?.provenance === "string")
+      .map((change) => [change.path, change]),
+  );
+}
+
+async function acceptBatch(session, options, state, outcome, signal, pending = null) {
   const authority = state.authorities.find((entry) => entry.authority === outcome.authority);
   if (authority === undefined) throw new KitRefusal("kit_sync_rejected", outcome.authority);
   const reached = changePosition(outcome, "batch");
   const recorded = new Map(authority.files.map((file) => [file.path, file]));
   const predicted = predictedReplica(authority, pending);
-  for (const file of outcome.contents ?? []) {
+  const provenance = (outcome.contents ?? []).some((file) => typeof file?.content === "string")
+    ? await servedProvenance(session, options, state, authority, signal)
+    : new Map();
+  for (const file of removalsFirst(outcome.contents ?? [], (file) => file?.content === null)) {
     const relative = relativeTo(authority.path, file.path);
     const destination = locate(options.replica, authority.path, relative);
     const local = await replicaText(destination);
@@ -319,8 +373,9 @@ async function acceptBatch(options, state, outcome, signal, pending = null) {
       local !== accepted &&
       local !== expected;
     if (accepted === null) {
-      if (!preserve) await removeReplicaFile(destination);
+      if (!preserve) await removeReplicaFile(options.replica, authority.path, relative);
       recorded.delete(relative);
+      await removeSidecar(options, authority, recorded, relative);
       continue;
     }
     if (!preserve) await writeReplicaFile(destination, accepted, signal);
@@ -330,6 +385,10 @@ async function acceptBatch(options, state, outcome, signal, pending = null) {
       sha256: "",
       content: accepted,
     });
+    const served = provenance.get(relative);
+    if (served !== undefined && served.revision === file.revision) {
+      await writeSidecar(options, authority, recorded, relative, served, signal);
+    }
   }
   authority.files = [...recorded.values()];
   Object.assign(authority, reached);
@@ -341,16 +400,65 @@ function sameBody(left, right) {
   return JSON.stringify(left) === JSON.stringify(right);
 }
 
+function contentMap(authority) {
+  return Object.fromEntries(authority.files.map((file) => [file.path, file.content]));
+}
+
+async function pullConfirmed(session, options, state, signal) {
+  const enumeration = await replicaDoor(
+    session,
+    options,
+    state,
+    KIT_DOOR_ENUMERATE_PATH,
+    { prefixes: state.prefixes, positions: positionsOf(state) },
+    signal,
+  );
+  return applyEnumeration(options, state, enumeration, signal);
+}
+
+async function settleExpired(session, options, state, pending, signal) {
+  const authority = authorityFor(state, pending.body.changes[0].path);
+  const predicted = predictedReplica(authority, pending);
+  const before = contentMap(authority);
+  await pullConfirmed(session, options, state, signal);
+  const confirmed = contentMap(authority);
+  const touched = [...new Set([...Object.keys(before), ...Object.keys(predicted)])]
+    .filter((path) => before[path] !== predicted[path])
+    .sort();
+  const landed = touched.length > 0 && touched.every((path) => confirmed[path] === predicted[path]);
+  await clearPending(options.directory, state.directory);
+  if (!landed) {
+    return { pending, accepted: await sendBatch(session, options, state, pending.body.changes, signal) };
+  }
+  const recorded = new Map(authority.files.map((file) => [file.path, file]));
+  return {
+    pending,
+    accepted: {
+      authority: authority.authority,
+      position: authority.position,
+      log_epoch: authority.log_epoch,
+      contents: touched.map((path) => ({
+        path: housePath(authority.path, path),
+        content: recorded.get(path)?.content ?? null,
+        revision: recorded.get(path)?.revision ?? null,
+      })),
+    },
+  };
+}
+
 async function recoverPending(session, options, state, signal) {
   const pending = await readPending(options.directory, state.directory);
   if (pending === null) return null;
   try {
     const outcome = await replicaDoor(session, options, state, pending.path, pending.body, signal);
-    const accepted = await acceptBatch(options, state, outcome, signal, pending);
+    const accepted = await acceptBatch(session, options, state, outcome, signal, pending);
     await clearPending(options.directory, state.directory);
     return { pending, accepted };
   } catch (error) {
     if (uncertainTransport(error)) throw error;
+    if (error.code === "operation_expired") {
+      return settleExpired(session, options, state, pending, signal);
+    }
     await clearPending(options.directory, state.directory);
     throw error;
   }
@@ -386,7 +494,7 @@ async function sendBatch(session, options, state, changes, signal) {
     );
     try {
       const outcome = await replicaDoor(session, options, state, KIT_DOOR_BATCH_PATH, body, signal);
-      last = await acceptBatch(options, state, outcome, signal);
+      last = await acceptBatch(session, options, state, outcome, signal);
       await clearPending(options.directory, state.directory);
     } catch (error) {
       if (uncertainTransport(error)) throw error;
@@ -431,16 +539,7 @@ export async function bootstrap(options, dependencies = {}) {
 
 export async function pull(options, dependencies = {}) {
   return withSession(options, dependencies, async (session, signal) => {
-    const state = await loadState(options);
-    const enumeration = await replicaDoor(
-      session,
-      options,
-      state,
-      KIT_DOOR_ENUMERATE_PATH,
-      { prefixes: state.prefixes, positions: positionsOf(state) },
-      signal,
-    );
-    const next = await applyEnumeration(options, state, enumeration, signal);
+    const next = await pullConfirmed(session, options, await loadState(options), signal);
     return { positions: positionsOf(next) };
   });
 }

package/src/tasks.js

@@ -0,0 +1,92 @@
+import { setTimeout as delay } from "node:timers/promises";
+
+export function coalescedTask(signal, fail, work) {
+  let pending = false;
+  let task = null;
+  const trigger = () => {
+    pending = true;
+    if (task !== null) return;
+    task = (async () => {
+      while (pending && !signal.aborted) {
+        pending = false;
+        await work();
+      }
+    })()
+      .catch((error) => {
+        if (signal.aborted) return;
+        fail(error);
+      })
+      .finally(() => {
+        task = null;
+        if (pending && !signal.aborted) trigger();
+      });
+  };
+  return { trigger, idle: () => task === null, settled: () => task };
+}
+
+export function orderedTask(signal, fail, work) {
+  let task = null;
+  const push = (...args) => {
+    const next = (task ?? Promise.resolve())
+      .then(() => (signal.aborted ? undefined : work(...args)))
+      .catch((error) => {
+        if (signal.aborted) return;
+        fail(error);
+      });
+    task = next;
+    void next.then(() => {
+      if (task === next) task = null;
+    });
+  };
+  return { push, idle: () => task === null, settled: () => task };
+}
+
+export function residentWake() {
+  let pending = false;
+  let release = null;
+  return {
+    notify() {
+      pending = true;
+      release?.();
+    },
+    async wait(milliseconds, signal) {
+      if (!pending && !signal.aborted) {
+        const pause = new AbortController();
+        const paused = AbortSignal.any([signal, pause.signal]);
+        try {
+          await Promise.race([
+            new Promise((resolve) => {
+              release = resolve;
+            }),
+            Number.isFinite(milliseconds)
+              ? delay(milliseconds, undefined, { signal: paused })
+              : new Promise((resolve) => paused.addEventListener("abort", resolve, { once: true })),
+          ]);
+        } catch (error) {
+          if (!signal.aborted) throw error;
+        } finally {
+          release = null;
+          pause.abort();
+        }
+      }
+      const woken = pending;
+      pending = false;
+      return woken;
+    },
+  };
+}
+
+const turns = new Map();
+
+export function exclusively(path, work) {
+  const turn = (turns.get(path) ?? Promise.resolve()).then(work);
+  const settled = turn.then(
+    () => undefined,
+    () => undefined,
+  );
+  turns.set(path, settled);
+  void settled.then(() => {
+    if (turns.get(path) === settled) turns.delete(path);
+  });
+  return turn;
+}

package/src/telegram-view.js

@@ -0,0 +1,217 @@
+import { createHash } from "node:crypto";
+import {
+  TELEGRAM_RICH_MESSAGE_BLOCKS_MAX,
+  TELEGRAM_RICH_MESSAGE_CHARACTERS_MAX,
+} from "./contract.js";
+
+const PARTS = Object.freeze({ draft: "blocks", plan: "steps" });
+const UNCOUNTED = new Set(["type", "language"]);
+
+function richCharacters(value, key = null) {
+  if (typeof value === "string") return UNCOUNTED.has(key) ? 0 : value.length;
+  if (Array.isArray(value)) return value.reduce((sum, item) => sum + richCharacters(item), 0);
+  if (typeof value === "object" && value !== null) {
+    return Object.entries(value).reduce((sum, [name, item]) => sum + richCharacters(item, name), 0);
+  }
+  return 0;
+}
+
+function richText(value, key = null) {
+  if (typeof value === "string") return UNCOUNTED.has(key) ? [] : [value];
+  if (Array.isArray(value)) return value.flatMap((item) => richText(item));
+  if (typeof value === "object" && value !== null) {
+    return Object.entries(value).flatMap(([name, item]) => richText(item, name));
+  }
+  return [];
+}
+
+export function slices(text, limit = TELEGRAM_RICH_MESSAGE_CHARACTERS_MAX) {
+  const pieces = [];
+  let start = 0;
+  while (start < text.length) {
+    let end = Math.min(start + limit, text.length);
+    const window = text.slice(start, end);
+    const line = window.lastIndexOf("\n");
+    const last = text.charCodeAt(end - 1);
+    if (end < text.length && line > 0 && /\S/.test(window.slice(0, line))) end = start + line + 1;
+    else if (end < text.length && last >= 0xd800 && last <= 0xdbff) end -= 1;
+    pieces.push(text.slice(start, end));
+    start = end;
+  }
+  return pieces;
+}
+
+function grouped(block, listed) {
+  const groups = [];
+  let group = [];
+  let characters = 0;
+  for (const entry of block[listed]) {
+    const length = richCharacters(entry);
+    if (group.length > 0 && characters + length > TELEGRAM_RICH_MESSAGE_CHARACTERS_MAX) {
+      groups.push(group);
+      group = [];
+      characters = 0;
+    }
+    group.push(entry);
+    characters += length;
+  }
+  if (group.length > 0) groups.push(group);
+  return groups.flatMap((entries) => fitted({ ...block, [listed]: entries }));
+}
+
+function fitted(block) {
+  if (richCharacters(block) <= TELEGRAM_RICH_MESSAGE_CHARACTERS_MAX) return [block];
+  if (typeof block.text === "string") return slices(block.text).map((text) => ({ ...block, text }));
+  if (typeof block.expression === "string") {
+    return slices(block.expression).map((expression) => ({ ...block, expression }));
+  }
+  for (const listed of ["items", "cells"]) {
+    if (Array.isArray(block[listed]) && block[listed].length > 1) return grouped(block, listed);
+  }
+  return slices(richText(block).join("\n")).map((text) => ({ type: "paragraph", text }));
+}
+
+export function richPieces(message) {
+  const pieces = [];
+  let blocks = [];
+  let characters = 0;
+  for (const part of message.blocks.flatMap(fitted)) {
+    const length = richCharacters(part);
+    if (
+      blocks.length > 0 &&
+      (blocks.length >= TELEGRAM_RICH_MESSAGE_BLOCKS_MAX ||
+        characters + length > TELEGRAM_RICH_MESSAGE_CHARACTERS_MAX)
+    ) {
+      pieces.push({ ...message, blocks });
+      blocks = [];
+      characters = 0;
+    }
+    blocks.push(part);
+    characters += length;
+  }
+  pieces.push({ ...message, blocks });
+  return pieces;
+}
+
+function listItems(items, ordered) {
+  return items.map((item, index) => ({
+    blocks: [{ type: "paragraph", text: item }],
+    ...(ordered ? { value: index + 1, type: "1" } : {}),
+  }));
+}
+
+function telegramBlock(block) {
+  if (typeof block !== "object" || block === null) return null;
+  if (block.type === "paragraph") return { type: "paragraph", text: block.text };
+  if (block.type === "heading") return { type: "heading", size: block.level, text: block.text };
+  if (block.type === "code") {
+    return {
+      type: "pre",
+      ...(typeof block.language === "string" ? { language: block.language } : {}),
+      text: block.text,
+    };
+  }
+  if (block.type === "list" && Array.isArray(block.items)) {
+    return { type: "list", items: listItems(block.items, block.ordered === true) };
+  }
+  if (block.type === "table" && Array.isArray(block.headers) && Array.isArray(block.rows)) {
+    return {
+      type: "table",
+      cells: [
+        block.headers.map((text) => ({ text, is_header: true })),
+        ...block.rows.map((row) => (Array.isArray(row) ? row : []).map((text) => ({ text }))),
+      ],
+      is_bordered: true,
+    };
+  }
+  if (block.type === "formula") return { type: "mathematical_expression", expression: block.expression };
+  return null;
+}
+
+function planBlocks(steps) {
+  return [
+    { type: "heading", text: "Plan", size: 3 },
+    {
+      type: "list",
+      items: steps.map((step, index) => ({
+        blocks: [{ type: "paragraph", text: [`${step.label} — `, { type: "bold", text: step.status }] }],
+        value: index + 1,
+        type: "1",
+      })),
+    },
+  ];
+}
+
+function draftId(turn) {
+  return (createHash("sha256").update(turn).digest().readUInt32BE(0) % 2_147_483_647) + 1;
+}
+
+function viewFrame(frame) {
+  if (typeof frame !== "object" || frame === null || !Object.hasOwn(PARTS, frame.type)) return null;
+  for (const named of ["conversation_id", "turn_id"]) {
+    if (typeof frame[named] !== "string" || frame[named] === "") return null;
+  }
+  if (!Number.isSafeInteger(frame.sequence) || frame.sequence < 1) return null;
+  if (!Number.isSafeInteger(frame.from) || frame.from < 0) return null;
+  return Array.isArray(frame[PARTS[frame.type]]) ? frame : null;
+}
+
+export function turnViews(options) {
+  let threads = new Map();
+  const views = new Map();
+
+  function applied(view, frame) {
+    const items = frame[PARTS[frame.type]];
+    const held = view[frame.type];
+    if (held !== null && frame.sequence <= held.sequence) return false;
+    if (frame.from === 0) {
+      view[frame.type] = { sequence: frame.sequence, items };
+      view.asked = false;
+      return true;
+    }
+    if (held === null || frame.sequence !== held.sequence + 1 || frame.from > held.items.length) {
+      if (!view.asked) {
+        view.asked = true;
+        options.gap({ type: "gap", conversation_id: frame.conversation_id, turn_id: frame.turn_id });
+      }
+      return false;
+    }
+    view[frame.type] = { sequence: frame.sequence, items: [...held.items.slice(0, frame.from), ...items] };
+    return true;
+  }
+
+  return {
+    address(listed) {
+      threads = new Map(listed.map((thread) => [thread.conversation_id, thread]));
+      for (const conversation of views.keys()) {
+        if (!threads.has(conversation)) views.delete(conversation);
+      }
+    },
+    receive(value) {
+      const frame = viewFrame(value);
+      if (frame === null || !threads.has(frame.conversation_id)) return null;
+      let view = views.get(frame.conversation_id);
+      if (view === undefined || view.turn !== frame.turn_id) {
+        view = { turn: frame.turn_id, draft: null, plan: null, asked: false };
+        views.set(frame.conversation_id, view);
+      }
+      return applied(view, frame) ? frame.conversation_id : null;
+    },
+    draft(conversation) {
+      const view = views.get(conversation);
+      const thread = threads.get(conversation);
+      if (view === undefined || thread === undefined) return null;
+      const blocks = [
+        ...(view.plan === null ? [] : planBlocks(view.plan.items)),
+        ...(view.draft === null ? [] : view.draft.items.map(telegramBlock).filter((block) => block !== null)),
+      ];
+      if (blocks.length === 0) return null;
+      return {
+        chat_id: thread.chat_id,
+        topic_id: thread.topic_id,
+        draft_id: draftId(view.turn),
+        rich_message: richPieces({ blocks })[0],
+      };
+    },
+  };
+}