@chloejs/core 0.2.0

package/model/tools/index.ts

@@ -0,0 +1,28 @@
+// What a model can be given, to the repo that installs chloe.
+//
+// Everything here is a tool: a wrapper that lets a model reach work it could
+// not have been told the rule for. Each one is a function an agent calls with
+// its own folder, its own mailbox, its own From line, so nothing here names an
+// agent or a person.
+//
+//   import { readMail, readWeb } from "@chloejs/core/tools";
+//
+// The notes tools, write_skill and run_script are not here: an agent turns
+// them on with `features` in its definition.
+//
+// The work itself is in do/, published from "@chloejs/core", and a job calls it
+// from a step rather than coming through here. If a job imports this file,
+// something is in the wrong place.
+//
+// Same rule as `index.ts`: adding a name here is publishing it.
+
+
+// One folder, as tools, for an agent that needs a different set.
+export { listIn, readIn, searchIn, writeIn } from "./files.ts";
+
+// Mail in, mail out.
+export { readMail } from "./gmail.ts";
+export { sendEmail } from "./send_email.ts";
+
+// Reading a public web page.
+export { readWeb } from "./web.ts";

package/model/tools/memory.ts

@@ -0,0 +1,23 @@
+// An agent's memory, as tools: list_notes, read_notes, search_notes and
+// write_notes, all inside the one folder its definition's `memory` names
+// (its own folder under the state directory when it names none). Every agent
+// has them: the loader adds them, so an agent's `tools` never lists them.
+import { mkdirSync } from "node:fs";
+
+import { listIn, readIn, searchIn, writeIn } from "./files.ts";
+import type { Tools } from "../tool.ts";
+
+/** The four notes tools for one agent's memory. A write is a git commit when `commit` says so. */
+export function memoryTools(memory: { folder: string; commit?: boolean }): Tools {
+  const { folder, commit } = memory;
+  const what = "your memory";
+  // A new agent has no folder yet, and the first thing it does should not be
+  // to fail on one missing.
+  mkdirSync(folder, { recursive: true });
+  return {
+    list_notes: listIn({ root: folder, what }),
+    read_notes: readIn({ root: folder, what }),
+    search_notes: searchIn({ root: folder, what }),
+    write_notes: writeIn({ root: folder, what, commit }),
+  };
+}

package/model/tools/run_script.ts

@@ -0,0 +1,44 @@
+// The tool over do/scripts.ts: an agent running one of its own scripts.
+//
+// This is the plug-and-play half of the system. A capability is a script in
+// the agent's `scripts/` folder plus a short file in its `skills/` folder
+// saying when to use it. Neither is TypeScript and neither needs a restart.
+import { existsSync, readdirSync } from "node:fs";
+
+import { z } from "zod";
+
+import { agentDir } from "#chloe/core/paths.ts";
+import { script, scripts } from "#chloe/do/scripts.ts";
+import { tool, type Tools } from "#chloe/model/tool.ts";
+
+/** A tool that runs one file from that agent's own `scripts/` folder. */
+export function runScript(agent: string) {
+  return tool({
+    id: "run_script",
+    description:
+      "Run one of your own scripts and return what it printed. Your skills say which script to " +
+      "use and what its arguments mean. Use `list` to see what you have.",
+    inputSchema: z.object({
+      script: z.string().describe(`A file in scripts/, or "list" to see them all.`),
+      args: z.array(z.string()).optional().describe("Arguments, one per item."),
+      timeoutSeconds: z.number().int().min(5).max(1800).optional(),
+    }),
+    execute: async ({ script: name, args = [], timeoutSeconds }) =>
+      name === "list"
+        ? { scripts: await scripts(agent) }
+        : script(agent, name, args, { timeoutMs: (timeoutSeconds ?? 300) * 1000 }),
+  });
+}
+
+/**
+ * run_script: the agent runs any file in its scripts/ folder, and nothing
+ * else. Refused as the agent loads when that folder has nothing in it.
+ */
+export function runScripts(): (agent: { name: string }) => Tools {
+  return ({ name }) => {
+    const dir = `${agentDir(name)}/scripts`;
+    const any = existsSync(dir) && readdirSync(dir, { withFileTypes: true }).some((e) => e.isFile() && !e.name.startsWith("."));
+    if (!any) throw new Error(`${name} has features.runScripts on, and ${dir} has no scripts in it.`);
+    return { run_script: runScript(name) };
+  };
+}

package/model/tools/send_email.ts

@@ -0,0 +1,29 @@
+// The tool over do/email.ts: sending one email.
+//
+// The agent binds its own From line and its own recipients. All the model
+// writes is the subject and the body.
+import { z } from "zod";
+
+import { type Address, send } from "#chloe/do/email.ts";
+import { tool } from "#chloe/model/tool.ts";
+
+interface Sender extends Address {
+  /** Who it reaches and when to use it, in the agent's own words. Shown to the model. */
+  when: string;
+}
+
+/**
+ * A tool that sends mail from the address the agent was given, to the address
+ * it was given.
+ */
+export function sendEmail({ when, ...address }: Sender) {
+  return tool({
+    id: "send_email",
+    description: `Send an email. ${when}`,
+    inputSchema: z.object({
+      subject: z.string().min(5).max(120),
+      body: z.string().min(20).describe("Plain text. Lead with what happened and what you did."),
+    }),
+    execute: ({ subject, body }) => send(address, subject, body),
+  });
+}

package/model/tools/web.ts

@@ -0,0 +1,23 @@
+// The tool over do/web.ts: reading one public web page.
+import { z } from "zod";
+
+import { readPage } from "#chloe/do/web.ts";
+import { tool } from "#chloe/model/tool.ts";
+
+/** A tool that reads one public web page as plain text. */
+export function readWeb() {
+  return tool({
+    id: "read_page",
+    description:
+      "Read a public web page as plain text. Links come back as `[text](url)`: to follow one, pass that url " +
+      "exactly as it came back, never one you rebuilt by hand, because one changed character can make a site " +
+      "quietly show a different page. A long page comes back in slices: pass `from` as the `next` it gave you. " +
+      "Only a page that came back nearly empty was built by JavaScript in the browser; if a page came back " +
+      "full but wrong, say what you asked for and what you got, and do not guess why.",
+    inputSchema: z.object({
+      url: z.url(),
+      from: z.number().int().min(0).optional().describe("Where to start, from the last slice's `next`."),
+    }),
+    execute: ({ url, from }) => readPage(url, from ?? 0),
+  });
+}

package/model/tools/write_skill.ts

@@ -0,0 +1,31 @@
+// An agent rewriting one of its own skills.
+//
+// This is the self-improving part, and it is deliberately the only part. A
+// skill is markdown: when to do something, which script does it, and what the
+// output means. An agent that learns something about its own job can put it
+// where the next run will read it, and the change is live in about fifteen
+// seconds with no restart.
+//
+// What an agent may NOT write is its tools and its scripts/ folder. Those are
+// code, and code it writes is code it then runs as itself. A skill can only
+// point at a script that a person already put there.
+//
+// Every write is a git commit, so self-improvement always leaves a diff.
+import { writeIn } from "./files.ts";
+import { agentDir } from "#chloe/core/paths.ts";
+import type { Tools } from "../tool.ts";
+
+/** A tool that rewrites one of the agent's own skills. Every write is a commit. */
+export function writeSkill(agent: string) {
+  return writeIn({
+    root: `${agentDir(agent)}/skills`,
+    what: "your own skills",
+    id: "write_skill",
+    commit: true,
+  });
+}
+
+/** write_skill: the agent rewrites its own skills, and every write is a commit. */
+export function selfImprovement(): (agent: { name: string }) => Tools {
+  return ({ name }) => ({ write_skill: writeSkill(name) });
+}

package/ops/account.ts

@@ -0,0 +1,109 @@
+// Making the one account, from a shell on the box that runs chloe.
+//
+//   npm run account            asks for a username and a password
+//   npm run account carlos     asks for the password only
+//
+// There used to be a setup form on the first visit. There is no page on this
+// port any more, so the first account is made here instead, which is better
+// anyway: it takes a shell rather than whoever reaches the address first.
+//
+// Changing the account later means deleting data/login.json and running this
+// again, which is deliberate and takes the same shell.
+import { createAccount, hasAccount } from "#chloe/serve/login.ts";
+
+if (hasAccount()) {
+  console.error("There is already an account. To change it, delete data/login.json and run this again.");
+  process.exit(1);
+}
+
+/**
+ * Whatever was typed past the end of the line just read, kept for the next
+ * question. Piped input arrives as one chunk holding every answer at once, so a
+ * reader that dropped the remainder would lose the second question's answer.
+ */
+let spare = "";
+
+/**
+ * One line from the terminal. A hidden one is not printed back, so a password is
+ * not left on the screen or in the scrollback. Read a character at a time rather
+ * than with readline, because readline reads ahead and the next question would
+ * find its answer already eaten.
+ */
+function line(question: string, hide: boolean): Promise<string> {
+  process.stdout.write(question);
+  return new Promise((done, fail) => {
+    const input = process.stdin;
+    const wasRaw = input.isRaw;
+    let said = "";
+
+    function finish(error?: Error): void {
+      input.off("data", more);
+      if (input.isTTY) input.setRawMode(Boolean(wasRaw));
+      input.pause();
+      process.stdout.write("\n");
+      if (error) fail(error);
+      else done(said.trim());
+    }
+
+    /** Takes one line out of `text`. True when it found the end of one. */
+    function take(text: string): boolean {
+      for (let at = 0; at < text.length; at++) {
+        const one = text[at];
+        if (one === "\r" || one === "\n") {
+          spare = text.slice(at + 1).replace(/^\n/, "");
+          return true;
+        }
+        if (one === "") throw new Error("Stopped.");
+        if (one === "" || one === "\b") {
+          said = said.slice(0, -1);
+          if (!hide && input.isTTY) process.stdout.write("\b \b");
+        } else {
+          said += one;
+          // Raw mode turns the terminal's own echo off, so an answer that is
+          // meant to be seen has to be written back here.
+          if (!hide && input.isTTY) process.stdout.write(one);
+        }
+      }
+      return false;
+    }
+
+    function more(chunk: string): void {
+      try {
+        if (take(chunk)) finish();
+      } catch (error) {
+        finish(error as Error);
+      }
+    }
+
+    const held = spare;
+    spare = "";
+    try {
+      if (take(held)) return void finish();
+    } catch (error) {
+      return void finish(error as Error);
+    }
+
+    if (input.isTTY) input.setRawMode(true);
+    input.setEncoding("utf8");
+    input.resume();
+    input.on("data", more);
+  });
+}
+
+const username = process.argv[2] ?? (await line("Username: ", false));
+const password = await line("Password: ", true);
+const again = await line("Again:    ", true);
+
+if (password !== again) {
+  console.error("Those two are not the same.");
+  process.exit(1);
+}
+
+try {
+  createAccount(username, password);
+} catch (error) {
+  console.error((error as Error).message);
+  process.exit(1);
+}
+
+console.log(`\nDone. ${username} can sign in at whatever serves the page.`);

package/ops/agent.ts

@@ -0,0 +1,290 @@
+#!/usr/bin/env node
+// One agent, from the terminal. Run by a person, to find out what an agent
+// actually knows and what its jobs actually do:
+//
+//   npm run agent <agent>               a conversation, until /exit
+//   npm run agent <agent> "question"    one question, then out
+//   npm run agent <agent> < draft.md    the same, piped
+//   npm run agent <agent> <job>    run one of its jobs now, and
+//                                       follow it step by step
+//
+// The last one is how a job is tried without waiting for its cron line. What
+// it is depends on what it matches: anything that is the id of one
+// of that agent's jobs runs that job, and everything else is a
+// question. So `npm run agent cc check-sites` is a run and `npm run agent cc
+// "is the disk full?"` is a question.
+//
+// It asks the running service over loopback rather than loading the runtime
+// itself. A second runtime would be a second writer on the database and a
+// second clock firing the same cron lines, so the nightly backup could go
+// twice.
+//
+// Every turn lands in the run history with its tool calls and its cost, the
+// same as one from the page or a job.
+import { readFileSync } from "node:fs";
+import { createInterface } from "node:readline/promises";
+
+import { ownCookie } from "#chloe/serve/login.ts";
+
+// Matches HOST and PORT in serve/http.ts, which are deliberately not settable.
+const BASE = "http://127.0.0.1:3067";
+
+interface Result {
+  runId: string;
+  text: string;
+  steps: number;
+  cost: number;
+  calls: { tool: string; args: unknown; result: unknown }[];
+}
+
+interface Listed {
+  name: string;
+  description?: string;
+  model: string;
+  tools: string[];
+  skills: string[];
+  jobs: ListedJob[];
+}
+
+interface ListedJob {
+  id: string;
+  description?: string;
+  cron?: string;
+  timezone: string;
+  /** "code" for a job, or the model a prompt asks. */
+  model: string;
+  code: boolean;
+}
+
+/** One finished step of a job, as the run record keeps it. */
+interface Step {
+  seq: number;
+  name: string;
+  kind: "step" | "model" | "ask";
+  ms: number;
+  cost: number;
+}
+
+interface Run {
+  id: string;
+  agent: string;
+  started: string;
+  finished?: string | null;
+  source?: string;
+  job?: string | null;
+  steps?: number;
+  cost?: number;
+  error?: string | null;
+  reply?: string | null;
+  parked?: string | null;
+  trace?: Step[];
+}
+
+async function api<T>(path: string, body?: unknown): Promise<T> {
+  // The API is behind the same login as the page. This signs itself in by
+  // reading the account file, which is the same permission as running this.
+  // The channel header is what makes the log say "terminal" rather than "api".
+  const headers: Record<string, string> = { cookie: ownCookie(), "x-chloe-channel": "terminal" };
+  const response = await fetch(
+    `${BASE}${path}`,
+    body === undefined
+      ? { headers }
+      : { method: "POST", headers: { ...headers, "content-type": "application/json" }, body: JSON.stringify(body) },
+  ).catch((error: unknown) => error as Error);
+
+  if (response instanceof Error) {
+    throw new Error(`Nothing is answering on ${BASE}. Start it with: systemctl --user start chloe.service`);
+  }
+  const text = await response.text();
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    throw new Error(`The service answered ${response.status}: ${text.slice(0, 200)}`);
+  }
+  const value = parsed as { error?: string };
+  if (!response.ok) throw new Error(value.error ?? `The service answered ${response.status}.`);
+  return parsed as T;
+}
+
+const ESC = String.fromCharCode(27);
+const dim = (s: string) => (process.stdout.isTTY ? `${ESC}[2m${s}${ESC}[0m` : s);
+const bold = (s: string) => (process.stdout.isTTY ? `${ESC}[1m${s}${ESC}[0m` : s);
+
+function show(result: Result): void {
+  console.log(`\n${result.text.trim()}\n`);
+  const parts = [`${result.steps} step${result.steps === 1 ? "" : "s"}`, `$${result.cost.toFixed(4)}`];
+  const used = result.calls.map((c) => c.tool);
+  if (used.length > 0) parts.push(`used ${used.join(", ")}`);
+  console.log(dim(`(${parts.join(", ")})\n`));
+}
+
+const [name, ...rest] = process.argv.slice(2);
+
+const agents = await api<Listed[]>("/api/agents").catch((error: unknown) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+});
+if (!name) {
+  console.error(`Which agent? One of: ${agents.map((a) => a.name).join(", ")}`);
+  process.exit(2);
+}
+const agent = agents.find((a) => a.name === name);
+if (!agent) {
+  console.error(
+    `There is no agent called ${JSON.stringify(name)}. There is: ${agents.map((a) => a.name).join(", ")}`,
+  );
+  process.exit(2);
+}
+
+// A thread is what gives the conversation a memory: core/turn.ts recalls it
+// before asking and writes to it after. One per session, so a new terminal
+// starts clean.
+let thread = `terminal:${name}:${new Date().toISOString()}`;
+let last: Result | undefined;
+
+async function ask(prompt: string): Promise<void> {
+  last = await api<Result>(`/api/agents/${encodeURIComponent(name)}/chat`, { prompt, thread });
+  show(last);
+}
+
+// A job, if what was typed is the id of one. Anything else is
+// a question, because a question is the common case and a job is spelled
+// exactly.
+// The first word is the job, when it is one, and the rest is what to start it
+// with: `npm run agent chloe reading-companion "a highlight"`. A job that takes
+// nothing ignores the rest, and anything that is not a job id is a question.
+const [head = "", ...said] = rest;
+const asked = rest.join(" ").trim();
+const wanted = head.replace(/\.(ts|md)$/i, "").toLowerCase();
+const job = agent.jobs.find(
+  (one) => one.id.toLowerCase() === wanted,
+);
+
+if (!job && /\.(ts|md)$/i.test(asked)) {
+  const has = agent.jobs.map((one) => one.id).join(", ");
+  console.error(
+    `${name} has no job called ${JSON.stringify(asked)}. It has: ${has || "none"}.\n` +
+      `Leave the name off to talk to it instead.`,
+  );
+  process.exit(2);
+}
+
+if (job) {
+  console.log(
+    `${bold(`${name}/${job.id}`)}${job.description ? ` (${job.description})` : ""}, ${job.cron ? `${job.cron} ${job.timezone}` : "when started"}, ` +
+      `${job.code ? "code" : `a prompt on ${job.model}`}.`,
+  );
+  console.log(dim("Running it now. This is the real thing: it sends, writes and spends.\n"));
+
+  const firedAt = Date.now();
+  // The same envelope a channel sends, so a job written for Telegram can be
+  // tried from here without being written for here as well.
+  const text = said.join(" ").trim();
+  await api(
+    `/api/agents/${encodeURIComponent(name)}/job/${encodeURIComponent(job.id)}`,
+    text ? { text, from: "terminal", user: "terminal" } : {},
+  ).catch((error: unknown) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  });
+
+  // The row appears as the run starts and its steps are written as they
+  // finish, so following it is reading the same row again. Nothing here holds
+  // the request open: a backup takes minutes and an ask waits for a person.
+  const wait = (ms: number) => new Promise((done) => setTimeout(done, ms));
+  let runId: string | undefined;
+  for (let i = 0; i < 50 && !runId; i++) {
+    const runs = await api<Run[]>(`/api/runs?agent=${encodeURIComponent(name)}&limit=20`);
+    runId = runs.find(
+      (one) => one.job === job.id && Date.parse(one.started) >= firedAt - 2000,
+    )?.id;
+    if (!runId) await wait(200);
+  }
+  if (!runId) {
+    console.error(
+      "It was started but no run appeared. Something refused it before it began: " +
+        "systemctl --user status chloe.service",
+    );
+    process.exit(1);
+  }
+
+  let shown = 0;
+  for (;;) {
+    const run = await api<Run>(`/api/runs/${runId}`);
+    for (const step of (run.trace ?? []).slice(shown)) {
+      const price = step.cost > 0 ? `  $${step.cost.toFixed(4)}` : "";
+      const kind = step.kind === "step" ? "" : `  ${step.kind}`;
+      console.log(`  ${step.name}${dim(`  ${(step.ms / 1000).toFixed(1)}s${price}${kind}`)}`);
+    }
+    shown = (run.trace ?? []).length;
+
+    if (run.parked) {
+      const waiting = JSON.parse(run.parked) as { who: string; question: string };
+      console.log(`\n${bold("Waiting on")} ${waiting.who}: ${waiting.question}`);
+      console.log(dim(`Answer it on the page, or leave it: ${runId}\n`));
+      break;
+    }
+    if (run.finished) {
+      if (run.error) console.error(`\n${bold("Failed")}: ${run.error}\n`);
+      else console.log(`\n${(run.reply ?? "").trim()}\n`);
+      const seconds = (Date.parse(run.finished) - Date.parse(run.started)) / 1000;
+      console.log(
+        dim(`(${run.steps ?? 0} steps, ${seconds.toFixed(1)}s, $${(run.cost ?? 0).toFixed(4)}, run ${runId})\n`),
+      );
+      process.exit(run.error ? 1 : 0);
+    }
+    await wait(400);
+  }
+  process.exit(0);
+}
+
+// Piped in, or a question on the command line: one turn and out, so it can be
+// used in a script.
+const piped = rest.length === 0 && !process.stdin.isTTY;
+if (rest.length > 0 || piped) {
+  const prompt = (rest.length > 0 ? asked : readFileSync(0, "utf8")).trim();
+  if (!prompt) {
+    console.error("The question is empty.");
+    process.exit(2);
+  }
+  await ask(prompt).catch((error: unknown) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  });
+  process.exit(0);
+}
+
+console.log(
+  `${bold(agent.name)} on ${agent.model}, ${agent.tools.length} tools, ${agent.skills.length} skills, ` +
+    `${agent.jobs.length} jobs.`,
+);
+console.log(dim("/exit to leave, /new to forget this conversation, /tools for the last turn's calls.\n"));
+
+const lines = createInterface({ input: process.stdin, output: process.stdout });
+for (;;) {
+  const line = (await lines.question("> ")).trim();
+  if (!line) continue;
+
+  if (line === "/exit" || line === "/quit") break;
+  if (line === "/new") {
+    await api(`/api/threads/${encodeURIComponent(thread)}/forget`, {});
+    thread = `terminal:${name}:${new Date().toISOString()}`;
+    console.log(dim("Forgotten. Starting fresh.\n"));
+    continue;
+  }
+  if (line === "/tools") {
+    if (!last || last.calls.length === 0) console.log(dim("No tool calls in the last turn.\n"));
+    else console.log(`${JSON.stringify(last.calls, null, 2)}\n`);
+    continue;
+  }
+
+  // A failed turn does not end the session: the gateway running out of credit
+  // is the common one, and it is fixable without losing the conversation.
+  try {
+    await ask(line);
+  } catch (error) {
+    console.error(`\n${error instanceof Error ? error.message : String(error)}\n`);
+  }
+}
+lines.close();

package/ops/check.ts

@@ -0,0 +1,37 @@
+// Saying whether one thing came out right, and counting what did not.
+//
+// This is what `@chloejs/core/test` means. It is its own file rather than part of
+// test.ts because test.ts runs its cases as it loads, and a test file that
+// imported it to get these two would run the whole suite again.
+//
+// A job is code, so it is tested rather than scored. A test file sits beside
+// the job it is about, is named `<job>.test.ts`, and runs its cases as it
+// loads:
+//
+//   import { about, is } from "@chloejs/core/test";
+//
+//   about("what the nightly backup calls wrong");
+//   is("a night like the last one says nothing", whatLooksWrong(tonight, good, 10), []);
+//
+// The runner finds it, so there is nothing to add anywhere else.
+
+let failures = 0;
+
+/** The heading a group of cases runs under. */
+export function about(what: string): void {
+  console.log(`\n${what}`);
+}
+
+/** Compared as JSON, so two objects of the same shape are the same answer. */
+export function is(what: string, got: unknown, want: unknown): void {
+  const a = JSON.stringify(got);
+  const b = JSON.stringify(want);
+  if (a === b) return void console.log(`  ok   ${what}`);
+  failures++;
+  console.log(`  FAIL ${what}\n       got  ${a}\n       want ${b}`);
+}
+
+/** How many cases failed, across every file the runner loaded. */
+export function failed(): number {
+  return failures;
+}