@chloejs/core 0.2.0

package/do/scripts.ts

@@ -0,0 +1,49 @@
+// Running one of an agent's own scripts.
+//
+// The boundary: an agent can run anything in its own scripts/ folder and
+// nothing anywhere else. The name is checked against what is on disk, so no
+// input can be a path, and arguments are passed one at a time, never as a
+// shell string.
+//
+// The tool a model reaches is model/tools/run_script.ts. A job calls
+// these from a step.
+import { readdir } from "node:fs/promises";
+
+import { agentDir } from "#chloe/core/paths.ts";
+import { settings } from "#chloe/core/settings.ts";
+import { run, type Result } from "./run.ts";
+
+/** What this agent has in scripts/, sorted. Nothing hidden. */
+export async function scripts(agent: string): Promise<string[]> {
+  const dir = `${agentDir(agent)}/scripts`;
+  const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
+  return entries
+    .filter((e) => e.isFile() && !e.name.startsWith("."))
+    .map((e) => e.name)
+    .sort();
+}
+
+/**
+ * Run one. A name not on disk is refused rather than resolved as a path, which
+ * is what stops a `../` in a name reaching anything else on the box.
+ *
+ * `cwd` defaults to the scripts folder, so a script may use relative paths.
+ * An agent whose scripts work on a tree somewhere else passes that instead.
+ */
+export async function script(
+  agent: string,
+  name: string,
+  args: string[] = [],
+  { timeoutMs = 300_000, cwd }: { timeoutMs?: number; cwd?: string } = {},
+): Promise<Result & { script: string; args: string[] }> {
+  const available = await scripts(agent);
+  if (!available.includes(name)) {
+    throw new Error(`No script called ${JSON.stringify(name)}. You have: ${available.join(", ")}`);
+  }
+  const dir = `${agentDir(agent)}/scripts`;
+  // A script is someone else's program, so what it needs from settings reaches
+  // it the way a program expects, as an environment variable.
+  const env: Record<string, string> = settings.google.GA_KEY_FILE ? { GA_KEY_FILE: settings.google.GA_KEY_FILE } : {};
+  const result = await run(`${dir}/${name}`, args, { timeoutMs, cwd: cwd ?? dir, env });
+  return { script: name, args, ...result };
+}

package/do/web.ts

@@ -0,0 +1,192 @@
+// Reading one web page, as text.
+//
+// Only public addresses: this box serves its own sites and the runtime on
+// loopback ports, and a page that redirects to one of them must not be read
+// back to whoever asked. Every redirect is checked the same way as the first
+// address.
+//
+// The tool a model reaches is model/tools/web.ts, which calls this.
+import { lookup } from "node:dns";
+import { request as http } from "node:http";
+import { request as https } from "node:https";
+import { isIP, type LookupFunction } from "node:net";
+
+/** One fetched web page as plain text, in slices when it is longer than one. */
+export interface Page {
+  url: string;
+  status: number;
+  title?: string;
+  /** The page as plain text, links written as `[text](url)`. */
+  text: string;
+  /** Where the next slice starts, when the page was longer than one. */
+  next?: number;
+}
+
+/** Enough for a results table, short enough not to fill a turn. */
+const SLICE = 20_000;
+const HOPS = 5;
+const LARGEST = 5_000_000;
+
+function privateV4([a, b]: number[]): boolean {
+  return a === 0 || a === 10 || a === 127 || a >= 224 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) ||
+    (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127);
+}
+
+/** An IPv6 address as its eight 16-bit groups, however it was written. */
+function groups(address: string): number[] | null {
+  let text = address.toLowerCase().replace(/%.*$/, "");
+  const v4 = /(\d+)\.(\d+)\.(\d+)\.(\d+)$/.exec(text);
+  if (v4) {
+    const [a, b, c, d] = v4.slice(1).map(Number);
+    text = text.slice(0, v4.index) + `${((a << 8) | b).toString(16)}:${((c << 8) | d).toString(16)}`;
+  }
+  const [head, tail] = text.split("::");
+  const left = head ? head.split(":") : [];
+  const right = tail ? tail.split(":") : [];
+  const fill = text.includes("::") ? 8 - left.length - right.length : 0;
+  const all = [...left, ...Array(fill).fill("0"), ...right].map((one) => parseInt(one, 16));
+  return all.length === 8 && all.every((one) => one >= 0 && one <= 0xffff) ? all : null;
+}
+
+/**
+ * Loopback, private ranges, link local, multicast, and the same in IPv6,
+ * including an IPv4 address carried inside an IPv6 one in any spelling.
+ */
+export function isPrivate(address: string): boolean {
+  if (isIP(address) === 4) return privateV4(address.split(".").map(Number));
+  const g = groups(address);
+  if (!g) return true;
+  const inner = [g[6] >> 8, g[6] & 0xff];
+  // ::/96 and ::ffff:0:0/96 carry an IPv4 address, and 64:ff9b::/96 translates to one.
+  if (g.slice(0, 5).every((one) => one === 0) && (g[5] === 0 || g[5] === 0xffff)) {
+    return (g[5] === 0 && g[6] === 0) || privateV4(inner);
+  }
+  if (g[0] === 0x64 && g[1] === 0xff9b && g.slice(2, 6).every((one) => one === 0)) return privateV4(inner);
+  return (g[0] & 0xfe00) === 0xfc00 || (g[0] & 0xffc0) === 0xfe80 || (g[0] & 0xff00) === 0xff00 || g[0] === 0x2001 && g[1] === 0xdb8;
+}
+
+/**
+ * The name is resolved once, here, and the socket connects to what was
+ * checked, so a name cannot answer public for the check and private for the
+ * connection.
+ */
+const publicOnly: LookupFunction = (host, options, done) => {
+  lookup(host, { ...options, all: true }, (error, found) => {
+    if (error) return done(error, "", 0);
+    const bad = found.find((one) => isPrivate(one.address));
+    if (bad) return done(new Error(`${host} is a private address, and those are not read.`), "", 0);
+    if (options.all) return (done as any)(null, found);
+    done(null, found[0].address, found[0].family);
+  });
+};
+
+interface Reply {
+  status: number;
+  location?: string;
+  type: string;
+  body: string;
+}
+
+function get(url: URL): Promise<Reply> {
+  if (url.protocol !== "https:" && url.protocol !== "http:") throw new Error(`Only http and https pages, not ${url.protocol}`);
+  // A literal address is connected to without a lookup, so it is checked here.
+  const literal = url.hostname.replace(/^\[|\]$/g, "");
+  if (isIP(literal) && isPrivate(literal)) throw new Error(`${url.hostname} is a private address, and those are not read.`);
+  return new Promise((resolve, reject) => {
+    const request = (url.protocol === "https:" ? https : http)(
+      url,
+      {
+        lookup: publicOnly,
+        headers: {
+          "User-Agent": "Mozilla/5.0 (compatible; chloe)",
+          Accept: "text/html,text/plain,application/json,*/*",
+          "Accept-Encoding": "identity",
+        },
+        timeout: 30_000,
+      },
+      (response) => {
+        const type = String(response.headers["content-type"] ?? "");
+        const chunks: Buffer[] = [];
+        let size = 0;
+        response.on("data", (chunk: Buffer) => {
+          size += chunk.length;
+          if (size > LARGEST) return request.destroy(new Error(`${url.href} is over ${LARGEST / 1_000_000} MB.`));
+          chunks.push(chunk);
+        });
+        response.on("end", () => {
+          const charset = /charset=([\w-]+)/i.exec(type)?.[1] ?? "utf-8";
+          let decoder: TextDecoder;
+          try {
+            decoder = new TextDecoder(charset);
+          } catch {
+            decoder = new TextDecoder();
+          }
+          resolve({ status: response.statusCode ?? 0, location: response.headers.location, type, body: decoder.decode(Buffer.concat(chunks)) });
+        });
+        response.on("error", reject);
+      },
+    );
+    request.on("timeout", () => request.destroy(new Error(`${url.href} did not answer in 30 seconds.`)));
+    request.on("error", reject);
+    request.end();
+  });
+}
+
+const ENTITIES: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", mdash: "-", ndash: "-" };
+
+function decode(text: string): string {
+  // Some sites write &nbsp with no semicolon, and browsers forgive it.
+  return text.replace(/&nbsp(?!;)/gi, " ").replace(/&(#x?[0-9a-f]+|[a-z]+);/gi, (whole, name: string) => {
+    if (name[0] === "#") {
+      const code = name[1].toLowerCase() === "x" ? parseInt(name.slice(2), 16) : Number(name.slice(1));
+      return Number.isFinite(code) ? String.fromCodePoint(code) : whole;
+    }
+    return ENTITIES[name.toLowerCase()] ?? whole;
+  });
+}
+
+/** HTML to readable text: blocks become lines, cells are split by " | ", links keep their address. */
+export function htmlToText(html: string, base: string): { title?: string; text: string } {
+  const title = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html)?.[1];
+  const text = html
+    .replace(/\s+/g, " ")
+    .replace(/<(script|style|noscript|svg|head)\b[\s\S]*?<\/\1>/gi, "")
+    .replace(/<!--[\s\S]*?-->|<![^>]*>/g, "")
+    .replace(/<a\b[^>]*?href\s*=\s*(["'])(.*?)\1[^>]*>([\s\S]*?)<\/a>/gi, (_whole, _quote, href: string, inner: string) => {
+      const words = inner.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+      let address = decode(href);
+      try {
+        address = new URL(address, base).href;
+      } catch {}
+      if (!words) return "";
+      return /^(javascript|mailto):|^#/.test(href) ? words : `[${words}](${address.replace(/ /g, "%20")})`;
+    })
+    .replace(/<\/(td|th)>/gi, " | ")
+    .replace(/<br\s*\/?>|<\/(p|div|tr|li|h[1-6]|table|section|article|header|footer)>/gi, "\n")
+    .replace(/<\/?[a-z][^>]*>/gi, " ");
+  return {
+    title: title ? decode(title).replace(/\s+/g, " ").trim() : undefined,
+    text: decode(text)
+      .split("\n")
+      .map((line) => line.replace(/[ \t]+/g, " ").replace(/(\s*\|\s*)+$/, "").trim())
+      .filter(Boolean)
+      .join("\n"),
+  };
+}
+
+/** Fetches `address` and returns it as text, a slice at a time starting at `from`. */
+export async function readPage(address: string, from = 0): Promise<Page> {
+  let url = new URL(address);
+  for (let hop = 0; ; hop++) {
+    const reply = await get(url);
+    if (reply.status >= 300 && reply.status < 400 && reply.location) {
+      if (hop >= HOPS) throw new Error(`More than ${HOPS} redirects from ${address}.`);
+      url = new URL(reply.location, url);
+      continue;
+    }
+    if (!/text|json|xml/.test(reply.type)) throw new Error(`${url.href} is ${reply.type || "not text"}, which this cannot read.`);
+    const { title, text } = /html/.test(reply.type) ? htmlToText(reply.body, url.href) : { title: undefined, text: reply.body };
+    const end = from + SLICE;
+    return { url: url.href, status: reply.status, title, text: text.slice(from, end), next: end < text.length ? end : undefined };
+  }
+}

package/index.ts

@@ -0,0 +1,52 @@
+// What chloe is, to the repo that installs it.
+//
+// Everything an agent, a job, a tool or a channel is written with is named
+// here, and nothing else in this folder is anybody's business. Import it as
+// "@chloejs/core": the three lines below say the rest.
+//
+//   import { defineJob, tool, note } from "@chloejs/core";
+//   import { telegramChannel } from "@chloejs/core/channels/telegram";  // reaching an agent
+//   import { calls, expectations } from "@chloejs/core/scorers";  // marking a run
+//
+// Adding a name here is publishing it, and taking one away is a break, so this
+// file is the one place to look when you want to know what may move freely.
+// The server itself is `server.ts`, and it is run rather than imported.
+
+// An agent, and the jobs it runs.
+export { defineAgent, defineConfig, markdownJob, load, loadAll, names, type Agent, type Channel, type ChannelRoute, type Running, type Job, type Skill, type Binding, type Home, type Features, type ChatHistory } from "./load/load.ts";
+export { defineJob } from "./load/job.ts";
+export { prompt, isPrompt, oneLineSummary, type Prompt } from "./core/markdown.ts";
+
+// A job: code first, with a model where a step needs judgement and an agent
+// where the order of the work cannot be known in advance.
+export { work, resume, answer, sweep, parkedRuns, waitingFor, waitingOn, checkInput, WrongInput, type AgentStep, type AskStep, type Line, type ModelStep, type ParkedRun, type Result as RunResult, type Work } from "./core/steps.ts";
+
+// A prompt: ask a model, run the tools it asked for, ask again.
+export { turn, type Result as TurnResult } from "./core/turn.ts";
+
+// What a model can be asked to do, and how it is asked.
+export { tool, type Approve, type Call, type Tool, type Tools } from "./model/tool.ts";
+export { ask, via, type Attachment, type Message } from "./model/model.ts";
+
+// Reaching a person, and being reached back.
+export { canReach, deliver, owner, reachBy, split, type Send } from "./model/ask.ts";
+
+// The floor: where things are, what the box was told, staying inside a
+// folder, a small file an agent keeps, the history.
+export { agentDir, ROOT, STATE } from "./core/paths.ts";
+export { readSettings, setting, settings, type Settings } from "./core/settings.ts";
+export { confine } from "./core/confine.ts";
+export { note, type Note } from "./core/notes.ts";
+export { copyDatabase, DATABASE, db, trim } from "./core/db.ts";
+
+// What a job can do without asking anybody: running a command, sending mail,
+// reading mail, reading and writing files in one folder, running one of an
+// agent's own scripts, reading a web page. The same work offered
+// to a model instead is "@chloejs/core/tools", and each of those is a wrapper over one
+// of these.
+export { run, type Result } from "./do/run.ts";
+export { send, type Address } from "./do/email.ts";
+export { messages, oneMessage, type Message as Mail } from "./do/mail.ts";
+export { list, read, search, write } from "./do/files.ts";
+export { script, scripts } from "./do/scripts.ts";
+export { readPage, htmlToText, isPrivate, type Page } from "./do/web.ts";

package/load/job.ts

@@ -0,0 +1,84 @@
+// A job written as code, for when markdown frontmatter is not enough.
+//
+// It takes one of two things, and this is the line the whole repo turns on:
+//
+//   markdown:  the job is a prompt. A model reads the situation and decides.
+//   run:       the job is code. Nothing asks a model unless the code does.
+import type { z } from "zod";
+
+import type { Prompt } from "#chloe/core/markdown.ts";
+import type { Work } from "#chloe/core/steps.ts";
+
+export interface Definition<
+  State extends z.ZodType = z.ZodType<Record<string, unknown>>,
+  Result = unknown,
+  Input extends z.ZodType = z.ZodType<Record<string, unknown>>,
+> {
+  /**
+   * What the run history files it under, and what `npm run agent` and the
+   * evals call it. Name the file after it: jobs/<id>.ts.
+   */
+  id: string;
+  /**
+   * When it runs by itself. Five fields: minute, hour, day of month, month,
+   * day of week. Without one it runs only when somebody starts it.
+   */
+  cron?: string;
+  /** The prompt: a string for a one-liner, or `prompt("./name.md")`. */
+  markdown?: string | Prompt;
+  /**
+   * The job, when it is code. Branch with `if`, loop with `for`, and put every
+   * piece of work inside a `step`: a step is written down so it never runs
+   * twice, and a line outside one runs again every time the job resumes.
+   */
+  run?: (work: Work<z.infer<State>, z.infer<Input>>) => Promise<Result>;
+  /**
+   * What this job is started with, when it is started by hand rather than by
+   * its cron line: a channel command, the API, or `npm run agent`. The shape is
+   * the contract, and a caller that does not fit it is refused before the run
+   * begins rather than halfway through it.
+   *
+   * A channel sends a fixed envelope, so a job meant to be reachable from one
+   * takes `text` and whichever of `from`, `chat`, `user`, `thread` and
+   * `replyTo` it cares about. See https://chloejs.org/docs/jobs.
+   *
+   * A job with a cron line and a required field cannot run on that line, so
+   * give those fields a default.
+   */
+  input?: Input;
+  /**
+   * Plain messages, with no command, that this job answers instead of the
+   * agent's chat: `answers: (text) => text.includes("https://a.co/")`. A
+   * channel that sees one starts the job with the whole message as `text`.
+   * Code, not a model: it is asked of every message, so it has to be quick and
+   * certain. The first job that says yes gets the message.
+   */
+  answers?: (text: string) => boolean;
+  /**
+   * What a finished run did, in one line, from what `run` returned: "15 sites,
+   * all up". It is what the overview shows. A job without one shows nothing
+   * there, unless it returned a string.
+   */
+  summary?: (result: Result) => string;
+  /**
+   * What a chat is sent when this job was started from one, from what `run`
+   * returned. Whole, not cut to one line. The summary when unsaid.
+   */
+  reply?: (result: Result) => string;
+  /** The shared store every step can read and write. It survives a pause. */
+  state?: State;
+  timezone?: string;
+  /** When this job should not run on the agent's own model. */
+  model?: string;
+  /** One line on what it does, shown beside the id. */
+  description?: string;
+}
+
+/** Only here so a job file is type checked as it is written. */
+export function defineJob<
+  State extends z.ZodType = z.ZodType<Record<string, unknown>>,
+  Result = unknown,
+  Input extends z.ZodType = z.ZodType<Record<string, unknown>>,
+>(definition: Definition<State, Result, Input>): Definition<State, Result, Input> {
+  return definition;
+}