@chloejs/core 0.2.0

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@chloejs/core",
+  "version": "0.2.0",
+  "type": "module",
+  "description": "A job runner where asking a model is one kind of step.",
+  "exports": {
+    ".": "./index.ts",
+    "./channels/*": "./channels/*.ts",
+    "./scorers": "./scorers/index.ts",
+    "./timer": "./timer/index.ts",
+    "./tools": "./model/tools/index.ts",
+    "./test": "./ops/check.ts"
+  },
+  "bin": {
+    "chloe": "./server.ts"
+  },
+  "imports": {
+    "#chloe/*": "./*"
+  },
+  "files": [
+    "channels",
+    "core",
+    "do",
+    "index.ts",
+    "load",
+    "model",
+    "ops",
+    "scorers",
+    "serve",
+    "server.ts",
+    "timer"
+  ],
+  "engines": {
+    "node": ">=22.18.0"
+  },
+  "dependencies": {
+    "zod": "^4.4.3"
+  },
+  "license": "MIT",
+  "author": "Carlos Martinez",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/carlosmartinezt/chloejs.git"
+  },
+  "homepage": "https://chloejs.org",
+  "keywords": [
+    "agents",
+    "jobs",
+    "cron",
+    "llm",
+    "typescript"
+  ],
+  "scripts": {
+    "test": "node ops/test.ts",
+    "typecheck": "tsc --noEmit -p .",
+    "check": "tsc --noEmit -p . && node ops/test.ts"
+  },
+  "devDependencies": {
+    "@types/node": "latest",
+    "typescript": "^6.0.3"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/scorers/calls.ts

@@ -0,0 +1,50 @@
+// Which tools a run had to use, and which it must never touch.
+//
+// A judge reads what an agent said. This reads what it did, and it does not
+// ask a model: emailing on a quiet morning, or restarting the same service
+// twice, are facts about the run, and a fact is worth checking exactly rather
+// than asking a second model whether it happened.
+//
+// What to expect arrives with the run, from the eval case, so this file names
+// no agent and no tool.
+import type { Result } from "#chloe/core/turn.ts";
+
+/**
+ * Which tools a run should have used, must not have used, and may use only
+ * once.
+ */
+export interface Expected {
+  mustCall?: string[];
+  mustNotCall?: string[];
+  /** Tools it may use, but only once in a run. Restarting twice is why this exists. */
+  atMostOnce?: string[];
+}
+
+/** A score between zero and one, and why it came out that way. */
+export interface Mark {
+  score: number;
+  reason: string;
+}
+
+/** Marks a run on the tools it used, which is arithmetic and asks nobody. */
+export function calls(result: Result, expected: Expected): Mark {
+  const used = result.calls.map((c) => c.tool);
+  const times = (tool: string) => used.filter((name) => name === tool).length;
+
+  const missing = (expected.mustCall ?? []).filter((tool) => !used.includes(tool));
+  const forbidden = (expected.mustNotCall ?? []).filter((tool) => used.includes(tool));
+  const repeated = (expected.atMostOnce ?? []).filter((tool) => times(tool) > 1);
+
+  const parts: string[] = [];
+  if (missing.length) parts.push(`never called ${missing.join(", ")}`);
+  if (forbidden.length) parts.push(`called ${forbidden.join(", ")}, which it must not`);
+  if (repeated.length) parts.push(`called ${repeated.join(", ")} more than once in one run`);
+
+  return {
+    score: parts.length === 0 ? 1 : 0,
+    reason:
+      parts.length === 0
+        ? `Used: ${used.join(", ") || "nothing"}.`
+        : `${parts.join("; ")}. Used: ${used.join(", ") || "nothing"}.`,
+  };
+}

package/scorers/expectations.ts

@@ -0,0 +1,118 @@
+// Judging one run against what a person said a good run looks like.
+//
+// The expectations are not in this file. They arrive with the run, from the
+// eval case, so one judge covers every agent and every job: the file that says
+// what good means is the eval file, which is JSON a person can read and change
+// without touching code.
+import { ask } from "#chloe/model/model.ts";
+import type { Result } from "#chloe/core/turn.ts";
+import type { Mark } from "./calls.ts";
+
+/**
+ * The situation a case sets up, and what the agent should and should not have
+ * done about it.
+ */
+export interface Expected {
+  situation?: string;
+  should?: string[];
+  shouldNot?: string[];
+}
+
+const JUDGE = `You grade one run of an agent against expectations written by the person who owns it.
+Grade only what is written. Do not invent standards, do not reward extra work nobody asked for, and
+do not excuse a miss because the run was otherwise good. An expectation about what the agent said is
+met only if it actually said it. An expectation about what it did is met only if the tool calls show
+it. When you are not sure, it is not met, and say what you would have needed to see.
+
+Answer with JSON and nothing else, in this shape:
+{"checks":[{"expectation":"copied back","met":true,"why":"one sentence, quoting the run where you can"}]}
+One check per expectation, in the order they are given.`;
+
+/**
+ * Marks what a run said against what it should have said, by asking a model to
+ * judge it.
+ */
+export async function expectations(
+  prompt: string,
+  result: Result,
+  expected: Expected,
+  model: string,
+): Promise<Mark> {
+  const lines = [
+    ...(expected.should ?? []).map((s) => `It should: ${s}`),
+    ...(expected.shouldNot ?? []).map((s) => `It should not: ${s}`),
+  ];
+  if (lines.length === 0) return { score: 1, reason: "Nothing was expected." };
+
+  const did = result.calls.map((c) => ({
+    tool: c.tool,
+    args: c.args,
+    // A tool result is often the whole site list or the whole disk table. The
+    // judge needs to know what came back, not to re-read all of it.
+    result: JSON.stringify(c.result ?? "").slice(0, 2000),
+  }));
+
+  const answer = await ask({
+    model,
+    messages: [
+      { role: "system", content: JUDGE },
+      {
+        role: "user",
+        content: `An agent was asked to do a job. Here is what it was told, what was true at the
+time, what it said, and what it did. Grade it against the expectations below.
+
+WHAT IT WAS ASKED
+${prompt}
+
+WHAT WAS TRUE AT THE TIME
+${expected.situation ?? "Not written down."}
+
+WHAT IT SAID
+${result.text || "(it said nothing)"}
+
+WHAT IT DID
+${JSON.stringify(did, null, 2)}
+
+EXPECTATIONS, one check each, in this order
+${lines.map((line, i) => `${i + 1}. ${line}`).join("\n")}`,
+      },
+    ],
+  });
+
+  const checks = parse(answer.text);
+  if (!checks) return { score: 0, reason: `The judge did not answer with JSON: ${answer.text.slice(0, 200)}` };
+
+  const met = checks.filter((c) => c.met).length;
+  const score = checks.length === 0 ? 1 : met / checks.length;
+  const missed = checks.filter((c) => !c.met);
+  return {
+    score,
+    reason:
+      missed.length === 0
+        ? "Met every expectation."
+        : `${score.toFixed(2)}. Missed: ${missed.map((c) => `${c.expectation} (${c.why})`).join(" ")}`,
+  };
+}
+
+interface Check {
+  expectation: string;
+  met: boolean;
+  why: string;
+}
+
+/**
+ * A model asked for JSON usually returns JSON, and sometimes returns JSON
+ * inside a code fence or with a sentence in front of it. Take the first
+ * object in the text rather than failing the case over punctuation.
+ */
+function parse(text: string): Check[] | null {
+  const start = text.indexOf("{");
+  const end = text.lastIndexOf("}");
+  if (start === -1 || end <= start) return null;
+  try {
+    const parsed = JSON.parse(text.slice(start, end + 1)) as { checks?: Check[] };
+    return Array.isArray(parsed.checks) ? parsed.checks : null;
+  } catch {
+    return null;
+  }
+}

package/scorers/index.ts

@@ -0,0 +1,5 @@
+// How a run is marked. Imported as "@chloejs/core/scorers", by whatever runs the
+// evals: the tool calls a turn made, and what somebody said should have
+// happened. Both name a type called Expected, so each keeps its own name.
+export { calls, type Expected as ExpectedCalls, type Mark } from "./calls.ts";
+export { expectations, type Expected as ExpectedOutcome } from "./expectations.ts";

package/serve/alerts.ts

@@ -0,0 +1,79 @@
+// Mail when somebody signs in from an address this copy has not seen before,
+// and when one gets locked out for guessing.
+//
+// The addresses that have been seen are kept beside the run history, so the
+// first sign-in after a fresh install is always a new one. That first mail is
+// not noise: it is the only one that proves the alert works.
+//
+// Sending is best effort and never blocks a sign-in. A person who cannot get
+// in because a mail server is down would be a worse failure than a sign-in
+// nobody was told about, and the sign-in is recorded either way.
+import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+
+import { send } from "#chloe/do/email.ts";
+import { STATE } from "#chloe/core/paths.ts";
+import { settings } from "#chloe/core/settings.ts";
+
+const FILE = `${STATE}/seen-addresses.json`;
+
+let seen: Record<string, string> | undefined;
+
+function read(): Record<string, string> {
+  if (seen) return seen;
+  try {
+    seen = JSON.parse(readFileSync(FILE, "utf8")) as Record<string, string>;
+  } catch {
+    seen = {};
+  }
+  return seen;
+}
+
+function write(all: Record<string, string>): void {
+  mkdirSync(STATE, { recursive: true });
+  writeFileSync(FILE, JSON.stringify(all, null, 2), { mode: 0o600 });
+  chmodSync(FILE, 0o600);
+  seen = all;
+}
+
+function where(): { to: string[]; from: string } | null {
+  const to = settings.alerts.email_to.split(",").map((one) => one.trim()).filter(Boolean);
+  if (!to.length || !settings.alerts.email_from) return null;
+  return { to, from: settings.alerts.email_from };
+}
+
+/** Sends, and says so in the log if it could not. Never throws at the caller. */
+function mail(subject: string, body: string): void {
+  const address = where();
+  if (!address) return;
+  void send({ from: address.from, to: address.to, tag: "chloe" }, subject, body).catch((error: unknown) => {
+    console.error("could not send the alert:", error instanceof Error ? error.message : error);
+  });
+}
+
+/** Records a successful sign-in, and mails when the address is a new one. */
+export function signedInFrom(username: string, address: string): void {
+  const all = read();
+  const first = !all[address];
+  all[address] = new Date().toISOString();
+  write(all);
+  if (!first) return;
+  mail(
+    `New sign-in from ${address}`,
+    `${username} signed in from ${address}, which this copy had not seen before.\n\n` +
+      `If that was not you, change the password: delete data/login.json on the box and run \`npm run account\`.\n` +
+      `Revoke every token at the same time, from the site's tokens page.\n`,
+  );
+}
+
+/** Mails when one address has been locked out for guessing. */
+export function lockedOut(address: string): void {
+  mail(
+    `Locked out ${address}`,
+    `Too many wrong passwords from ${address}, so it is locked out for fifteen minutes.\n`,
+  );
+}
+
+/** Forgets what was read, so a test can write the file and be believed. */
+export function forgetAddresses(): void {
+  seen = undefined;
+}

package/serve/errors.ts

@@ -0,0 +1,10 @@
+// The two refusals every route can make, in a file of their own so that the
+// files doing the work can throw them without importing the router that
+// catches them. http.ts re-exports both, because that is where a route author
+// is already looking.
+
+/** Nothing at that address, or nothing this caller is allowed to know is there. */
+export class NotFound extends Error {}
+
+/** The caller sent something this route cannot take. Answered 400 with the reason. */
+export class BadRequest extends Error {}

package/serve/files.ts

@@ -0,0 +1,70 @@
+// An agent's own folder, read from the page and written back to.
+//
+// Markdown is writable and nothing else is. Words are what the page is for,
+// and the watcher makes a saved skill live in under a second. A .ts saved from
+// a browser would be live just as fast with nothing type checking it, so code
+// is edited where `npm run check` runs.
+//
+// The edge of the folder is confine()'s job, inside the calls below: a path
+// from the page is as untrusted as a path from a model.
+import { existsSync, statSync } from "node:fs";
+
+import { confine } from "#chloe/core/confine.ts";
+import { agentDir } from "#chloe/core/paths.ts";
+import { list, read, write } from "#chloe/do/files.ts";
+
+export interface Entry {
+  name: string;
+  path: string;
+  dir: boolean;
+  children?: Entry[];
+}
+
+/** Deep enough for any agent folder, and an end to it if something links in a circle. */
+const DEPTH = 6;
+
+/** Made by a program, not by anybody, so it is not part of what an agent is. */
+const JUNK = ["__pycache__", "node_modules"];
+
+/** Everything in one agent's folder, folders first, as a tree. */
+export async function tree(agent: string, path = "", depth = 0): Promise<Entry[]> {
+  const { entries } = await list(agentDir(agent), path || undefined);
+  const out: Entry[] = [];
+  for (const entry of entries) {
+    const dir = entry.endsWith("/");
+    const name = dir ? entry.slice(0, -1) : entry;
+    if (JUNK.includes(name)) continue;
+    const at = path ? `${path}/${name}` : name;
+    out.push({
+      name,
+      path: at,
+      dir,
+      children: dir && depth < DEPTH ? await tree(agent, at, depth + 1) : undefined,
+    });
+  }
+  return out.sort((a, b) => Number(b.dir) - Number(a.dir) || a.name.localeCompare(b.name));
+}
+
+/** What the page may write back. */
+export const editable = (path: string): boolean => path.endsWith(".md");
+
+/**
+ * One thing in the folder, whichever kind it is. Every path in the tree is an
+ * address, so a folder answers with what is in it rather than with an error.
+ * Nothing there is `null`, which the route turns into a 404.
+ */
+export async function open(agent: string, path: string) {
+  const resolved = confine(agentDir(agent), path);
+  if (!existsSync(resolved)) return null;
+  if (statSync(resolved).isDirectory()) {
+    return { path, dir: true as const, entries: await tree(agent, path) };
+  }
+  const file = await read(agentDir(agent), path);
+  return { path, dir: false as const, content: file.content, editable: editable(path) };
+}
+
+export async function save(agent: string, path: string, content: string) {
+  if (!editable(path)) throw new Error(`${path} is not markdown.`);
+  const written = await write(agentDir(agent), path, content);
+  return { path, bytes: written.bytes };
+}