@flow-engineer/flow 0.0.0-stage → 0.2.0

package/dist/testing/src/check-endpoint.js

@@ -0,0 +1,158 @@
+// `flow check-endpoint <url>`: checks an agent's HTTP endpoint against the contract in spec/
+// (agent-endpoint.openapi.yaml, request.schema.json, response.schema.json) before any test run. It
+// POSTs a fixed set of synthetic messages, as `flow test --agent-url` would, and checks each
+// answer: the status and body (the same rules a run applies, testing/src/http-agent.ts), the
+// response schema, what WhatsApp can show, and that every id a reply names is one of the person's
+// messages. No cloud, no key and no model call on Flow Framework's side: whatever the agent does with the
+// messages (its own model, its tools) runs on its side.
+import { randomBytes } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+import { WHATSAPP, z } from "../../src/index.js";
+import { setting } from "../../src/traces.js";
+import { checkReply, postMessage, TurnError } from "./http-agent.js";
+import { packageRoot } from "./new.js";
+/** A 1×1 PNG. */
+const PHOTO = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==";
+export const CASES = [
+    { id: "message", title: "a typed message", messages: [{ messageId: "m1", message: { text: "hi, what time are you open today?" } }] },
+    {
+        id: "double-text",
+        title: "two messages in a row, in one conversation",
+        messages: [
+            { messageId: "m1", message: { text: "can i get a large latte" } },
+            { messageId: "m2", message: { text: "oat milk actually" } },
+        ],
+    },
+    { id: "file", title: "a photo with no caption", messages: [{ messageId: "m1", message: { text: "", attachments: [{ kind: "image", mime: "image/png", name: "photo.png", data: PHOTO }] } }] },
+    {
+        id: "message-ids",
+        title: "a tapped button quoting the agent, with an id to react to",
+        messages: [
+            { messageId: "m1", message: { text: "do you have oat milk?" } },
+            { messageId: "m2", message: { text: "Oat", choice: { title: "Oat" }, replyTo: { text: "Which milk would you like?", fromAgent: true } } },
+        ],
+    },
+    { id: "no-reply", title: "a message that may need no answer", messages: [{ messageId: "m1", message: { text: "ok 👍" } }] },
+];
+const schemaFile = (name) => JSON.parse(readFileSync(join(packageRoot(), "spec", name), "utf8"));
+function schemas() {
+    const response = schemaFile("response.schema.json");
+    const shape = z.fromJSONSchema({ ...response, properties: { replies: { type: "array" } } });
+    // One schema per reply type, so a bad reply is reported against its own type.
+    const replies = new Map(Object.keys(response.$defs).map((type) => [type, z.fromJSONSchema({ $schema: response.$schema, $defs: response.$defs, $ref: `#/$defs/${type}` })]));
+    return { shape, replies };
+}
+const issues = (error, at) => error.issues.map((i) => `${[at, ...i.path].join(".")}: ${i.message}`).join("; ");
+/** Why this answer breaks the contract, or null. `ids` are the person's messages so far. */
+function problem(replies, ids, s) {
+    for (const [n, reply] of replies.entries()) {
+        const at = `replies[${n}]`;
+        const type = reply?.type;
+        const schema = typeof type === "string" ? s.replies.get(type) : undefined;
+        if (!schema)
+            return `${at}.type: must be one of ${[...s.replies.keys()].join(", ")}`;
+        const valid = schema.safeParse(reply);
+        if (!valid.success)
+            return issues(valid.error, at);
+        const shown = checkReply(reply, WHATSAPP);
+        if ("refused" in shown)
+            return `${at}: would not be sent on WhatsApp: ${shown.refused}`;
+        const r = reply;
+        for (const id of [r.replyTo, r.messageId, ...(r.messageIds ?? [])]) {
+            if (id !== undefined && !ids.includes(id))
+                return `${at} names ${JSON.stringify(id)}, which is not one of the person's messages (${ids.join(", ")})`;
+        }
+    }
+    return null;
+}
+const described = (answers) => answers
+    .map((a) => {
+    const types = a.replies.map((r) => String(r.type));
+    return `${a.messageId}: ${types.length ? `${types.length} ${types.length === 1 ? "reply" : "replies"} (${types.join(", ")})` : "no reply"}`;
+})
+    .join("; ");
+export async function checkEndpoint(options) {
+    const s = schemas();
+    const timeoutMs = options.timeoutMs ?? 60_000;
+    // Fresh conversations on every check, in the cloud's form, so an endpoint that keeps history never mixes runs.
+    const session = `check_${randomBytes(4).toString("hex")}`;
+    const results = [];
+    for (const c of CASES) {
+        const conversationId = `whatsapp:${session}.${c.id}.whatsapp.1`;
+        const answers = [];
+        let failed = null;
+        for (const [n, { messageId, message }] of c.messages.entries()) {
+            let replies;
+            try {
+                replies = await postMessage(options, timeoutMs, { conversationId, messageId, channel: "whatsapp", message });
+            }
+            catch (error) {
+                failed = `${messageId}: ${error instanceof TurnError ? error.message : error.message}`;
+                break;
+            }
+            answers.push({ messageId, replies });
+            const wrong = problem(replies, c.messages.slice(0, n + 1).map((m) => m.messageId), s);
+            if (wrong) {
+                failed = `${messageId}: ${wrong}`;
+                break;
+            }
+        }
+        results.push({ id: c.id, title: c.title, pass: failed === null, detail: failed ?? described(answers), answers });
+    }
+    return results;
+}
+const USAGE = `Usage: flow check-endpoint <url> [--token-env NAME] [--agent-timeout <s>] [--json]
+
+Checks an agent's HTTP endpoint against Flow Framework's contract (spec/agent-endpoint.openapi.yaml) with
+a fixed set of synthetic messages. No cloud, no key, no model call on Flow Framework's side.
+
+  --token-env NAME     Send the value of NAME as authorization: Bearer <value>
+                       (default: FLOW_FRAMEWORK_AGENT_TOKEN, when it is set)
+  --agent-timeout <s>  How long one message waits for its answer (default: 60)
+  --json               Print the results as JSON
+
+Exit status: 0 when every case passed, 1 otherwise.`;
+/** The command: prints one line per case (or JSON) and returns the exit status. */
+export async function checkEndpointCli(args, env = process.env, print = console.log, printError = console.error) {
+    let parsed;
+    try {
+        parsed = parseArgs({ args, allowPositionals: true, options: { "token-env": { type: "string" }, "agent-timeout": { type: "string", default: "60" }, json: { type: "boolean", default: false }, help: { type: "boolean", short: "h", default: false } } });
+    }
+    catch (error) {
+        printError(`${error.message}\n\n${USAGE}`);
+        return 1;
+    }
+    const { values, positionals } = parsed;
+    if (values.help) {
+        print(USAGE);
+        return 0;
+    }
+    const url = positionals.length === 1 ? positionals[0] : "";
+    if (!/^https?:\/\/[^/]/.test(url) || !URL.canParse(url)) {
+        printError(`check-endpoint takes one http:// or https:// URL, such as http://localhost:3000/flow.\n\n${USAGE}`);
+        return 1;
+    }
+    const timeout = Number(values["agent-timeout"]);
+    if (!(timeout > 0)) {
+        printError("--agent-timeout must be a number of seconds above 0.");
+        return 1;
+    }
+    const tokenEnv = values["token-env"];
+    const token = (tokenEnv ? env[tokenEnv] : setting("AGENT_TOKEN", env)) || undefined;
+    if (tokenEnv && !token) {
+        printError(`${tokenEnv} is not set.`);
+        return 1;
+    }
+    const results = await checkEndpoint({ url, token, timeoutMs: timeout * 1000 });
+    const passed = results.filter((r) => r.pass).length;
+    if (values.json)
+        print(JSON.stringify({ url, passed: passed === results.length, cases: results }, null, 2));
+    else {
+        for (const r of results)
+            print(`${r.pass ? "PASS" : "FAIL"}  ${r.id} (${r.title}): ${r.detail}`);
+        print(`\n${passed} of ${results.length} cases passed.`);
+    }
+    return passed === results.length ? 0 : 1;
+}

package/dist/testing/src/chinwag.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/testing/src/chinwag.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+// The `chinwag` command from before the rename, kept for one release: it says so on stderr (stdout
+// stays the report's, for --json), then runs the flow command (testing/src/cli.ts) as it is.
+console.error("chinwag is now Flow Framework (@flow-engineer/flow): run npx flow, and npm i @flow-engineer/flow.");
+await import("./cli.js");
+export {};

package/dist/testing/src/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};