@flow-engineer/flow 0.2.0 → 0.2.1

package/dist/flow-messaging/src/index.js

@@ -1,18 +1,24 @@
 // Flow Framework channels on Flow Messaging (api.flow.engineer), Flow's unified messaging API: WhatsApp,
-// Telegram and US SMS from senders Flow hosts, and a webhook that turns Flow's signed events into
-// messages for the agent.
+// Telegram, iMessage and US SMS from senders Flow hosts. Events reach the agent one of two ways: the live
+// stream (an outbound WebSocket, so a laptop needs no public URL or tunnel) or a webhook that checks
+// Flow's signature. Both hand each event to the same handler.
 //
-//   const fm = flowMessaging();                    // FLOW_MESSAGING_KEY, FLOW_MESSAGING_WEBHOOK_SECRET
-//   const agent = createAgent({ ..., channels: [fm.whatsapp(), fm.telegram(), fm.sms()] });
-//   http.createServer(fm.webhook(agent)).listen(8080);
+//   const fm = flowMessaging();                    // FLOW_MESSAGING_KEY (and FLOW_MESSAGING_WEBHOOK_SECRET for a webhook)
+//   const agent = createAgent({ ..., channels: [fm.whatsapp(), fm.telegram(), fm.imessage(), fm.sms()] });
+//   fm.start(agent);                               // the stream, or the webhook on PORT when a webhook secret is set
 //
 // Addresses are what Flow Framework joins people by: the phone number on WhatsApp and SMS, the numeric
-// user id on Telegram. A reply goes into the Flow conversation the person last wrote in; with none
-// known yet (a nudge, or after a restart) it starts one from the channel's sender, which Flow puts
-// into the open conversation if there is one. What this process learns (conversations, contacts,
+// user id on Telegram, the phone number on iMessage, else the email address that is the person's
+// handle there. A reply goes into the Flow conversation the person last wrote in; with none known
+// yet (a nudge, or after a restart) it starts one from the channel's sender, which Flow puts into the
+// open conversation if there is one. On Flow's shared iMessage sender, as on WhatsApp after 24
+// hours, the agent can only answer someone who wrote first: a start to anyone else fails. What this process learns (conversations, contacts,
 // event ids) is kept in memory only.
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { createServer } from "node:http";
+import { dirname } from "node:path";
 import { sms as smsChannel, whatsapp as whatsappChannel } from "../../src/index.js";
-import { Client, FlowMessagingError, verifySignature } from "./client.js";
+import { Client, FlowMessagingError, idSegment, verifySignature } from "./client.js";
 export { API_VERSION, FlowMessagingError, verifySignature } from "./client.js";
 /** Telegram as Flow sends it: an inline keyboard of up to 10 buttons (or one link), photos, Markdown, no read receipts for bots. */
 export const TELEGRAM = {
@@ -26,11 +32,34 @@ export const TELEGRAM = {
     images: { captionMax: 1024 },
     readReceipts: false,
 };
+/**
+ * iMessage as Flow sends it: plain text up to 9999 characters, photos, inline replies, tapbacks
+ * (other emoji go as emoji reactions), typing and read receipts (the whole conversation). No buttons,
+ * lists or templates. No reply window is set: the line only answers people who wrote to it, and a
+ * send it will not deliver comes back as Not delivered (outside_window).
+ */
+export const IMESSAGE = {
+    reactions: true,
+    status: false,
+    formatting: "plain",
+    maxLength: 9999,
+    replies: true,
+    images: { captionMax: 1024 },
+    readReceipts: true,
+};
 /** Flow sends a list as buttons, whose labels take at most 20 characters, and has no row descriptions. */
 const LIST_TITLE_MAX = 20;
 const MAX_BODY_BYTES = 1_000_000;
 /** Event ids remembered to drop Flow's redeliveries (deliveries are at least once). */
 const SEEN_MAX = 10_000;
+/** The events the handler acts on; the stream asks for only these. */
+const HANDLED = ["message.received", "message.failed", "reaction.added", "contact.deleted"];
+/** The longest wait between two tries to reconnect the stream (ms). */
+const STREAM_BACKOFF_MAX = 30_000;
+/** Tries of one event before the stream gives up on it and goes on to the next. */
+const STREAM_EVENT_TRIES = 5;
+/** Close codes after which reconnecting unchanged fails again: authentication, permission, a bad request. */
+const STREAM_FINAL = new Set([4401, 4403, 4400]);
 export function flowMessaging(options = {}) {
     const env = (...names) => names.map((n) => process.env[n]).find(Boolean);
     const apiKey = options.apiKey ?? env("FLOW_MESSAGING_KEY", "FLOW_MESSAGING_API_KEY") ?? "";
@@ -54,7 +83,7 @@ export function flowMessaging(options = {}) {
         const known = flowConversation.get(key(channel, to));
         if (known || !messageId)
             return known;
-        const found = (await client.get(`/v1/messages/${messageId}`)).conversation;
+        const found = (await client.get(`/v1/messages/${idSegment("msg", messageId)}`)).conversation;
         flowConversation.set(key(channel, to), found);
         return found;
     }
@@ -63,6 +92,8 @@ export function flowMessaging(options = {}) {
             return { contact: to };
         if (kind === "telegram")
             return { telegram_user_id: to };
+        if (kind === "imessage" && !/^\+?[0-9]{7,15}$/.test(to))
+            return { handle: to }; // an email address
         return { phone: to.startsWith("+") ? to : `+${to}` };
     }
     /** One message: into the person's conversation, or a new one from the channel's sender. */
@@ -71,7 +102,7 @@ export function flowMessaging(options = {}) {
             const conversation = await conversationFor(route.name, to, about);
             let sent;
             if (conversation)
-                sent = await client.post(`/v1/conversations/${conversation}/messages`, { content, ...(replyTo ? { reply_to: replyTo } : {}) });
+                sent = await client.post(`/v1/conversations/${idSegment("conv", conversation)}/messages`, { content, ...(replyTo ? { reply_to: replyTo } : {}) });
             else {
                 if (!route.sender)
                     throw new Error(`no conversation with them on ${route.name} yet, and no sender to start one (set FLOW_MESSAGING_${route.kind.toUpperCase()}_SENDER)`);
@@ -88,7 +119,7 @@ export function flowMessaging(options = {}) {
         try {
             const conversation = await conversationFor(route.name, to, messageId);
             if (conversation)
-                await client.post(`/v1/conversations/${conversation}/${action}`, body, 1);
+                await client.post(`/v1/conversations/${idSegment("conv", conversation)}/${action}`, body, 1);
         }
         catch (error) {
             console.warn(`@flow-engineer/flow/flow-messaging: ${action} on ${route.name} failed (${error instanceof FlowMessagingError ? error.type : "error"})`);
@@ -122,7 +153,7 @@ export function flowMessaging(options = {}) {
             const all = [];
             let after = "";
             for (;;) {
-                const page = await client.get(`/v1/templates?sender=${sender}&status=approved&limit=100${after ? `&after=${after}` : ""}`);
+                const page = await client.get(`/v1/templates?${new URLSearchParams({ sender: idSegment("snd", sender), status: "approved", limit: "100", ...(after ? { after } : {}) })}`);
                 all.push(...page.data);
                 if (!page.has_more || !page.data.length)
                     break;
@@ -168,12 +199,23 @@ export function flowMessaging(options = {}) {
     }
     /** A Telegram bot connected to Flow. Flow Framework has no Telegram kind yet, so it is a custom channel. */
     function telegram(o = {}) {
-        const route = register("telegram", o);
+        return custom(register("telegram", o), TELEGRAM, o);
+    }
+    /**
+     * An iMessage line connected to Flow (Flow's shared one by default). Flow Framework has no iMessage
+     * kind yet, so it is a custom channel. Its addresses are not marked as phone numbers, since a handle
+     * may be an email address: the person on iMessage is their own, not joined with their WhatsApp or SMS.
+     */
+    function imessage(o = {}) {
+        return custom(register("imessage", o), IMESSAGE, o);
+    }
+    /** A Flow channel Flow Framework has no kind for: text, quotes, buttons and links as `info` allows, photos, reactions, typing, read receipts. */
+    function custom(route, info, o) {
         const p = provider(route);
         return {
             name: route.name,
             kind: "custom",
-            info: TELEGRAM,
+            info,
             instructions: o.instructions,
             send: (to, outbound) => {
                 switch (outbound.type) {
@@ -181,17 +223,19 @@ export function flowMessaging(options = {}) {
                         return p.sendMessage(to, {
                             text: outbound.text,
                             ...(outbound.replyTo ? { replyTo: outbound.replyTo } : {}),
-                            ...(outbound.buttons ? { buttons: outbound.buttons } : {}),
-                            ...(outbound.link ? { link: outbound.link } : {}),
-                            ...(outbound.image ? { image: outbound.image } : {}),
+                            ...(outbound.buttons && info.buttons ? { buttons: outbound.buttons } : {}),
+                            ...(outbound.link && info.linkButton ? { link: outbound.link } : {}),
+                            ...(outbound.image && info.images ? { image: outbound.image } : {}),
                         });
                     case "reaction":
                         return p.react(to, outbound.messageId, outbound.emoji);
                     case "typing":
                         return p.typing(to, outbound.on);
+                    case "read":
+                        return info.readReceipts ? p.markRead(to, outbound.messageIds) : undefined; // Telegram bots cannot send them
                     case "template":
-                        throw new Error("Telegram has no templates");
-                    // read: Telegram bots cannot send read receipts; status: Telegram has no status line.
+                        throw new Error(`${route.kind === "telegram" ? "Telegram" : "iMessage"} has no templates`);
+                    // status: neither has a status line.
                 }
             },
         };
@@ -207,14 +251,15 @@ export function flowMessaging(options = {}) {
         const known = contacts.get(contactId);
         if (known)
             return known;
-        const contact = await client.get(`/v1/contacts/${contactId}`);
-        const address = (route.kind === "telegram" ? contact.address?.telegram_user_id : contact.address?.phone) ?? contactId;
+        const contact = await client.get(`/v1/contacts/${idSegment("ct", contactId)}`);
+        const a = contact.address;
+        const address = (route.kind === "telegram" ? a?.telegram_user_id : route.kind === "imessage" ? (a?.phone ?? a?.handle) : a?.phone) ?? contactId;
         contacts.set(contactId, address);
         return address;
     }
     async function quoted(id) {
         try {
-            const m = await client.get(`/v1/messages/${id}`);
+            const m = await client.get(`/v1/messages/${idSegment("msg", id)}`);
             const text = textOf(m.content);
             return text ? { text, fromAgent: m.direction === "out" } : undefined;
         }
@@ -306,6 +351,21 @@ export function flowMessaging(options = {}) {
                 agent.undelivered(conversationId, text, reason({ type, channelCode: m.error?.channel_code, message: m.error?.message ?? "The channel did not take it.", retryable: type === "api_error" || type === "rate_limited" }));
         }
     }
+    /** Hands one event to the agent once: a redelivery (or the same event by webhook and stream) is dropped. */
+    async function once(agent, event) {
+        if (seen.has(event.id))
+            return;
+        let handled = handling.get(event.id);
+        if (!handled) {
+            handled = handle(agent, event).finally(() => handling.delete(event.id));
+            handling.set(event.id, handled);
+        }
+        await handled;
+        seen.add(event.id);
+        if (seen.size > SEEN_MAX)
+            seen.delete(seen.values().next().value);
+    }
+    const failed = (event, error) => console.warn(`@flow-engineer/flow/flow-messaging: event ${event.id} (${event.type}) failed: ${error instanceof FlowMessagingError ? error.type : error instanceof Error ? error.name : "error"}`);
     /**
      * One delivery: checks Flow-Signature, drops a redelivered event, hands the event to the agent.
      * 2xx tells Flow it arrived; a failure answers 500, so Flow delivers it again.
@@ -327,27 +387,14 @@ export function flowMessaging(options = {}) {
         }
         if (typeof event?.id !== "string" || typeof event.type !== "string")
             return { status: 400, body: '{"error":"not an event"}' };
-        if (seen.has(event.id))
-            return { status: 200, body: "{}" };
-        let handled = handling.get(event.id);
-        if (!handled) {
-            handled = handle(agent, event);
-            handling.set(event.id, handled);
-        }
         try {
-            await handled;
-            seen.add(event.id);
-            if (seen.size > SEEN_MAX)
-                seen.delete(seen.values().next().value);
+            await once(agent, event);
             return { status: 200, body: "{}" };
         }
         catch (error) {
-            console.warn(`@flow-engineer/flow/flow-messaging: event ${event.id} (${event.type}) failed: ${error instanceof FlowMessagingError ? error.type : error instanceof Error ? error.name : "error"}`);
+            failed(event, error);
             return { status: 500, body: '{"error":"failed"}' };
         }
-        finally {
-            handling.delete(event.id);
-        }
     }
     /** A Node http handler for the webhook endpoint: give it the requests to the URL Flow posts to. */
     function webhook(agent) {
@@ -369,7 +416,156 @@ export function flowMessaging(options = {}) {
             answer(result.status, result.body);
         };
     }
-    return { whatsapp, telegram, sms, webhook, receive };
+    // ---- the stream ----
+    /**
+     * The live event stream (GET /v1/stream): an outbound WebSocket, so the agent needs no public URL.
+     * Events are handled one at a time, in log order, by the webhook's handler. There is no ack frame
+     * for events: the stream resumes from `after`, so an event counts as acknowledged once the agent
+     * has handled it and the cursor (in memory, or `cursorFile`) has moved past it. A failed event, a
+     * dropped socket or Flow's `reconnect` frame reconnects from that cursor, and Flow replays what
+     * came after it: at once the first time, then with a backoff up to 30 s, or after Flow's
+     * retry_after. An event that fails 5 times in a row is passed over, so it cannot hold up every
+     * later one. A refused key or request stops the stream (`stopped`). Replies go over HTTP
+     * (with Idempotency-Keys), so they do not depend on the stream being up.
+     */
+    function stream(agent, o = {}) {
+        if (!apiKey)
+            throw new Error("@flow-engineer/flow/flow-messaging: set FLOW_MESSAGING_KEY (or pass apiKey) to open the stream");
+        let cursor = o.cursorFile ? readCursor(o.cursorFile) : undefined;
+        let socket;
+        let timer;
+        let closed = false;
+        let tries = 0;
+        let failing;
+        let generation = 0;
+        let queue = Promise.resolve();
+        let stop;
+        const stopped = new Promise((resolve) => (stop = resolve));
+        const moveCursor = (id) => {
+            cursor = id;
+            if (o.cursorFile)
+                writeCursor(o.cursorFile, id);
+        };
+        function connect() {
+            const mine = ++generation;
+            const url = new URL(`${client.baseUrl.replace(/^http/, "ws")}/v1/stream`);
+            if (cursor)
+                url.searchParams.set("after", cursor);
+            for (const type of HANDLED)
+                url.searchParams.append("type", type);
+            // Node's WebSocket cannot set headers: the key goes as a subprotocol beside "flow", which Flow selects.
+            const ws = new WebSocket(url, ["flow", `flow.key.${apiKey}`]);
+            socket = ws;
+            let refusal;
+            ws.onopen = () => {
+                if (!failing)
+                    tries = 0; // a failing event keeps its backoff across reconnects
+            };
+            ws.onmessage = (message) => {
+                let frame;
+                try {
+                    frame = JSON.parse(String(message.data));
+                }
+                catch {
+                    return;
+                }
+                if (frame.type === "event" && frame.event && typeof frame.event.id === "string") {
+                    const event = frame.event;
+                    queue = queue.then(() => take(event, mine, ws));
+                }
+                else if (frame.type === "error" && !frame.ref)
+                    refusal = frame.error ?? {};
+            };
+            ws.onerror = () => { }; // a close always follows
+            ws.onclose = ({ code, reason }) => {
+                if (socket === ws)
+                    socket = undefined;
+                if (closed || mine !== generation)
+                    return;
+                if (STREAM_FINAL.has(code)) {
+                    if (code === 4400 && refusal?.param === "after" && cursor) {
+                        // The cursor's event has left Flow's log (past retention): start again from now.
+                        console.warn("@flow-engineer/flow/flow-messaging: the last handled event is no longer in Flow's log; the stream starts again from now");
+                        moveCursor(undefined);
+                        return void later(0);
+                    }
+                    closed = true;
+                    const type = refusal?.type ?? (code === 4401 ? "authentication" : code === 4403 ? "permission" : "invalid_request");
+                    const why = refusal?.message ?? reason ?? "Flow Messaging refused the stream.";
+                    console.warn(`@flow-engineer/flow/flow-messaging: the stream stopped: ${type} (${code})`);
+                    return void stop({ code, type, message: why });
+                }
+                // 1012 follows a reconnect frame: at once. 4429 and 45xx say when (retry_after). Else back off.
+                if (code === 1012)
+                    return void later(0);
+                if (refusal?.retry_after !== undefined)
+                    return void later(refusal.retry_after * 1000);
+                later(Math.min(STREAM_BACKOFF_MAX, tries === 0 ? 0 : 1000 * 2 ** (tries - 1)));
+                tries++;
+            };
+        }
+        function later(ms) {
+            if (closed)
+                return;
+            timer = setTimeout(connect, ms);
+        }
+        /** Handles one event of connection `mine`; one that fails makes the stream reconnect from the cursor. */
+        async function take(event, mine, ws) {
+            if (closed || mine !== generation)
+                return; // a newer connection replays it from the cursor
+            try {
+                await once(agent, event);
+            }
+            catch (error) {
+                failed(event, error);
+                const times = failing?.id === event.id ? failing.times + 1 : 1;
+                if (times < STREAM_EVENT_TRIES) {
+                    failing = { id: event.id, times };
+                    if (mine === generation)
+                        ws.close(1000, "event failed; resuming");
+                    return;
+                }
+                console.warn(`@flow-engineer/flow/flow-messaging: event ${event.id} failed ${times} times; going on without it (it stays in Flow's log, GET /v1/events)`);
+            }
+            failing = undefined;
+            tries = 0;
+            moveCursor(event.id);
+        }
+        connect();
+        return {
+            stopped,
+            async close() {
+                closed = true;
+                clearTimeout(timer);
+                socket?.close(1000, "closing");
+                await queue;
+            },
+        };
+    }
+    /**
+     * Starts receiving events: the webhook on `port` (default PORT, else 8080) when a webhook secret is
+     * set, which means Flow posts to a public URL; otherwise the stream, which needs no URL.
+     */
+    function start(agent, o = {}) {
+        if (!secrets.length)
+            return { mode: "stream", ...stream(agent, o) };
+        const server = createServer(webhook(agent)).listen(o.port ?? Number(process.env.PORT ?? 8080));
+        return { mode: "webhook", stopped: new Promise(() => { }), close: () => new Promise((resolve) => server.close(() => resolve())) };
+    }
+    return { whatsapp, telegram, imessage, sms, webhook, receive, stream, start };
+}
+function readCursor(file) {
+    try {
+        const id = readFileSync(file, "utf8").trim();
+        return /^evt_[A-Za-z0-9_]+$/.test(id) ? id : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function writeCursor(file, id) {
+    mkdirSync(dirname(file), { recursive: true });
+    writeFileSync(file, id ? `${id}\n` : "");
 }
 /** The text of a message's content, as the agent knew it. */
 function textOf(content) {

package/dist/src/names.d.ts

@@ -1,5 +1,10 @@
 /** A setting from the environment: FLOW_FRAMEWORK_<name>, else CHINWAG_<name> from before the rename. */
 export declare function setting(name: string, env?: NodeJS.ProcessEnv): string | undefined;
+/**
+ * The command to print in a next step. `npx flow` would fetch another npm package wherever this one is
+ * not installed, so a hint always names the package.
+ */
+export declare const RUN = "npx @flow-engineer/flow";
 /** An agent's results folder, in the agent's folder: written here, and git-ignored by `flow new`. */
 export declare const RESULTS_DIR = ".flow";
 /** The results folder from before the rename, still read until a run writes {@link RESULTS_DIR}. */

package/dist/src/names.js

@@ -8,6 +8,11 @@ import { join } from "node:path";
 export function setting(name, env = process.env) {
     return env[`FLOW_FRAMEWORK_${name}`] ?? env[`CHINWAG_${name}`];
 }
+/**
+ * The command to print in a next step. `npx flow` would fetch another npm package wherever this one is
+ * not installed, so a hint always names the package.
+ */
+export const RUN = "npx @flow-engineer/flow";
 /** An agent's results folder, in the agent's folder: written here, and git-ignored by `flow new`. */
 export const RESULTS_DIR = ".flow";
 /** The results folder from before the rename, still read until a run writes {@link RESULTS_DIR}. */

package/dist/testing/src/account.d.ts

@@ -0,0 +1,4 @@
+export declare const ACCOUNT_COMMANDS: Set<string>;
+export declare const ACCOUNT_LINES = "  flow login [--wait | --no-wait] [--timeout <s>] [--force] [--no-browser] [--json]\n                            Sign in to flow.engineer: opens the browser, prints the link and code,\n                            and waits up to --timeout (default 120) for approval; --wait goes on\n                            waiting for the same code, --no-wait returns at once\n  flow logout [--json]      End this machine's sign-in and delete its token\n  flow whoami [--json]      Show the account you are signed in as\n  flow setup [--replace] [--ci] [--json]\n                            Write this agent's keys (test cloud, traces, Flow Messaging) and each\n                            open channel's sender into ./.env (in an agent's folder: package.json)\n  flow link [--channel telegram|whatsapp|imessage] [--timeout <s>] [--no-browser] [--json]\n                            Print the link that opens a chat with this agent from a phone; signs in\n                            and runs setup first when this folder is not set up yet";
+/** Runs one account command; resolves to the exit status. */
+export declare function accountCli(command: string, args: string[]): Promise<number>;