@pmndrs/pine 0.0.0-stage → 0.1.1

package/src/sources/ftp.js

@@ -0,0 +1,307 @@
+import fs from "node:fs";
+import net from "node:net";
+import path from "node:path";
+import { timingSafeEqual } from "../security.js";
+import { Type } from "typebox";
+import { errText } from "../util.js";
+/**
+ * An FTP drop: a small FTP server that accepts uploads and nothing else. For cameras, doorbells,
+ * scanners and other devices whose only way of sending a file is "upload to FTP server".
+ *
+ * options: { port, user, password, host?, allow?, pasvPorts?, maxFileMb?, keepDays? }
+ *   port, user, password  what you enter in the device. The password must have 8 characters or more.
+ *   host       address to listen on. Default "0.0.0.0": devices on your network have to reach it.
+ *   allow      only these client addresses, as prefixes: ["192.168.1.40", "192.168.2."]
+ *   pasvPorts  [from, to]: ports for the data connections, if a firewall needs to know them
+ *   maxFileMb  largest file accepted (default 50)
+ *   keepDays   received files are deleted after this many days (default 7)
+ *
+ * event "upload": { file, path, size, from, images? }   file is the name the device gave it
+ *
+ * It speaks plain FTP, as these devices do: the password crosses your network unencrypted. It can
+ * only receive. Nothing can be listed, read, changed or deleted through it, and every file lands
+ * under a name of pine's choosing inside the source's own folder.
+ */
+const TYPES = { ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png", ".webp": "image/webp", ".gif": "image/gif" };
+export const type = {
+    summary: "an FTP drop: a small server that accepts uploads and nothing else. For cameras, doorbells and scanners whose only way of sending a file is \"upload to FTP server\"",
+    options: Type.Object({
+        port: Type.Number({ description: "port to listen on, e.g. 2121" }),
+        user: Type.Optional(Type.String({ description: 'user name the device logs in with. Default "device"' })),
+        password: Type.Optional(Type.String({ description: "password the device logs in with, 8 characters or more. Leave it out and pine makes one", secret: true })),
+        host: Type.Optional(Type.String({ description: 'address to listen on. Default "0.0.0.0", so that devices on the network reach it' })),
+        allow: Type.Optional(Type.Array(Type.String(), { description: 'only these client addresses, as prefixes: ["192.168.1.40", "192.168.2."]' })),
+        pasvPorts: Type.Optional(Type.Array(Type.Number(), { description: "[from, to]: ports for the data connections, if a firewall needs to know them" })),
+        maxFileMb: Type.Optional(Type.Number({ description: "largest file accepted, in MB. Default 50" })),
+        keepDays: Type.Optional(Type.Number({ description: "received files are deleted after this many days. Default 7" })),
+    }, { additionalProperties: false }),
+    plan(o, setup) {
+        if (!Number.isInteger(o.port) || o.port < 1 || o.port > 65535)
+            throw new Error(`"port" must be a port number, for example 2121`);
+        if (o.password !== undefined && String(o.password).length < 8)
+            throw new Error("the password must have 8 characters or more");
+        if (o.pasvPorts && o.pasvPorts.length !== 2)
+            throw new Error(`"pasvPorts" is two numbers: [from, to]`);
+        const user = o.user ?? "device";
+        return {
+            options: { ...o, user, password: o.password ?? setup.random(9) },
+            does: [`open an upload-only FTP server on port ${o.port} of this computer for the user "${user}"${o.password ? "" : " with a password made now"}${o.allow?.length ? `, for the addresses ${o.allow.join(", ")}` : ""}`],
+            opens: true,
+            tell: `The device uploads to port ${o.port} as user "${user}". The password is in pine's .env file as ${setup.envName("password")}, where your owner reads it.`,
+        };
+    },
+};
+export default async function ftp(source) {
+    const o = source.options;
+    const port = Number(o.port);
+    if (!Number.isInteger(port) || port < 1 || port > 65535)
+        throw new Error(`needs "port" (for example 2121)`);
+    const user = String(o.user ?? "");
+    const password = String(o.password ?? "");
+    if (!user || password.length < 8)
+        throw new Error(`needs "user" and a "password" of 8 characters or more: without them anyone on the network could hand files to your agents`);
+    source.secret(password);
+    const host = String(o.host ?? "0.0.0.0");
+    const allow = (Array.isArray(o.allow) ? o.allow : []).map(String);
+    const range = Array.isArray(o.pasvPorts) && o.pasvPorts.length === 2 ? [Number(o.pasvPorts[0]), Number(o.pasvPorts[1])] : undefined;
+    const maxBytes = Number(o.maxFileMb ?? 50) * 1024 * 1024;
+    const keepMs = Number(o.keepDays ?? 7) * 86_400_000;
+    const dir = path.join(source.dataDir, "files");
+    fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+    source.media(dir);
+    const prune = () => {
+        for (const name of fs.readdirSync(dir)) {
+            const file = path.join(dir, name);
+            try {
+                if (Date.now() - fs.statSync(file).mtimeMs > keepMs)
+                    fs.rmSync(file, { force: true });
+            }
+            catch {
+                // gone already
+            }
+        }
+    };
+    prune();
+    const pruner = setInterval(prune, 6 * 3_600_000);
+    pruner.unref?.();
+    const plain = (addr) => (addr ?? "").replace(/^::ffff:/, "");
+    const open = new Set();
+    let counter = 0;
+    const server = net.createServer((control) => {
+        const remote = plain(control.remoteAddress);
+        if (open.size >= 20 || (allow.length && !allow.some((a) => remote === a || (a.endsWith(".") && remote.startsWith(a)))))
+            return void control.destroy();
+        open.add(control);
+        control.setTimeout(120_000, () => control.destroy());
+        control.setEncoding("utf8");
+        let name = "";
+        let authed = false;
+        let failures = 0;
+        let cwd = "/";
+        let passive;
+        let data;
+        const say = (line) => control.writable && control.write(`${line}\r\n`);
+        const closePassive = () => {
+            passive?.close();
+            passive = undefined;
+            data?.then((s) => s.destroy()).catch(() => { });
+            data = undefined;
+        };
+        control.on("close", () => {
+            open.delete(control);
+            closePassive();
+        });
+        control.on("error", () => { });
+        /** Open a port for the next transfer. Only the device on the control connection may connect to it. */
+        const listen = async () => {
+            closePassive();
+            const s = net.createServer();
+            passive = s;
+            data = new Promise((resolve, reject) => {
+                const timer = setTimeout(() => reject(new Error("no data connection")), 20_000);
+                s.on("connection", (sock) => {
+                    if (plain(sock.remoteAddress) !== remote)
+                        return void sock.destroy();
+                    clearTimeout(timer);
+                    s.close();
+                    resolve(sock);
+                });
+                s.on("close", () => clearTimeout(timer));
+            });
+            data.catch(() => { });
+            const tryPort = (p) => new Promise((resolve, reject) => {
+                s.once("error", reject);
+                s.listen(p, host === "0.0.0.0" ? plain(control.localAddress) : host, () => (s.removeListener("error", reject), resolve(s.address().port)));
+            });
+            if (!range)
+                return tryPort(0);
+            for (let p = range[0]; p <= range[1]; p++) {
+                try {
+                    return await tryPort(p);
+                }
+                catch {
+                    // taken; try the next one
+                }
+            }
+            throw new Error("no free data port");
+        };
+        const receive = async (given) => {
+            if (!data)
+                return say("425 Use PASV first");
+            const wanted = path.posix.join(cwd, given);
+            const clean = path.basename(given).replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^\.+/, "").slice(-80) || "file";
+            const target = path.join(dir, `${Date.now().toString(36)}${(counter++).toString(36)}-${clean}`);
+            say("150 Send it");
+            let sock;
+            try {
+                sock = await data;
+            }
+            catch {
+                return say("425 No data connection");
+            }
+            data = undefined;
+            let size = 0;
+            let tooBig = false;
+            // "wx": the name is new; nothing that exists is ever written over.
+            const out = fs.createWriteStream(target, { flags: "wx", mode: 0o600 });
+            sock.on("data", (chunk) => {
+                size += chunk.length;
+                if (size > maxBytes) {
+                    tooBig = true;
+                    sock.destroy();
+                }
+            });
+            sock.pipe(out);
+            await new Promise((resolve) => {
+                out.on("close", resolve);
+                out.on("error", () => resolve());
+                // A connection that is cut (by the device, or here because the file got too large) never
+                // says "end"; the file is closed all the same, so the transfer gets its answer.
+                sock.on("close", () => out.end());
+                sock.on("error", () => out.end());
+            });
+            if (tooBig || size === 0) {
+                fs.rmSync(target, { force: true });
+                return say(tooBig ? "552 File too large" : "451 Nothing received");
+            }
+            say("226 Stored");
+            const mimeType = TYPES[path.extname(clean).toLowerCase()];
+            try {
+                source.emit("upload", { file: wanted, path: target, size, from: remote, images: mimeType && size < 10 * 1024 * 1024 ? [{ path: target, mimeType }] : undefined });
+            }
+            catch (e) {
+                source.log.warn("could not report an upload", errText(e));
+            }
+        };
+        const handle = async (line) => {
+            const [verb, ...rest] = line.trim().split(" ");
+            const arg = rest.join(" ");
+            const cmd = verb.toUpperCase();
+            if (cmd === "QUIT")
+                return void (say("221 Bye"), control.end());
+            if (cmd === "USER")
+                return void ((name = arg), say("331 Password"));
+            if (cmd === "PASS") {
+                authed = timingSafeEqual(name, user) && timingSafeEqual(arg, password);
+                if (authed)
+                    return say("230 Welcome");
+                if (++failures >= 3)
+                    return void (say("530 Bye"), control.end());
+                return say("530 Login incorrect");
+            }
+            if (cmd === "SYST")
+                return say("215 UNIX Type: L8");
+            if (cmd === "FEAT")
+                return say("211-Features:\r\n PASV\r\n EPSV\r\n UTF8\r\n211 End");
+            if (!authed)
+                return say("530 Log in first");
+            switch (cmd) {
+                case "OPTS":
+                case "TYPE":
+                case "MODE":
+                case "STRU":
+                case "NOOP":
+                case "ALLO":
+                    return say("200 OK");
+                case "PWD":
+                case "XPWD":
+                    return say(`257 "${cwd}"`);
+                case "CWD":
+                case "XCWD":
+                    cwd = path.posix.resolve(cwd, arg || "/");
+                    return say("250 OK");
+                case "CDUP":
+                    cwd = path.posix.resolve(cwd, "..");
+                    return say("250 OK");
+                // Folders exist only as part of the name a file is reported under.
+                case "MKD":
+                case "XMKD":
+                    return say(`257 "${path.posix.resolve(cwd, arg)}"`);
+                case "PASV": {
+                    const p = await listen();
+                    const ip = plain(control.localAddress).split(".");
+                    if (ip.length !== 4)
+                        return say("425 Use EPSV");
+                    return say(`227 Entering Passive Mode (${ip.join(",")},${p >> 8},${p & 255})`);
+                }
+                case "EPSV":
+                    return say(`229 Entering Extended Passive Mode (|||${await listen()}|)`);
+                case "STOR":
+                case "APPE":
+                case "STOU":
+                    return receive(arg || "file");
+                case "LIST":
+                case "NLST":
+                case "MLSD": {
+                    // Always empty: this server gives nothing back.
+                    if (!data)
+                        return say("425 Use PASV first");
+                    say("150 Here it comes");
+                    const sock = await data.catch(() => undefined);
+                    data = undefined;
+                    sock?.end();
+                    return say("226 Done");
+                }
+                // Some devices upload under a temporary name and rename afterwards. The file is already reported; say yes.
+                case "RNFR":
+                    return say("350 Ready");
+                case "RNTO":
+                case "DELE":
+                case "RMD":
+                case "SITE":
+                    return say("250 OK");
+                case "SIZE":
+                case "MDTM":
+                case "RETR":
+                    return say("550 Not available");
+                default:
+                    return say("502 Not implemented");
+            }
+        };
+        let buffer = "";
+        let busy = Promise.resolve();
+        control.on("data", (chunk) => {
+            buffer += chunk;
+            if (buffer.length > 8192)
+                return void control.destroy();
+            let i;
+            while ((i = buffer.indexOf("\n")) >= 0) {
+                const line = buffer.slice(0, i).replace(/\r$/, "");
+                buffer = buffer.slice(i + 1);
+                busy = busy.then(() => handle(line)).catch((e) => (source.log.debug("ftp command failed", errText(e)), say("451 Failed")));
+            }
+        });
+        say("220 pine file drop");
+    });
+    await new Promise((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(port, host, () => (server.removeListener("error", reject), resolve()));
+    });
+    source.onStop(() => new Promise((resolve) => {
+        clearInterval(pruner);
+        for (const s of open)
+            s.destroy();
+        server.close(() => resolve());
+    }));
+    source.log.info(`accepting uploads on ${host}:${port}`);
+}

package/src/sources/http.d.ts

@@ -0,0 +1,4 @@
+import type { SourceType } from "../source-types.ts";
+import type { SourceApi } from "../sources.ts";
+export declare const type: SourceType;
+export default function http(source: SourceApi): void;

package/src/sources/http.js

@@ -0,0 +1,201 @@
+import net from "node:net";
+import { Type } from "typebox";
+import { fetch } from "undici";
+import { isLocalName, isPrivateAddress, publicOnlyAgent } from "../net.js";
+import { errText, getPath, parseDuration } from "../util.js";
+import { clip, found, harmless, lineDiff, pageText, pattern, shown } from "../watching.js";
+/**
+ * A web address, looked at again and again: reports when what it answers changes, and when it
+ * stops or starts answering. For pages and services that cannot call you: a price, a version, a
+ * status page, a device with a state page on your network.
+ *
+ * options: { url, every?, path?, match?, headers?, token?, tokenHeader?, local? }
+ *   every        how often to look ("5m" by default)
+ *   path         for a JSON answer: watch only this field ("data.price")
+ *   match        for a page or text: watch only what this regular expression finds
+ *   headers      headers to send
+ *   token        sent as "Authorization: Bearer <token>", or as it is in the header `tokenHeader`
+ *   local        the address is on the owner's own network, under a name that does not say so
+ *
+ * event "changed": { url, value, previous }           with path or match
+ *                  { url, added, removed, more? }     without: the lines that are new and gone
+ * event "down":    { url, error, looks }              after three failed looks in a row
+ * event "up":      { url }                            when it answers again
+ *
+ * The first look is the starting point and reports nothing.
+ */
+const MIN = 30_000;
+const DOWN_AFTER = 3;
+const MAX_BODY = 1024 * 1024;
+const TIMEOUT = 20_000;
+/**
+ * Whether an address says by itself that it is on this computer or the local network. Such an
+ * address the owner confirmed as what it is. Any other one must stay on the public internet: a
+ * name that one day points into the local network is not followed there.
+ */
+const isLocal = (url) => {
+    const host = url.hostname.replace(/^\[|\]$/g, "");
+    return isLocalName(host) || (net.isIP(host) !== 0 && isPrivateAddress(host));
+};
+/** Why a look failed, in words: what lies behind "fetch failed". */
+const reason = (e, url) => {
+    const text = errText(e?.cause ?? e);
+    return /resolves to a private address/.test(text) ? `${url.hostname} leads to an address on your own network. If that is meant, set the source up again with "local": true` : text;
+};
+/** Headers that carry a secret. Those go into `token`, which is kept out of the configuration. */
+const SECRET_HEADERS = ["authorization", "proxy-authorization", "cookie"];
+export const type = {
+    summary: "a web address looked at again and again: reports when what it answers changes (the whole page, or one value in it: a price, a version, a state), and when it stops or starts answering. For pages and services that cannot call you",
+    options: Type.Object({
+        url: Type.String({ description: "the address to look at, https://... (http:// for a device on the local network)" }),
+        every: Type.Optional(Type.String({ description: 'how often to look, e.g. "5m", "1h". Default "5m", at least "30s"' })),
+        path: Type.Optional(Type.String({ description: 'for an answer in JSON: watch only this field, e.g. "data.price" or "items.0.state"' })),
+        match: Type.Optional(Type.String({ description: 'for a page or text: watch only what this regular expression finds (its first group, if it has one), e.g. "Price: ([0-9.,]+)". Without path or match the whole text counts, and a page that shows the time or a counter changes at every look' })),
+        headers: Type.Optional(Type.Record(Type.String(), Type.String(), { description: 'headers to send, e.g. {"accept": "application/json"}. Not for secrets' })),
+        token: Type.Optional(Type.String({ description: "an access token or key, if the address wants one. Sent as Authorization: Bearer <token>", secret: true })),
+        tokenHeader: Type.Optional(Type.String({ description: 'send the token as this header instead, as it is, e.g. "x-api-key"' })),
+        local: Type.Optional(Type.Boolean({ description: "true if the address is on your owner's own network under a name that does not say so (nas.home, ha.lan). Not needed for an IP address, localhost or a .local name" })),
+    }, { additionalProperties: false }),
+    plan(o) {
+        let url;
+        try {
+            url = new URL(o.url);
+        }
+        catch {
+            throw new Error(`"${o.url}" is not an address. It looks like https://example.com/page`);
+        }
+        if (!["http:", "https:"].includes(url.protocol))
+            throw new Error("the address must start with https:// or http://");
+        if (url.username || url.password)
+            throw new Error('an address with a user name or password in it is not taken: leave it out of the address and name "token" in "ask"');
+        const every = o.every ?? "5m";
+        if (parseDuration(every) < MIN)
+            throw new Error('"every" must be at least "30s"');
+        if (o.path && o.match)
+            throw new Error('"path" is for an answer in JSON, "match" for a page or text: give one of them');
+        if (o.match)
+            pattern(o.match);
+        const secret = Object.keys(o.headers ?? {}).filter((h) => SECRET_HEADERS.includes(h.toLowerCase()));
+        if (secret.length)
+            throw new Error(`the header ${secret.join(", ")} carries a secret: leave it out, name "token" in "ask" and give "tokenHeader": "${secret[0].toLowerCase()}"`);
+        if (o.tokenHeader && !/^[A-Za-z0-9-]+$/.test(o.tokenHeader))
+            throw new Error('"tokenHeader" is the name of a header, e.g. "x-api-key"');
+        const what = o.path ? `the field ${o.path} of what it answers` : o.match ? `what /${o.match}/ finds in it` : "what it answers";
+        return {
+            options: { ...o, url: url.href },
+            does: [`ask ${clip(url.href, 160)}${o.local || isLocal(url) ? " (an address on your own network or this computer)" : ""} every ${every} and report when ${what} changes, and when it stops or starts answering`, ...(o.token ? [`with the token given, sent only to ${url.host}`] : [])],
+            opens: true,
+            tell: "Its first look is the starting point and reports nothing; from then on only a change does. The sources tool shows what it found at the last look.",
+        };
+    },
+};
+/** One look: the answer's text, at most 1 MB of it. A redirect is followed only on the same host. */
+async function get(start, o, guard, signal) {
+    const headers = { ...(o.headers ?? {}) };
+    if (o.token)
+        headers[o.tokenHeader ?? "authorization"] = o.tokenHeader ? o.token : `Bearer ${o.token}`;
+    let at = start;
+    for (let hop = 0; hop < 6; hop++) {
+        const res = await fetch(at, { headers, redirect: "manual", signal: AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT)]), ...(guard ? { dispatcher: guard } : {}) });
+        const to = res.status >= 300 && res.status < 400 ? res.headers.get("location") : null;
+        if (to) {
+            await res.body?.cancel().catch(() => { });
+            const next = new URL(to, at);
+            // The owner confirmed one host. Another one is another decision, and must not get the token.
+            const upgrade = at.protocol === "http:" && next.protocol === "https:";
+            if (next.hostname !== start.hostname || (next.protocol !== at.protocol && !upgrade) || (next.port !== at.port && !upgrade))
+                throw new Error(`it sends on to ${next.host}, which is not the address that was confirmed. If that is the right one, set the source up with it`);
+            at = next;
+            continue;
+        }
+        if (!res.ok) {
+            await res.body?.cancel().catch(() => { });
+            throw new Error(`it answered ${res.status}`);
+        }
+        const chunks = [];
+        let size = 0;
+        const reader = res.body?.getReader();
+        while (reader) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            chunks.push(Buffer.from(value.subarray(0, MAX_BODY - size)));
+            size += value.length;
+            if (size >= MAX_BODY) {
+                await reader.cancel().catch(() => { });
+                break;
+            }
+        }
+        return { type: res.headers.get("content-type") ?? "", body: Buffer.concat(chunks).toString("utf8") };
+    }
+    throw new Error("it sends on and on (more than five redirects)");
+}
+export default function http(source) {
+    const o = source.options;
+    if (typeof o.url !== "string")
+        throw new Error(`needs "url": the address to look at`);
+    const url = new URL(o.url);
+    if (!["http:", "https:"].includes(url.protocol))
+        throw new Error("the address must start with https:// or http://");
+    const re = o.match ? pattern(String(o.match)) : undefined;
+    source.secret(o.token);
+    const guard = o.local === true || isLocal(url) ? undefined : publicOnlyAgent(TIMEOUT);
+    if (guard)
+        source.onStop(() => guard.close());
+    // How many looks in a row failed. Whether "down" was said is kept, so that "up" follows it also across a restart.
+    let failed = 0;
+    source.poll(o.every ?? "5m", async (signal) => {
+        let answer;
+        try {
+            answer = await get(url, o, guard, signal);
+        }
+        catch (e) {
+            if (signal.aborted)
+                return;
+            const why = reason(e, url);
+            failed++;
+            if (failed === DOWN_AFTER && !source.kv.get("down")) {
+                source.kv.set("down", true);
+                source.emit("down", { url: o.url, error: why, looks: failed });
+            }
+            throw new Error(why);
+        }
+        failed = 0;
+        if (source.kv.get("down")) {
+            source.kv.set("down", false);
+            source.emit("up", { url: o.url });
+        }
+        const html = /html/i.test(answer.type) || /^\s*<(!doctype html|html)\b/i.test(answer.body);
+        let json;
+        if (!html) {
+            try {
+                json = JSON.parse(answer.body);
+            }
+            catch {
+                // plain text
+            }
+        }
+        if (o.path || re) {
+            let value = null;
+            if (o.path)
+                value = json === undefined ? null : (getPath(json, String(o.path)) ?? null);
+            // In what a reader sees first; a pattern that is about the markup finds it in the source.
+            else
+                value = (html ? found(re, pageText(answer.body)) : null) ?? found(re, answer.body);
+            source.status(value === null ? `found nothing ${o.path ? `at ${o.path}` : "with the pattern"}` : `now: ${harmless(value)}`);
+            const was = source.changed(value);
+            if (was)
+                source.emit("changed", { url: o.url, value: shown(value), previous: shown(was.previous) });
+            return;
+        }
+        const text = (html ? pageText(answer.body) : json !== undefined ? JSON.stringify(json, null, 1) : answer.body).slice(0, 200_000);
+        source.status(`now: ${text.split("\n").length} lines`);
+        const was = source.changed(text);
+        if (!was)
+            return;
+        const diff = lineDiff(String(was.previous), text);
+        // The same lines in another order are not worth a turn.
+        if (diff.added.length || diff.removed.length)
+            source.emit("changed", { url: o.url, ...diff });
+    });
+}

package/src/sources/mqtt.d.ts

@@ -0,0 +1,18 @@
+import type { SourceType } from "../source-types.ts";
+import type { SourceApi } from "../sources.ts";
+/**
+ * An MQTT broker: reports the messages on the topics you name. For home automation hubs, sensors,
+ * and devices that speak MQTT themselves (a Bambu printer does, on mqtts://<printer>:8883 with the
+ * user "bblp", its access code as password, the topic "device/<serial>/report" and insecureTls).
+ *
+ * options: { url, topics, publish?, username?, password?, clientId?, minInterval?, insecureTls? }
+ *   topics       one MQTT topic filter or a list ("zigbee2mqtt/+", "device/+/report")
+ *   publish      topics an agent may send to, to act on a device ("zigbee2mqtt/+/set"). Without it nothing can be sent
+ *   minInterval  drop messages that follow the last one on the same topic sooner than this ("30s")
+ *   insecureTls  accept a certificate that cannot be verified, as devices with self-made ones need
+ *
+ * event "message": { topic, value }   value is parsed JSON when the message is JSON, else text
+ * It can send, to the topics on `publish` only: { topic, message, retain?, qos? }
+ */
+export declare const type: SourceType;
+export default function mqttSource(source: SourceApi): Promise<void>;

package/src/sources/mqtt.js

@@ -0,0 +1,97 @@
+import { Type } from "typebox";
+import { errText, matchesAny, parseDuration } from "../util.js";
+/**
+ * An MQTT broker: reports the messages on the topics you name. For home automation hubs, sensors,
+ * and devices that speak MQTT themselves (a Bambu printer does, on mqtts://<printer>:8883 with the
+ * user "bblp", its access code as password, the topic "device/<serial>/report" and insecureTls).
+ *
+ * options: { url, topics, publish?, username?, password?, clientId?, minInterval?, insecureTls? }
+ *   topics       one MQTT topic filter or a list ("zigbee2mqtt/+", "device/+/report")
+ *   publish      topics an agent may send to, to act on a device ("zigbee2mqtt/+/set"). Without it nothing can be sent
+ *   minInterval  drop messages that follow the last one on the same topic sooner than this ("30s")
+ *   insecureTls  accept a certificate that cannot be verified, as devices with self-made ones need
+ *
+ * event "message": { topic, value }   value is parsed JSON when the message is JSON, else text
+ * It can send, to the topics on `publish` only: { topic, message, retain?, qos? }
+ */
+export const type = {
+    summary: "an MQTT broker: reports the messages on the topics you name. For home automation hubs, sensors, and devices that speak MQTT themselves (printers, plugs). With \"publish\" it can also send, to act on a device",
+    sends: 'an MQTT message to a topic on its "publish" list, to act on a device: {"topic": "lamp/set", "message": "on"} (message: text or JSON)',
+    options: Type.Object({
+        url: Type.String({ description: "address of the broker: mqtt://host:1883, or mqtts://host:8883 for TLS" }),
+        topics: Type.Union([Type.String(), Type.Array(Type.String())], { description: 'the topic filters to listen to, e.g. "zigbee2mqtt/+" or ["device/+/report"]' }),
+        publish: Type.Optional(Type.Array(Type.String(), { description: 'topics you may send to, to act on a device, e.g. ["zigbee2mqtt/+/set"]. Only when your owner wants you to control something; without it nothing can be sent' })),
+        username: Type.Optional(Type.String({ description: "user name, if the broker wants one" })),
+        password: Type.Optional(Type.String({ description: "password, access code or token, if the broker wants one", secret: true })),
+        clientId: Type.Optional(Type.String({ description: "client id, for brokers that want a particular one" })),
+        minInterval: Type.Optional(Type.String({ description: 'drop messages that follow the last one on the same topic sooner than this, e.g. "30s". For devices that report many times a second' })),
+        insecureTls: Type.Optional(Type.Boolean({ description: "accept a certificate that cannot be verified, as devices with self-made ones need" })),
+    }, { additionalProperties: false }),
+    plan(o) {
+        let url;
+        try {
+            url = new URL(o.url);
+        }
+        catch {
+            throw new Error(`"${o.url}" is not an address. An MQTT broker's looks like mqtt://192.168.1.10:1883`);
+        }
+        if (!["mqtt:", "mqtts:", "ws:", "wss:"].includes(url.protocol))
+            throw new Error("an MQTT broker's address starts with mqtt://, mqtts://, ws:// or wss://");
+        const topics = (Array.isArray(o.topics) ? o.topics : String(o.topics).split(",")).map((t) => t.trim()).filter(Boolean);
+        if (!topics.length)
+            throw new Error(`needs "topics": which MQTT topics to listen to`);
+        if (o.minInterval)
+            parseDuration(o.minInterval);
+        const publish = (Array.isArray(o.publish) ? o.publish : []).map((t) => String(t).trim()).filter(Boolean);
+        return { options: { ...o, topics, ...(publish.length ? { publish } : {}) }, does: [`connect to the MQTT broker ${url.host}${o.username ? ` as ${o.username}` : ""} and listen to ${topics.join(", ")}`, ...(publish.length ? [`let the agent send messages to ${publish.join(", ")}, which can switch and control devices`] : []), ...(o.password ? ["with the password given"] : []), ...(o.insecureTls ? ["without checking its certificate"] : [])], opens: true };
+    },
+};
+export default async function mqttSource(source) {
+    const { default: mqtt } = await import("mqtt");
+    const o = source.options;
+    if (typeof o.url !== "string")
+        throw new Error(`needs "url", for example mqtt://192.168.1.10:1883`);
+    const topics = (Array.isArray(o.topics) ? o.topics : o.topics ? [o.topics] : []).map(String);
+    if (!topics.length)
+        throw new Error(`needs "topics": which MQTT topics to listen to`);
+    const minMs = o.minInterval !== undefined ? parseDuration(o.minInterval) : 0;
+    source.secret(o.password);
+    const client = mqtt.connect(o.url, { username: o.username, password: o.password, clientId: o.clientId ?? `pine-${source.name}-${process.pid}`, reconnectPeriod: 5000, connectTimeout: 10_000, rejectUnauthorized: o.insecureTls !== true });
+    source.onStop(() => new Promise((r) => client.end(true, {}, () => r())));
+    client.on("error", (e) => source.log.warn("mqtt error", errText(e)));
+    client.on("connect", () => {
+        source.log.info(`connected to ${new URL(o.url).host}`);
+        for (const t of topics)
+            client.subscribe(t, (err) => err && source.log.warn(`subscribe ${t} failed`, errText(err)));
+    });
+    const last = new Map();
+    client.on("message", (topic, buf) => {
+        const now = Date.now();
+        if (minMs && now - (last.get(topic) ?? 0) < minMs)
+            return;
+        last.set(topic, now);
+        if (last.size > 5000)
+            last.clear();
+        const text = buf.length > 256 * 1024 ? "" : buf.toString("utf8");
+        let value = text;
+        try {
+            value = JSON.parse(text);
+        }
+        catch {
+            // plain text
+        }
+        source.emit("message", { topic, value });
+    });
+    // Sending: only to the topics the owner put on the list, and never to a filter.
+    const allowed = (Array.isArray(o.publish) ? o.publish : []).map(String);
+    source.onSend(async (m) => {
+        const topic = String(m.topic ?? "");
+        if (!topic)
+            throw new Error('say which topic: {"topic": "...", "message": "..."}');
+        if (/[+#]/.test(topic) || !matchesAny(allowed, topic))
+            throw new Error(`"${topic}" is not a topic this source may send to (${allowed.join(", ") || "none: it was set up to listen only"})`);
+        const message = m.message ?? m.text ?? "";
+        await new Promise((resolve, reject) => client.publish(topic, typeof message === "string" ? message : JSON.stringify(message), { retain: m.retain === true, qos: [0, 1, 2].includes(m.qos) ? m.qos : 0 }, (err) => (err ? reject(err) : resolve())));
+        return `published to ${topic}`;
+    });
+}

package/src/sources/telegram.d.ts

@@ -0,0 +1,22 @@
+import type { SourceType } from "../source-types.ts";
+import type { SourceApi } from "../sources.ts";
+/**
+ * A Telegram bot: the owner writes to it from wherever they are, and the agent answers there.
+ *
+ * options: { token, allow?, approvals?, apiBase? }
+ *   token      the bot's token from @BotFather
+ *   allow      Telegram user ids that may talk to it. Without it the bot is paired: it sends a
+ *              code to the account that writes to it, and the owner types that code into pine
+ *   approvals  also send approval requests to that chat, with buttons (default true)
+ *   apiBase    a Bot API server of your own instead of api.telegram.org
+ *
+ * event "message": { text, name, username, chatId, messageId, images? }
+ * It can send: { text, chatId? } goes to the owner's chat.
+ *
+ * Messages are fetched by long polling: one request that Telegram answers the moment something
+ * arrives. Nothing listens on a port, so it works behind any router. Only accounts on `allow`, or
+ * the one that was paired, are heard. What anyone else writes reaches no agent; once an account
+ * is paired they get no answer at all, and nothing is ever sent to them.
+ */
+export declare const type: SourceType;
+export default function telegram(source: SourceApi): Promise<void>;