@rubytech/create-maxy-code 0.1.604 → 0.1.605

package/payload/platform/plugins/scheduling/mcp/dist/tools/scheduling-gate.js

@@ -19,12 +19,37 @@
  * on every surface, which is the Task 2313 class this must not recreate.
  */
 import { createHash } from "node:crypto";
-import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { dirname, resolve } from "node:path";
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
 import { int } from "neo4j-driver";
 import { spawnProviderCall } from "../lib/connector-freebusy.js";
 import { unflaggedLabel } from "../lib/gate-flag.js";
 import { getSession } from "../lib/neo4j.js";
+/**
+ * Task 2689 — how long after its newest report the memory sweep counts as
+ * stale.
+ *
+ * The sweep runs hourly and `graph-health`'s own silence condition is "the
+ * sweep ran within the past day", so 26 hours gives it two missed runs of slack
+ * before the routine is woken.
+ */
+export const SWEEP_STALE_AFTER_MS = 26 * 60 * 60 * 1000;
+/**
+ * Task 2689 — one week of growth worth a person's attention on a Pi-class disk.
+ *
+ * The shipped `filesystem-health` template names its threshold explicitly; a
+ * gate written by hand that omits the field inherits this.
+ */
+export const FS_DEFAULT_MIN_GROWTH_BYTES = 64 * 1024 * 1024;
+/**
+ * Task 2689 — the most files one account-directory measurement walks.
+ *
+ * A bound, not a budget. A directory above it is exactly what
+ * `filesystem-health` exists to report, and the walk that stopped there has
+ * measured a floor rather than a total, so the check reports work and records
+ * no position.
+ */
+export const FS_WALK_ENTRY_CAP = 200_000;
 /**
  * Task 2684 — the most node ids one `graph-unflagged` check hands over per fire.
  *
@@ -53,7 +78,19 @@ export const ACCEPTED_CHECK_TYPES = [
     "telegram",
     "graph",
     "graph-unflagged",
+    "connector",
+    "sweep",
+    "filesystem",
 ];
+/**
+ * Task 2688 — the accepted `scope` values on a `connector` check, in one place.
+ *
+ * The same single-sourcing `ACCEPTED_MAIL_PROVIDERS` has: the probe asks the
+ * registry for queues of this kind, and `validateGate` refuses anything outside
+ * it at write time. A scope outside the set matches no queue, so the gate would
+ * answer quiet on every fire and the routine could never run.
+ */
+export const ACCEPTED_CONNECTOR_SCOPES = ["operational", "accounting"];
 /**
  * Task 2657 — the accepted `provider` values on a `mail` check, in one place.
  *
@@ -88,6 +125,9 @@ export const ACCEPTED_CHECK_FIELDS = {
     telegram: ["type", "scope", "botId"],
     graph: ["type", "scope"],
     "graph-unflagged": ["type", "label"],
+    connector: ["type", "scope"],
+    sweep: ["type"],
+    filesystem: ["type", "minGrowthBytes"],
 };
 /**
  * Rooted at the INSTALL, not at the account.
@@ -241,6 +281,10 @@ export function slotKey(slot) {
     // commit over two bots names two entries rather than one repeated twice.
     if (slot.kind === "telegram" && slot.botId !== undefined)
         return `telegram:${slot.botId}`;
+    // Task 2688 — the connector's name, so a commit over two connectors names two
+    // entries rather than one repeated twice.
+    if (slot.kind === "connector")
+        return `connector:${slot.key}`;
     return slot.kind;
 }
 /**
@@ -273,6 +317,19 @@ export function slotPosition(slot, marks) {
             // The ISO `updatedAt` verbatim, as the Telegram position is: it IS the
             // value the comparison is made on.
             return marks.graph?.updatedAt ?? null;
+        case "connector":
+            // The ISO instant sent as the queue's changed-since argument, printed
+            // verbatim for the same reason.
+            return marks.connector?.[slot.key]?.changedSince ?? null;
+        case "sweep":
+            // Truncated for the reason the calendar digest is: the whole sha256 tells
+            // a reader nothing the first eight do not, and the file holds it all.
+            return marks.sweep?.digest?.slice(0, 8) ?? null;
+        case "filesystem":
+            // The optional chain, as every arm above has it: a file holding
+            // `"filesystem": {}` would otherwise print the string "undefined" as this
+            // routine's position in `op=gate-read` and in `positions=`.
+            return marks.filesystem?.bytes === undefined ? null : String(marks.filesystem.bytes);
     }
 }
 /**
@@ -299,6 +356,12 @@ export function slotsIn(marks) {
         slots.push({ kind: "calendar" });
     if (marks.graph?.updatedAt !== undefined)
         slots.push({ kind: "graph" });
+    for (const key of Object.keys(marks.connector ?? {}))
+        slots.push({ kind: "connector", key });
+    if (marks.sweep?.digest !== undefined)
+        slots.push({ kind: "sweep" });
+    if (marks.filesystem?.bytes !== undefined)
+        slots.push({ kind: "filesystem" });
     return slots;
 }
 /**
@@ -409,6 +472,107 @@ export function promoteSlot(slot, from, into) {
             into.graph = { ...(into.graph ?? {}), updatedAt };
             return true;
         }
+        case "connector": {
+            const entry = from.connector?.[slot.key];
+            if (entry === undefined)
+                return false;
+            into.connector = { ...(into.connector ?? {}), [slot.key]: entry };
+            return true;
+        }
+        case "sweep": {
+            const digest = from.sweep?.digest;
+            if (digest === undefined)
+                return false;
+            into.sweep = { ...(into.sweep ?? {}), digest };
+            return true;
+        }
+        case "filesystem": {
+            const bytes = from.filesystem?.bytes;
+            if (bytes === undefined)
+                return false;
+            into.filesystem = { ...(into.filesystem ?? {}), bytes };
+            return true;
+        }
+    }
+}
+/**
+ * Task 2689 — the memory sweep's state as one comparable value.
+ *
+ * The two review-queue counts and a staleness flag, never the newest report's
+ * identity. The sweep runs hourly and `graph-health` is proposed weekly, so a
+ * digest carrying the report id would differ on essentially every fire and the
+ * gate could never go quiet. A digest over the queue counts alone would go
+ * quiet correctly, but a sweep that STOPPED running moves no queue, so the
+ * routine could never report the one failure it exists to catch. The flag gives
+ * both: a constant value while the sweep is healthy, one change at the
+ * transition into staleness, and one more on the way out.
+ *
+ * `now` is a parameter rather than a `Date.now()` read inside, so the boundary
+ * is testable without moving the clock. A `newestReportAt` this cannot parse
+ * throws, through `instantOf`, and a throw counts as work.
+ */
+export function digestSweepState(state, now) {
+    const stale = state.newestReportAt === null || now - instantOf(state.newestReportAt) > SWEEP_STALE_AFTER_MS;
+    return createHash("sha256").update(`${state.orphans}|${state.citations}|${stale}`).digest("hex");
+}
+/**
+ * Task 2689 — total bytes of the regular files under `root`.
+ *
+ * A symlink is never followed, so the walk cannot leave the account directory
+ * even when something inside points out of it — the rule the filesystem
+ * plugin's own walker holds to, and the reason this does not shell out to `du`.
+ *
+ * A vanished entry (`ENOENT` on a stat, or on a nested directory read) is
+ * SKIPPED: an account directory is written to continuously, and a file deleted
+ * between the read and the stat contributes nothing to a size. Every other
+ * error propagates. The ROOT is read outside that rule, so a missing account
+ * directory throws: a wrongly derived root that read zero would answer "nothing
+ * grew" for ever, which is the permanent mute this design forbids.
+ */
+export function measureAccountBytes(root, cap) {
+    let bytes = 0;
+    // Every entry the walk touches, not only the files. Counting files alone
+    // would leave the directories unbounded, and the pending stack grows by one
+    // per directory: a tree that is wide in directories and thin in files would
+    // then walk all of it with the cap never reached.
+    let seen = 0;
+    const pending = [];
+    let dir = root;
+    let entries = readdirSync(root, { withFileTypes: true });
+    for (;;) {
+        for (const entry of entries) {
+            if (entry.isSymbolicLink())
+                continue;
+            if (seen >= cap)
+                return { bytes, capped: true };
+            seen++;
+            const full = join(dir, entry.name);
+            if (entry.isDirectory()) {
+                pending.push(full);
+                continue;
+            }
+            if (!entry.isFile())
+                continue;
+            try {
+                bytes += statSync(full).size;
+            }
+            catch (err) {
+                if (err.code !== "ENOENT")
+                    throw err;
+            }
+        }
+        const next = pending.pop();
+        if (next === undefined)
+            return { bytes, capped: false };
+        dir = next;
+        try {
+            entries = readdirSync(dir, { withFileTypes: true });
+        }
+        catch (err) {
+            if (err.code !== "ENOENT")
+                throw err;
+            entries = [];
+        }
     }
 }
 // ---------------------------------------------------------------------------
@@ -623,6 +787,30 @@ const realMail = async (check, sinceUid, accountId) => {
         }
     }
 };
+/**
+ * Task 2688 — one call to the connector plugin's own tools.
+ *
+ * A spawn rather than a loopback route or a direct read of the registry files,
+ * because the credential, the host binding and the SSRF guard all live inside
+ * that plugin, and reimplementing any of them here would put a second copy of a
+ * security control in a second package. It is the transport the Outlook mail
+ * probe already uses from this same heartbeat: same timeout, same ACCOUNT_ID
+ * scoping, same captured child stderr, same PLATFORM_PORT bridge.
+ *
+ * The import is dynamic because `lib/gate.ts` imports this module's types; a
+ * static value import would close that loop at run time.
+ */
+async function callConnectorTool(accountId, tool, args) {
+    const { callPluginTool } = await import("../lib/gate.js");
+    const call = await callPluginTool("connector", tool, JSON.stringify(args), accountId);
+    const text = (Array.isArray(call.result.content) ? call.result.content : [])
+        .filter((c) => c.type === "text")
+        .map((c) => String(c.text))
+        .join("\n");
+    if (call.result.isError === true)
+        throw new Error(`${tool} returned an error: ${text}`);
+    return JSON.parse(text);
+}
 export const REAL_SOURCES = {
     mail: realMail,
     // Task 2657 — the provider owns the decryption of its own tokens, so this is
@@ -796,6 +984,122 @@ export const REAL_SOURCES = {
             await session.close();
         }
     },
+    connectorQueues: async (accountId, kind) => {
+        // The connector plugin is optional. Its absence is an ANSWER (nothing is
+        // registered, so nothing registered can have moved), so it is detected
+        // before any spawn rather than by catching the transport error, which is
+        // indistinguishable from a child that crashed on boot.
+        const serverPath = resolve(process.env.PLATFORM_ROOT, "plugins", "connector", "mcp", "dist", "index.js");
+        if (!existsSync(serverPath))
+            return { pluginPresent: false, bindings: [] };
+        const payload = await callConnectorTool(accountId, "connector-list", {});
+        if (!Array.isArray(payload)) {
+            throw new Error(`connector-list returned ${typeof payload}, expected an array`);
+        }
+        const bindings = [];
+        for (const entry of payload) {
+            const e = entry;
+            const q = e.queue;
+            if (typeof e.name !== "string" || !q || q.kind !== kind)
+                continue;
+            // A descriptor missing a field it needs THROWS rather than being called
+            // with `undefined` in the URL: that call would 404, the 404 would count as
+            // work, and the routine would wake on every fire with nothing to show for
+            // it. The throw names the connector once and counts as work once.
+            if (typeof q.path !== "string" ||
+                typeof q.changedSinceParam !== "string" ||
+                typeof q.itemsField !== "string") {
+                throw new Error(`connector ${e.name} carries an incomplete queue descriptor`);
+            }
+            bindings.push({ name: e.name, queue: q });
+        }
+        return { pluginPresent: true, bindings };
+    },
+    connectorQueueRead: async (accountId, binding, changedSince) => {
+        const payload = await callConnectorTool(accountId, "connector-call", {
+            name: binding.name,
+            method: "GET",
+            path: binding.queue.path,
+            query: { [binding.queue.changedSinceParam]: changedSince },
+        });
+        const answer = payload;
+        if (typeof answer.status !== "number") {
+            throw new Error(`connector-call returned no status for ${binding.name}`);
+        }
+        // Every non-2xx is handed back with its status and no count. The probe
+        // decides which of them is an answer (401 and 403) and which is a failure,
+        // so that rule lives in one place rather than being split across the source
+        // and its caller.
+        if (answer.status < 200 || answer.status >= 300) {
+            return { status: answer.status, items: 0, truncated: false };
+        }
+        // A truncated body cannot be counted — the array it carries is a prefix of
+        // the real one, and the cut usually lands mid-token, so it will not even
+        // parse. It is reported rather than thrown, because the cap is itself proof
+        // the queue moved: the probe counts it as work and records the position, and
+        // a throw would record none, leaving the same window to truncate again on
+        // every fire for ever.
+        if (answer.truncated === true)
+            return { status: answer.status, items: 0, truncated: true };
+        const parsed = typeof answer.body === "string" ? JSON.parse(answer.body) : answer.body;
+        const items = parsed?.[binding.queue.itemsField];
+        // A shape this cannot read throws, and a throw counts as work. Coercing it
+        // to zero would answer "quiet" about a queue this never understood, which
+        // is the one answer that loses a record.
+        if (!Array.isArray(items)) {
+            throw new Error(`connector ${binding.name} queue field ${binding.queue.itemsField} is not an array`);
+        }
+        return { status: answer.status, items: items.length, truncated: false };
+    },
+    sweepState: async (accountId) => {
+        // Three reads on this install's own Neo4j, the way `calendarDigest` reads
+        // `:ConnectorBusy`: the nodes are local and there is no cross-plugin call.
+        // `dream-cycle` is the keyword `memory-dream-run` puts on every report it
+        // writes, and it is the same keyword `graph-health`'s own prompt reads
+        // through `memory-report-read-latest`.
+        const session = getSession();
+        try {
+            const reportRes = await session.run(`MATCH (r:Report)
+         WHERE r.accountId = $accountId AND 'dream-cycle' IN r.keywords
+         RETURN toString(r.occurredAt) AS at
+         ORDER BY r.occurredAt DESC LIMIT 1`, { accountId });
+            const at = reportRes.records[0]?.get("at") ?? null;
+            // An account whose sweep has never run holds no report, which reads as
+            // stale. A value this cannot read THROWS rather than coercing to null:
+            // a throw counts as work, while coercing would report a sweep age this
+            // never established.
+            if (at !== null && typeof at !== "string") {
+                throw new Error(`sweep report occurredAt is type ${typeof at}, expected an ISO string`);
+            }
+            // The labels are literals in this file, never operator input, so
+            // interpolating them cannot inject; the account is a parameter.
+            const count = async (label) => {
+                const res = await session.run(`MATCH (n:${label}) WHERE n.accountId = $accountId RETURN count(n) AS c`, { accountId });
+                // neo4j returns a Cypher Integer. `Number` on a shape this cannot read
+                // is NaN, which would digest as the string "NaN" and then never change
+                // again — a gate that has gone quiet for the wrong reason.
+                const n = Number(res.records[0]?.get("c") ?? 0);
+                if (!Number.isFinite(n))
+                    throw new Error(`${label} count is not a number`);
+                return n;
+            };
+            return {
+                newestReportAt: at,
+                orphans: await count("OrphanCandidate"),
+                citations: await count("CitationProposal"),
+            };
+        }
+        finally {
+            await session.close();
+        }
+    },
+    filesystemBytes: async (accountId) => {
+        // `$PLATFORM_ROOT/../data/accounts` is the derivation `dispatcher-env.ts`
+        // and `agent-dispatch.ts` already use. Re-deriving it differently here is
+        // how the gate would come to measure a directory the routine never reads.
+        const root = resolve(process.env.PLATFORM_ROOT, "..", "data/accounts", accountId);
+        return measureAccountBytes(root, FS_WALK_ENTRY_CAP);
+    },
     calendarDigest: async (check, accountId) => {
         const session = getSession();
         try {
@@ -860,6 +1164,18 @@ export function checkSource(check) {
         // going unreadable is not the other.
         case "graph-unflagged":
             return `graph:${check.label}`;
+        // Task 2688 — the scope rather than a connector name: which connectors the
+        // scope resolves to is a property of the account and can change between
+        // fires, so a set built from a name would read as a new outage whenever the
+        // operator registered another one.
+        case "connector":
+            return `connector:${check.scope}`;
+        // Task 2689 — one sweep and one account directory per account, so each
+        // names itself, the way `whatsapp` and `graph` do.
+        case "sweep":
+            return "sweep";
+        case "filesystem":
+            return "filesystem";
         default:
             return String(check.type);
     }
@@ -1033,6 +1349,124 @@ flagKey) {
             const { ids } = await sources.graphUnflagged(accountId, label, flagKey, UNFLAGGED_ID_CAP);
             return { hasWork: ids.length > 0, mark: null, unflagged: { label, ids } };
         }
+        case "connector": {
+            // The instant this arm STARTED, recorded as the new position of every
+            // connector that answers. Taken once, before the first call, so a slow
+            // queue cannot advance its own mark past records that arrived while it
+            // was answering.
+            const startedAt = new Date().toISOString();
+            const { pluginPresent, bindings } = await sources.connectorQueues(accountId, check.scope);
+            // Two quiet answers, each with its own reason. Neither is an error: an
+            // install with no connector plugin, and an account that has described no
+            // queue of this kind, both mean nothing registered here can have moved.
+            // The line is load-bearing — a routine that will never fire has to be
+            // visible somewhere, and `op=checked` alone reads exactly like a healthy
+            // quiet fire.
+            if (!pluginPresent) {
+                console.error(`[scheduling-gate] op=connector-quiet reason=no-plugin scope=${check.scope}`);
+                return { hasWork: false, mark: null };
+            }
+            if (bindings.length === 0) {
+                console.error(`[scheduling-gate] op=connector-quiet reason=no-queue scope=${check.scope}`);
+                return { hasWork: false, mark: null };
+            }
+            let hasWork = false;
+            const observed = [];
+            for (const binding of bindings) {
+                const record = () => observed.push({
+                    slot: { kind: "connector", key: binding.name },
+                    write: (m) => {
+                        m.connector = { ...(m.connector ?? {}), [binding.name]: { changedSince: startedAt } };
+                    },
+                });
+                const since = marks.connector?.[binding.name]?.changedSince;
+                if (since === undefined) {
+                    // A first run: everything already queued counts as work, the same as
+                    // every other watermarked check. No call is made, because a read with
+                    // no changed-since filter is precisely the response `connector-call`
+                    // caps, and a capped body cannot be counted.
+                    console.error(`[scheduling-gate] op=connector-first-run connector=${binding.name}`);
+                    hasWork = true;
+                    record();
+                    continue;
+                }
+                const res = await sources.connectorQueueRead(accountId, binding, since);
+                // An unauthorised connector is an ANSWER: it reports no work, records no
+                // position, and says so on its own line. No position, because a
+                // connector this gate could not read must not be advanced past — the
+                // records that arrived while the credential was dead are still unseen.
+                if (res.status === 401 || res.status === 403) {
+                    console.error(`[scheduling-gate] op=connector-quiet reason=unauthorised ` +
+                        `connector=${binding.name} status=${res.status}`);
+                    continue;
+                }
+                // Every other non-2xx throws, which counts as work and fires. A 500 or a
+                // 404 is a source this gate could not read at all, and answering quiet
+                // about it would mute the routine for the length of an outage.
+                if (res.status < 200 || res.status >= 300) {
+                    throw new Error(`connector ${binding.name} queue answered ${res.status}`);
+                }
+                console.error(`[scheduling-gate] op=connector-probe connector=${binding.name} ` +
+                    `status=${res.status} items=${res.items} truncated=${res.truncated}`);
+                // A capped body is proof of movement that cannot be counted, so it reads
+                // as work and still records its position: recording none would leave the
+                // same window in place, it would truncate again, and the routine would
+                // wake the model on every fire with no path back to quiet.
+                if (res.items > 0 || res.truncated)
+                    hasWork = true;
+                record();
+            }
+            return { hasWork, mark: observed.length > 0 ? observed : null };
+        }
+        case "sweep": {
+            const state = await sources.sweepState(accountId);
+            const digest = digestSweepState(state, Date.now());
+            const stored = marks.sweep?.digest;
+            return {
+                // An absent digest is a first run, which reports work exactly as every
+                // other stored-position check does on its first fire.
+                hasWork: stored !== digest,
+                mark: {
+                    slot: { kind: "sweep" },
+                    write: (m) => {
+                        m.sweep = { ...(m.sweep ?? {}), digest };
+                    },
+                },
+            };
+        }
+        case "filesystem": {
+            const minGrowth = check.minGrowthBytes ?? FS_DEFAULT_MIN_GROWTH_BYTES;
+            // Refused HERE, before the source is reached, exactly as the mail arm
+            // refuses an unknown provider: the throw lands inside
+            // `runSchedulingGate`'s try, where it counts as work and names itself.
+            // Zero would make every fire pass, because growth is never below zero,
+            // and a gate that can never go quiet is the class this design removes.
+            if (!Number.isInteger(minGrowth) || minGrowth <= 0) {
+                throw new Error(`minGrowthBytes must be a positive integer, got ${JSON.stringify(check.minGrowthBytes)}`);
+            }
+            const { bytes, capped } = await sources.filesystemBytes(accountId);
+            if (capped) {
+                // A floor, not a total, so it records NO position: promoting a floor
+                // would make the next fire measure growth from a number the directory
+                // never held, and the error would be in the muting direction.
+                console.error(`[scheduling-gate] op=fs-measure capped=true bytes=${bytes} treatedAs=has-work`);
+                return { hasWork: true, mark: null };
+            }
+            const stored = marks.filesystem?.bytes;
+            return {
+                // An absent position is a first run: whatever is already there counts
+                // as work, the way every other watermarked check reads its first fire.
+                // A directory that SHRANK reports no work — growth is the question, and
+                // the stored value stays put until an accepted wake moves it.
+                hasWork: stored === undefined || bytes - stored >= minGrowth,
+                mark: {
+                    slot: { kind: "filesystem" },
+                    write: (m) => {
+                        m.filesystem = { ...(m.filesystem ?? {}), bytes };
+                    },
+                },
+            };
+        }
         // Task 2653 — a type outside the union. The switch was exhaustive over
         // `GateCheck` and had no default, so `probe` returned `undefined` and the
         // caller's `outcome.hasWork` read threw OUTSIDE its own try: the TypeError
@@ -1105,18 +1539,29 @@ export async function runSchedulingGate(params, accountId, platformRoot, sources
         }
         // A check that threw took the `continue` above, so it records no position:
         // its state is unknown, not absent.
-        if (outcome.mark) {
+        //
+        // Task 2688 — one check may hand back several positions (one per registered
+        // connector), so the single mark is normalised to a list and each is logged
+        // on its own line.
+        const observedMarks = outcome.mark === null ? [] : Array.isArray(outcome.mark) ? outcome.mark : [outcome.mark];
+        for (const mark of observedMarks) {
             // Every probe records what it saw, whatever the verdict: the position is a
             // fact about the source.
-            outcome.mark.write(fresh);
+            mark.write(fresh);
             // Task 2663 — BOTH sides of the comparison. This line printed only
             // `position=`, which was `fresh` — this probe's own reading — so the value
             // the verdict was actually made against appeared in no log line anywhere,
             // and a gate stuck for ever on an empty slot read exactly like a healthy
             // one. Measured on 2026-08-10: 18 wakes whose `position=` never moved.
-            console.error(`[scheduling-gate] op=gate-read eventId=${id8} check=${check.type} ` +
-                `stored=${slotPosition(outcome.mark.slot, marks) ?? "null"} ` +
-                `observed=${slotPosition(outcome.mark.slot, fresh) ?? "null"}`);
+            console.error(
+            // Task 2688 — `slot=` is appended rather than inserted, because one
+            // check can now print several of these lines and only the slot key
+            // tells them apart. It goes last so the fields every existing reader
+            // greps for keep their positions.
+            `[scheduling-gate] op=gate-read eventId=${id8} check=${check.type} ` +
+                `stored=${slotPosition(mark.slot, marks) ?? "null"} ` +
+                `observed=${slotPosition(mark.slot, fresh) ?? "null"} ` +
+                `slot=${slotKey(mark.slot)}`);
         }
         // Task 2684 — an empty set carries nothing to flag, so it contributes no
         // entry: the dispatcher's `op=gate-flag` then names only labels that