@reticlehq/server 3.2.0 → 3.4.0

package/dist/memory/cloud/sync-cycle.js

@@ -30,7 +30,7 @@
  * Every dependency is injected — the clock, the fetch, the reads and writes — because this is the
  * one piece of Reticle that talks to a network AND to a disk, and it must be provable without either.
  */
-import { ReticleDir } from '@reticlehq/core';
+import { FlowErrorCode, ReticleDir, SYNC_BATCH_LIMITS } from '@reticlehq/core';
 import { hashPayload } from './sync-hash.js';
 /** The server's own doors. Kept beside the code that calls them, like the other cloud paths. */
 const SYNC_PATH = '/v1/sync';
@@ -46,6 +46,10 @@ const DERIVED_RECORDS = [
     { kind: 'impact', file: ReticleDir.IMPACT_FILE },
     { kind: 'flake', file: ReticleDir.FLAKE_FILE },
     { kind: 'intent', file: ReticleDir.INTENT_FILE },
+    // How each page normally behaves, and how strong each flow's checks are. Kept only here, they
+    // cannot tell a server a page that drifted from one that always behaved that way.
+    { kind: 'envelopes', file: ReticleDir.ENVELOPES_FILE },
+    { kind: 'assertion-tiers', file: ReticleDir.TIERS_FILE },
 ];
 const asJson = (text) => {
     try {
@@ -56,6 +60,57 @@ const asJson = (text) => {
     }
 };
 const isRecord = (v) => 'object' === typeof v && null !== v;
+/** A list of numbers, or undefined when the server did not send a list at all. */
+const numbersIn = (v) => Array.isArray(v) ? v.filter((n) => 'number' === typeof n) : undefined;
+/** A numeric field of an artifact, or undefined when it has none. */
+const numberAt = (v, field) => {
+    const n = isRecord(v) ? v[field] : undefined;
+    return 'number' === typeof n ? n : undefined;
+};
+/** The name a person knows a flow by, falling back to its position. */
+const flowName = (flow, index) => {
+    const name = isRecord(flow) ? flow['name'] : undefined;
+    return 'string' === typeof name && name.length > 0 ? name : `flow #${String(index)}`;
+};
+const FLOW_VERSION_FIX = 'update the platform or remove `compare` from the flow';
+const RUN_VERSION_FIX = 'update the platform';
+/** The server's per-part answer: accepted count and rejection list, each read defensively. */
+function partResult(body, part) {
+    const raw = isRecord(body[part]) ? body[part] : {};
+    const accepted = 'number' === typeof raw['accepted'] ? raw['accepted'] : 0;
+    const rejected = (Array.isArray(raw['rejected']) ? raw['rejected'] : [])
+        .filter(isRecord)
+        .map((r) => ({
+        index: 'number' === typeof r['index'] ? r['index'] : -1,
+        reason: 'string' === typeof r['reason'] ? r['reason'] : 'no reason given',
+        code: r['code'],
+        version: r['version'],
+    }));
+    return { accepted, rejected };
+}
+/**
+ * Split runs into requests bounded by count and by serialized size. A run bigger than the byte
+ * bound on its own still goes, alone: it cannot be split, and holding it back would hide it.
+ */
+function batchRuns(runs) {
+    const batches = [];
+    let current = [];
+    let bytes = 0;
+    for (const run of runs) {
+        const size = Buffer.byteLength(JSON.stringify(run.payload));
+        const full = current.length >= SYNC_BATCH_LIMITS.MAX_RUNS || bytes + size > SYNC_BATCH_LIMITS.MAX_BYTES;
+        if (current.length > 0 && full) {
+            batches.push(current);
+            current = [];
+            bytes = 0;
+        }
+        current.push(run);
+        bytes += size;
+    }
+    if (current.length > 0)
+        batches.push(current);
+    return batches;
+}
 /**
  * Run one cycle. Never throws: the local record is already safe on disk, and a sync that can take
  * down the thing it is backing up is worse than no sync.
@@ -68,6 +123,10 @@ export async function runSyncCycle(deps) {
         flowsSent: 0,
         capsulesSent: 0,
         derivedSent: [],
+        refused: [],
+        held: [],
+        notRetried: [],
+        setsNotRetried: [],
         pulled: 0,
         morePending: false,
     };
@@ -99,66 +158,237 @@ export async function runSyncCycle(deps) {
          * something the server does not have. Silent data loss is the one failure this protocol must
          * not have, and an unreadable answer has to mean "it knows nothing", never "it knows this".
          */
-        const known = new Set(Array.isArray(held.knownRunIds)
-            ? held.knownRunIds.filter((id) => 'string' === typeof id)
-            : []);
+        const onlyStrings = (value) => Array.isArray(value) ? value.filter((id) => 'string' === typeof id) : [];
+        /*
+         * The server may answer "here are the ids I hold, and there are more than I listed". Read as the
+         * whole truth, that short list made every run past the server's page look unsent, so a project
+         * with enough runs to truncate it re-uploaded its history on every single cycle, forever. The
+         * field was declared on this response and read nowhere.
+         *
+         * Only then is the machine's own record consulted, and it is subject to the same rule as the
+         * server's: anything that is not a list of strings reads as "nothing is known".
+         */
+        const known = new Set([
+            ...onlyStrings(held.knownRunIds),
+            ...(true === held.truncated ? onlyStrings(deps.state.sentRunIds) : []),
+        ]);
         // Same rule for the hashes: only a string can equal a hash we computed, so anything else reads
         // as "unknown" and the record is sent once. Re-sending costs a request; skipping costs the data.
         const hashes = isRecord(held.stateHashes) ? held.stateHashes : {};
-        // 2. SEND — only what the server does not already have.
+        /*
+         * What this platform reads. `accepts` says so outright; a platform that predates it still lists
+         * every record kind it knows in `stateHashes`, null when it holds nothing. A kind it does not
+         * list is one it does not know, and it will never answer with a hash for it: sending on "hash
+         * differs" then re-sent that record, and every flow and capsule riding with it, on every cycle.
+         */
+        const accepts = isRecord(held.accepts) ? held.accepts : {};
+        const acceptedKinds = new Set(Array.isArray(accepts['derived']) ? onlyStrings(accepts['derived']) : Object.keys(hashes));
+        const flowVersions = numbersIn(accepts['flowVersions']);
+        const runVersions = numbersIn(accepts['runVersions']);
+        const heldBack = [];
+        /** Held runs and flows mean the dashboard is missing something; unknown record kinds do not. */
+        let heldArtifacts = false;
+        // 2. SEND — only what the server does not already have, and only what it can read.
         const allRuns = deps.source.runs();
-        const unsent = allRuns.filter((r) => !known.has(r.runId));
-        const derivedSent = [];
+        const acceptsHash = hashPayload(held.accepts ?? null);
+        const priorRefusals = isRecord(deps.state.refusedRuns) ? deps.state.refusedRuns : {};
+        const refusedRuns = {};
+        const notRetried = [];
+        const unsent = allRuns.filter((run) => {
+            if (known.has(run.runId))
+                return false;
+            const prior = priorRefusals[run.runId];
+            const unchanged = prior !== undefined &&
+                'string' === typeof prior.reason &&
+                prior.acceptsHash === acceptsHash &&
+                prior.payloadHash === hashPayload(run.payload);
+            if (!unchanged)
+                return true;
+            refusedRuns[run.runId] = prior;
+            notRetried.push({ runId: run.runId, reason: prior.reason });
+            return false;
+        });
+        const sendable = unsent.filter((run) => {
+            const version = numberAt(run.payload, 'schemaVersion');
+            if (runVersions === undefined || version === undefined || runVersions.includes(version))
+                return true;
+            heldBack.push(`this platform reads run versions ${runVersions.join(', ')}; run ${run.runId} is version ${String(version)} — ${RUN_VERSION_FIX}`);
+            heldArtifacts = true;
+            return false;
+        });
+        const derivedOffered = [];
+        const unsupported = [];
         const bundle = {};
-        if (unsent.length > 0)
-            bundle['runs'] = unsent.map((r) => r.payload);
         for (const { kind } of DERIVED_RECORDS) {
             const payload = deps.source.derived(kind);
             if (payload === undefined)
                 continue;
+            if (!acceptedKinds.has(kind)) {
+                unsupported.push(kind);
+                continue;
+            }
             // The one comparison the whole protocol rests on. Same hash, do not send it.
             if (hashPayload(payload) === hashes[kind])
                 continue;
             bundle[kind] = payload;
-            derivedSent.push(kind);
+            derivedOffered.push(kind);
         }
-        // Flows are small and upserted by name, so they ride along whenever anything else does rather
-        // than earning a round trip of their own.
-        const ridesAlong = bundle['runs'] !== undefined || derivedSent.length > 0;
-        const flows = ridesAlong ? deps.source.flows() : [];
-        if (flows.length > 0)
+        if (unsupported.length > 0)
+            heldBack.push(`this platform does not accept ${unsupported.join(', ')} yet`);
+        // Flows and capsules are upserted by name/id, so the whole set goes whenever it differs from the
+        // set the platform last answered for, and rides along with anything else being sent.
+        const flows = deps.source.flows().filter((flow, index) => {
+            const version = numberAt(flow, 'version');
+            if (flowVersions === undefined || version === undefined || flowVersions.includes(version))
+                return true;
+            heldBack.push(`this platform reads flow versions ${flowVersions.join(', ')}; ${flowName(flow, index)} is version ${String(version)} — ${FLOW_VERSION_FIX}`);
+            heldArtifacts = true;
+            return false;
+        });
+        const capsulesAll = deps.source.capsules();
+        const flowsHash = hashPayload(flows);
+        const capsulesHash = hashPayload(capsulesAll);
+        const priorSets = isRecord(deps.state.refusedSets) ? deps.state.refusedSets : {};
+        const setsNotRetried = [];
+        const refusedSets = {};
+        /** A set is due when it changed, or when it was refused and the platform now reads more. */
+        const due = (part, hash, size, sentHash) => {
+            if (0 === size)
+                return false;
+            const prior = priorSets[part];
+            if (prior !== undefined && prior.hash === hash) {
+                if (prior.acceptsHash !== acceptsHash)
+                    return true;
+                refusedSets[part] = prior;
+                setsNotRetried.push({ part, count: prior.count, reason: prior.reason });
+                return false;
+            }
+            return hash !== sentHash;
+        };
+        const flowsChanged = due('flow', flowsHash, flows.length, deps.state.sentFlowsHash);
+        const capsulesChanged = due('capsule', capsulesHash, capsulesAll.length, deps.state.sentCapsulesHash);
+        const ridesAlong = sendable.length > 0 || derivedOffered.length > 0 || flowsChanged || capsulesChanged;
+        if (ridesAlong && flows.length > 0)
             bundle['flows'] = flows;
-        // Same gate as flows: only when something else is already going. A cycle that sent capsules and
-        // nothing else would wake the server on every tick of an idle machine.
-        const capsules = ridesAlong ? deps.source.capsules() : [];
+        const capsules = ridesAlong ? capsulesAll : [];
         if (capsules.length > 0)
             bundle['capsules'] = capsules;
         let runsSent = 0;
         let flowsSent = 0;
         let capsulesSent = 0;
-        let runsRejected = [];
-        if (Object.keys(bundle).length > 0) {
-            const pushed = await call(SYNC_PATH, { method: 'POST', body: JSON.stringify(bundle) });
+        const runsRejected = [];
+        const refused = [];
+        const derivedSent = [];
+        let pushError;
+        /*
+         * Runs go up in bounded batches; everything else rides in the first. One request used to carry
+         * the whole backlog, and a backlog over the platform's body limit was refused whole, forever.
+         */
+        const batches = batchRuns(sendable);
+        const requests = Object.keys(bundle).length > 0 || batches.length > 0 ? Math.max(1, batches.length) : 0;
+        let offset = 0;
+        for (let i = 0; i < requests; i += 1) {
+            const batch = batches[i] ?? [];
+            const body = 0 === i ? { ...bundle } : {};
+            if (batch.length > 0)
+                body['runs'] = batch.map((r) => r.payload);
+            const pushed = await call(SYNC_PATH, { method: 'POST', body: JSON.stringify(body) });
             if (200 !== pushed.status) {
-                const error = `sync ${String(pushed.status)}: ${pushed.text.slice(0, 200)}`;
-                deps.sink.writeState({ ...nextState, lastError: error });
-                return { ...empty, error };
+                pushError = `sync ${String(pushed.status)}: ${pushed.text.slice(0, 200)}`;
+                break;
+            }
+            const answer = isRecord(pushed.json) ? pushed.json : {};
+            const runs = partResult(answer, 'runs');
+            runsSent += runs.accepted;
+            // Accepted means the server has it. A rejected run was refused by index, so it is exactly as
+            // unsent as it was before and must never be remembered as delivered.
+            const refusedHere = new Map(runs.rejected.map((r) => [r.index, r.reason]));
+            batch.forEach((run, index) => {
+                const reason = refusedHere.get(index);
+                if (reason === undefined)
+                    known.add(run.runId);
+                else
+                    refusedRuns[run.runId] = { reason, payloadHash: hashPayload(run.payload), acceptsHash };
+            });
+            runsRejected.push(...runs.rejected.map((r) => ({ index: r.index + offset, reason: r.reason })));
+            offset += batch.length;
+            if (0 !== i)
+                continue;
+            // Absent means the server said nothing about a part — an older one that does not know the
+            // field. Read as zero accepted, never as an error: an added field must not break a sync.
+            const flowsPart = partResult(answer, 'flows');
+            const capsulesPart = partResult(answer, 'capsules');
+            flowsSent = flowsPart.accepted;
+            capsulesSent = capsulesPart.accepted;
+            for (const r of flowsPart.rejected) {
+                const name = flowName(flows[r.index], r.index);
+                refused.push({
+                    part: 'flow',
+                    reason: FlowErrorCode.WRONG_VERSION === r.code && 'number' === typeof r.version
+                        ? `${name} is version ${String(r.version)} — ${FLOW_VERSION_FIX}`
+                        : `flow ${name}: ${r.reason}`,
+                });
+            }
+            for (const r of capsulesPart.rejected)
+                refused.push({ part: 'capsule', reason: `capsule #${String(r.index)}: ${r.reason}` });
+            const firstOf = (part) => refused.find((r) => r.part === part)?.reason ?? 'no reason given';
+            if (flowsPart.rejected.length > 0)
+                refusedSets.flow = {
+                    hash: flowsHash,
+                    acceptsHash,
+                    count: flowsPart.rejected.length,
+                    reason: firstOf('flow'),
+                };
+            if (capsulesPart.rejected.length > 0)
+                refusedSets.capsule = {
+                    hash: capsulesHash,
+                    acceptsHash,
+                    count: capsulesPart.rejected.length,
+                    reason: firstOf('capsule'),
+                };
+            // `state` has a key per record sent: "accepted", or the message that refused it.
+            const state = isRecord(answer['state']) ? answer['state'] : {};
+            for (const kind of derivedOffered) {
+                const verdict = state[kind];
+                if ('accepted' === verdict)
+                    derivedSent.push(kind);
+                else if ('string' === typeof verdict)
+                    refused.push({ part: kind, reason: `${kind}: ${verdict}` });
             }
-            const body = isRecord(pushed.json) ? pushed.json : {};
-            const runs = isRecord(body['runs']) ? body['runs'] : {};
-            const flowsPart = isRecord(body['flows']) ? body['flows'] : {};
-            // Absent means the server said nothing about capsules — an older one that does not know the
-            // field. Read as zero accepted, never as an error: an added field must not break a sync that
-            // was otherwise fine, and the push's own status code is what reports a real refusal.
-            const capsulesPart = isRecord(body['capsules']) ? body['capsules'] : {};
-            runsSent = 'number' === typeof runs['accepted'] ? runs['accepted'] : 0;
-            flowsSent = 'number' === typeof flowsPart['accepted'] ? flowsPart['accepted'] : 0;
-            capsulesSent = 'number' === typeof capsulesPart['accepted'] ? capsulesPart['accepted'] : 0;
-            runsRejected = Array.isArray(runs['rejected'])
-                ? runs['rejected']
-                : [];
+        }
+        if (requests > 0 && pushError === undefined) {
             nextState.lastPushAt = deps.now();
+            // The platform answered for these sets, refusals included (reported above, and not re-offered
+            // until the set changes — the same rule as a refused run).
+            if (ridesAlong && flows.length > 0)
+                nextState.sentFlowsHash = flowsHash;
+            if (capsules.length > 0)
+                nextState.sentCapsulesHash = capsulesHash;
         }
+        nextState.refusedRuns = refusedRuns;
+        nextState.refusedSets = refusedSets;
+        if (pushError !== undefined) {
+            nextState.sentRunIds = allRuns.map((r) => r.runId).filter((id) => known.has(id));
+            deps.sink.writeState({ ...nextState, lastError: pushError });
+            return {
+                ...empty,
+                runsSent,
+                runsRejected,
+                flowsSent,
+                capsulesSent,
+                derivedSent,
+                refused,
+                held: heldBack,
+                notRetried,
+                error: pushError,
+            };
+        }
+        /*
+         * Bounded by the runs still on disk rather than by a cap. Retention already decides how many
+         * runs a workspace keeps, and an id for a run nobody holds any more cannot stop an upload that
+         * will never be attempted — so the record follows the artifacts and needs no number of its own.
+         */
+        nextState.sentRunIds = allRuns.map((r) => r.runId).filter((id) => known.has(id));
         // 3. COLLECT — always, even when there was nothing to send.
         const query = nextState.cursor === undefined ? '' : `?since=${encodeURIComponent(nextState.cursor)}`;
         const pull = await call(`${SYNC_PULL_PATH}${query}`);
@@ -174,6 +404,9 @@ export async function runSyncCycle(deps) {
                 capsulesSent,
                 derivedSent,
                 runsRejected,
+                refused,
+                held: heldBack,
+                notRetried,
                 error,
             };
         }
@@ -196,15 +429,22 @@ export async function runSyncCycle(deps) {
         if ('string' === typeof pulled.cursor)
             nextState.cursor = pulled.cursor;
         nextState.lastPullAt = deps.now();
-        delete nextState.lastError;
-        deps.sink.writeState(nextState);
-        return {
-            ok: true,
+        const report = {
+            // Anything refused, or any artifact held back, means the dashboard is missing something.
+            ok: 0 === runsRejected.length &&
+                0 === refused.length &&
+                0 === notRetried.length &&
+                0 === setsNotRetried.length &&
+                !heldArtifacts,
             runsSent,
             runsRejected,
             flowsSent,
             capsulesSent,
             derivedSent,
+            refused,
+            held: heldBack,
+            notRetried,
+            setsNotRetried,
             pulled: decisions.length,
             morePending: true === pulled.more,
             /*
@@ -216,6 +456,16 @@ export async function runSyncCycle(deps) {
                 0 === deps.source.flows().length &&
                 DERIVED_RECORDS.every(({ kind }) => deps.source.derived(kind) === undefined),
         };
+        /*
+         * A refusal is kept as the last error rather than cleared by a completed exchange: it is what
+         * `reticle whoami` shows, and "the transfer worked" is not "the dashboard has it".
+         */
+        if (report.ok)
+            delete nextState.lastError;
+        else
+            nextState.lastError = problemsOf(report).join('; ');
+        deps.sink.writeState(nextState);
+        return report;
     }
     catch (error) {
         // A network that is down is not an error condition for a local-first tool; it is Tuesday.
@@ -268,9 +518,10 @@ export function describeSync(report) {
      * to send, 3 rejected" — something was very much sent, and the reader is told both that it was
      * not and nothing about why.
      */
+    const rejectedCount = report.runsRejected.length + report.refused.length;
     const push = sent.length > 0
         ? `sent ${sent.join(' + ')}`
-        : report.runsRejected.length > 0
+        : rejectedCount > 0
             ? 'nothing accepted'
             : true === report.localIsEmpty
                 ? // Not the same statement as "nothing to send", which describes a repo that is simply up
@@ -281,15 +532,32 @@ export function describeSync(report) {
     const pull = 0 === report.pulled
         ? ''
         : `, pulled ${String(report.pulled)} decision(s)${report.morePending ? ' (more waiting)' : ''}`;
-    /*
-     * One reason, not a count. A rejection count tells somebody they have a problem and nothing about
-     * which problem — and these arrive from a BACKGROUND daemon, so the summary line is often the only
-     * place anybody ever sees it. Rejections in one cycle almost always share a cause (a version skew
-     * refuses every payload the same way), so the first reason plus the count is the whole story
-     * without printing a line per run; `reticle sync` still lists them all.
-     */
-    const bad = 0 === report.runsRejected.length
-        ? ''
-        : `, ${String(report.runsRejected.length)} rejected — ${report.runsRejected[0]?.reason ?? 'no reason given'}`;
-    return `${push}${pull}${bad}`;
+    return `${push}${pull}${problemsOf(report)
+        .map((problem) => `, ${problem}`)
+        .join('')}`;
+}
+/**
+ * What went wrong, one phrase per kind of problem.
+ *
+ * One reason, not a count. A rejection count tells somebody they have a problem and nothing about
+ * which problem — and these arrive from a BACKGROUND daemon, so the summary line is often the only
+ * place anybody ever sees it. Rejections in one cycle almost always share a cause (a version skew
+ * refuses every payload the same way), so the first reason plus the count is the whole story
+ * without printing a line per item; `reticle sync` still lists them all.
+ */
+function problemsOf(report) {
+    const problems = [];
+    const count = report.runsRejected.length + report.refused.length;
+    if (count > 0) {
+        const first = report.runsRejected[0]?.reason ?? report.refused[0]?.reason ?? 'no reason given';
+        problems.push(`${String(count)} rejected — ${first}`);
+    }
+    const firstRefusal = report.notRetried[0];
+    if (firstRefusal !== undefined)
+        problems.push(`refused, not retried: ${String(report.notRetried.length)} run(s) (${firstRefusal.reason})`);
+    for (const set of report.setsNotRetried)
+        problems.push(`refused, not retried: ${String(set.count)} ${set.part}(s) (${set.reason})`);
+    if (report.held.length > 0)
+        problems.push(`not sent: ${report.held.join('; ')}`);
+    return problems;
 }

package/dist/memory/cloud/sync-daemon.d.ts

@@ -45,6 +45,19 @@ export interface SyncDaemon {
      */
     nudge: () => void;
     stop: () => void;
+    /**
+     * Stop, but send what is still owed first.
+     *
+     * `stop()` cancels the pending timer, and the last thing a session writes is the run artifact the
+     * whole drive produced — so a daemon asked to close moments after a verdict dropped the one push
+     * anybody was waiting on. Measured against app.reticle.sh: a drive wrote its run, the daemon
+     * exited, and the dashboard's newest row stayed three weeks old.
+     *
+     * Bounded, because shutdown is not allowed to hang: an in-flight cycle is waited out only until
+     * the deadline, and a cycle that cannot finish in time is simply the interval's problem next
+     * time — the artifact is still on disk and the next daemon for this project sends it.
+     */
+    flush: () => Promise<void>;
 }
 /**
  * Start the loop. Safe to call for an UNLINKED project: it resolves the link on every tick and does

package/dist/memory/cloud/sync-daemon.js

@@ -54,6 +54,21 @@ const FIRST_CYCLE_DELAY_MS = 5_000;
  * written just after, so cycling on the very first byte would ship a bundle that is missing them.
  */
 const NUDGE_DELAY_MS = 1_500;
+/**
+ * How long a flush waits for a cycle already in flight before running its own.
+ *
+ * Shutdown budget, not a sync budget: every call inside a cycle already carries its own timeout, so
+ * this only bounds the WAIT. Generous enough for an ordinary cycle to finish, short enough that a
+ * stalled one cannot hold a daemon open.
+ */
+const FLUSH_WAIT_MS = 3_000;
+const FLUSH_POLL_MS = 25;
+/** Whether a cycle landed or collected anything — what earns the fast interval and a log line. */
+const movedAnything = (report) => report.runsSent > 0 ||
+    report.flowsSent > 0 ||
+    report.capsulesSent > 0 ||
+    report.derivedSent.length > 0 ||
+    report.pulled > 0;
 const defaultRequest = async (url, init) => {
     const res = await fetch(url, init);
     return { status: res.status, text: await res.text() };
@@ -141,11 +156,9 @@ export function startSyncDaemon(deps) {
                     log('reticle_cloud_sync_failed', { root, error: report.error });
                     continue;
                 }
-                const moved = report.runsSent > 0 ||
-                    report.flowsSent > 0 ||
-                    report.capsulesSent > 0 ||
-                    report.derivedSent.length > 0 ||
-                    report.pulled > 0;
+                if (!report.ok)
+                    log('reticle_cloud_sync_failed', { root, error: describeSync(report) });
+                const moved = movedAnything(report);
                 // The root is NAMED here and not in the single-root log below, because with several repos
                 // reporting, "synced 3 runs" without a directory is not an answer to "synced from where".
                 if (moved)
@@ -196,27 +209,27 @@ export function startSyncDaemon(deps) {
             // failed. The log below is deliberately quiet when idle, which is right for a log and wrong
             // for a hook: from outside, "nothing changed" and "sync has been broken since Tuesday" are
             // the same silence, and a consumer that cannot tell them apart has to guess.
+            /*
+             * `ok`, not "no transport error": a push the server refused completed its HTTP exchange and
+             * still left the dashboard without the thing. The refusal's reason rides as the error.
+             */
+            const failure = report.error ?? (report.ok ? undefined : describeSync(report));
             emitSyncHook({
                 runsPushed: report.runsSent,
-                ok: report.error === undefined,
-                ...(report.error === undefined ? {} : { error: report.error }),
+                ok: report.ok,
+                ...(failure === undefined ? {} : { error: failure }),
             });
-            if (report.error !== undefined) {
-                if (report.error !== reportedError) {
-                    reportedError = report.error;
-                    log('reticle_cloud_sync_failed', { error: report.error });
-                }
+            if (failure !== undefined && failure !== reportedError) {
+                reportedError = failure;
+                log('reticle_cloud_sync_failed', { error: failure });
             }
-            else {
-                reportedError = undefined;
+            if (report.error === undefined) {
+                if (failure === undefined)
+                    reportedError = undefined;
                 // Only when something actually moved. A per-minute "nothing to send" is noise that trains
                 // people to stop reading the log.
-                const moved = report.runsSent > 0 ||
-                    report.flowsSent > 0 ||
-                    report.capsulesSent > 0 ||
-                    report.derivedSent.length > 0 ||
-                    report.pulled > 0;
-                if (moved)
+                const moved = movedAnything(report);
+                if (moved && failure === undefined)
                     log('reticle_cloud_synced', { summary: describeSync(report) });
                 /*
                  * A cycle that moved something means a drive is in progress, so the next one comes sooner;
@@ -270,6 +283,18 @@ export function startSyncDaemon(deps) {
                 clearTimeout(timer);
             timer = undefined;
         },
+        flush: async () => {
+            stopped = true;
+            if (timer !== undefined)
+                clearTimeout(timer);
+            timer = undefined;
+            // A cycle already running may have started BEFORE the write being flushed, so its report says
+            // nothing about that artifact. Wait it out rather than skipping on the overlap guard.
+            const deadline = Date.now() + FLUSH_WAIT_MS;
+            while (running && Date.now() < deadline)
+                await new Promise((resolve) => setTimeout(resolve, FLUSH_POLL_MS));
+            await cycle();
+        },
     };
 }
 /**

package/dist/memory/cloud/sync-disk.js

@@ -12,7 +12,7 @@
  */
 import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync, } from 'node:fs';
 import { dirname, join } from 'node:path';
-import { ReticleDir } from '@reticlehq/core';
+import { IntentDir, ReticleDir } from '@reticlehq/core';
 import { subjectFor } from '../intent/intent-subject.js';
 const JSON_SUFFIX = '.json';
 /** Read + parse, or undefined for absent/unreadable/malformed. Never throws. */
@@ -55,13 +55,25 @@ function readJsonDir(dir) {
         return [];
     }
 }
-/** Flows live one directory deeper, under the app's own project id. */
+/**
+ * Flows: one directory deeper under the app's own project id, and also flat in `.reticle/flows/`,
+ * which the flow store still saves and loads (a flow saved with no project id, or from before flows
+ * were kept per project). Reading only the subfolders skipped every flat flow while the cycle
+ * reported ok.
+ */
 function readFlows(root) {
     const dir = join(root, ReticleDir.FLOWS_SUBDIR);
     try {
         if (!existsSync(dir))
             return [];
-        return readdirSync(dir).flatMap((scope) => readJsonDir(join(dir, scope)));
+        return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+            if (entry.isDirectory())
+                return readJsonDir(join(dir, entry.name));
+            if (!entry.name.endsWith(JSON_SUFFIX))
+                return [];
+            const flow = readJson(join(dir, entry.name));
+            return flow === undefined ? [] : [flow];
+        });
     }
     catch {
         return [];
@@ -71,10 +83,12 @@ const DERIVED_FILE = {
     impact: ReticleDir.IMPACT_FILE,
     flake: ReticleDir.FLAKE_FILE,
     intent: ReticleDir.INTENT_FILE,
+    envelopes: ReticleDir.ENVELOPES_FILE,
+    'assertion-tiers': ReticleDir.TIERS_FILE,
 };
 /** The directory the sharded intent store writes into, beside the legacy flat file. */
-const INTENT_SUBDIR = 'intent';
-const INTENT_INDEX = 'index.json';
+const INTENT_SUBDIR = ReticleDir.INTENT_SUBDIR;
+const INTENT_INDEX = IntentDir.INDEX_FILE;
 /**
  * Every intent this project holds, flat file and shards merged.
  *
@@ -98,10 +112,15 @@ function readIntent(reticleRoot) {
         ...(legacy?.intents ?? {}),
     };
     let shards = 0;
-    for (const file of safeReaddir(join(reticleRoot, INTENT_SUBDIR))) {
-        if (!file.endsWith(JSON_SUFFIX) || INTENT_INDEX === file)
+    for (const entry of safeReaddir(join(reticleRoot, INTENT_SUBDIR))) {
+        if (INTENT_INDEX === entry)
             continue;
-        const shard = readJson(join(reticleRoot, INTENT_SUBDIR, file));
+        // `<subject>/intent.json` is the layout; a bare `<subject>.json` is an interim shard the store
+        // has not migrated yet, and is still this project's memory until it has.
+        const path = entry.endsWith(JSON_SUFFIX)
+            ? join(reticleRoot, INTENT_SUBDIR, entry)
+            : join(reticleRoot, INTENT_SUBDIR, entry, IntentDir.SHARD_FILE);
+        const shard = readJson(path);
         // A hand-edited or half-written shard costs one subject, never the whole sync.
         if (shard?.intents === undefined)
             continue;