@reticlehq/server 3.7.0 → 3.8.1

package/dist/memory/cloud/sync-cycle.js

@@ -33,6 +33,10 @@
 import { FlowErrorCode, ReticleDir, SYNC_BATCH_LIMITS, flowJsonReplacer } from '@reticlehq/core';
 import { hashPayload } from './sync-hash.js';
 import { syncScreenshots } from './sync-screenshots.js';
+import { describeTransportError, problemsOf } from './sync-describe.js';
+import { conflictOf, flowBases, versionsIn } from './sync-flow-versions.js';
+import { asJson, flowName, isRecord, numberAt, numbersIn, partResult, } from './sync-answer.js';
+export { FLOW_CONFLICT_CODE } from './sync-flow-versions.js';
 /** The server's own doors. Kept beside the code that calls them, like the other cloud paths. */
 const SYNC_PATH = '/v1/sync';
 const SYNC_STATUS_PATH = '/v1/sync/status';
@@ -57,49 +61,19 @@ export const DERIVED_RECORDS = [
     // The notes a human pinned in the HUD, pending and resolved: the review trail a team reads.
     { kind: 'notes', file: ReticleDir.NOTES_FILE },
 ];
-const asJson = (text) => {
-    try {
-        return JSON.parse(text);
-    }
-    catch {
-        return undefined;
-    }
-};
-const isRecord = (v) => 'object' === typeof v && null !== v;
-/** A list of numbers, or undefined when the server did not send a list at all. */
-const numbersIn = (v) => Array.isArray(v) ? v.filter((n) => 'number' === typeof n) : undefined;
-/** A numeric field of an artifact, or undefined when it has none. */
-const numberAt = (v, field) => {
-    const n = isRecord(v) ? v[field] : undefined;
-    return 'number' === typeof n ? n : undefined;
-};
-/** The name a person knows a flow by, falling back to its position. */
-const flowName = (flow, index) => {
-    const name = isRecord(flow) ? flow['name'] : undefined;
-    return 'string' === typeof name && name.length > 0 ? name : `flow #${String(index)}`;
-};
+/** A refusal older than this is offered again, whatever else is unchanged. */
+export const REFUSAL_RETRY_AFTER_MS = 7 * 24 * 60 * 60 * 1000;
+/** Does an earlier refusal still hold? An entry from before the stamp existed is asked once more. */
+function refusalHolds(prior, now) {
+    return (prior.acceptsHash === now.acceptsHash &&
+        prior.version === now.version &&
+        'number' === typeof prior.at &&
+        now.at - prior.at < REFUSAL_RETRY_AFTER_MS);
+}
 const FLOW_VERSION_FIX = 'update the platform or remove `compare` from the flow';
 const RUN_VERSION_FIX = 'update the platform';
-/** The server's per-part answer: accepted count and rejection list, each read defensively. */
 /** A 200 that never confirmed the runs it was sent. Kept queued, and said, rather than dropped. */
 export const MSG_RUNS_UNCONFIRMED = 'sync 200 without a runs result: the server did not confirm the runs it was sent, so they stay queued';
-function partResult(body, part) {
-    const raw = isRecord(body[part]) ? body[part] : {};
-    const accepted = 'number' === typeof raw['accepted'] ? raw['accepted'] : 0;
-    const rejected = (Array.isArray(raw['rejected']) ? raw['rejected'] : [])
-        .filter(isRecord)
-        .map((r) => ({
-        index: 'number' === typeof r['index'] ? r['index'] : -1,
-        reason: 'string' === typeof r['reason'] ? r['reason'] : 'no reason given',
-        code: r['code'],
-        version: r['version'],
-    }));
-    return { accepted, rejected };
-}
-/**
- * Split runs into requests bounded by count and by serialized size. A run bigger than the byte
- * bound on its own still goes, alone: it cannot be split, and holding it back would hide it.
- */
 /** The delivery record, kept only for the runs still on disk — the same bound as `sentRunIds`. */
 function heldHashes(runs, recorded) {
     const out = {};
@@ -110,6 +84,10 @@ function heldHashes(runs, recorded) {
     }
     return out;
 }
+/**
+ * Split runs into requests bounded by count and by serialized size. A run bigger than the byte
+ * bound on its own still goes, alone: it cannot be split, and holding it back would hide it.
+ */
 function batchRuns(runs) {
     const batches = [];
     let current = [];
@@ -129,10 +107,6 @@ function batchRuns(runs) {
         batches.push(current);
     return batches;
 }
-/**
- * Run one cycle. Never throws: the local record is already safe on disk, and a sync that can take
- * down the thing it is backing up is worse than no sync.
- */
 /** The command that moves a folder's earlier runs to the link it has now. */
 export const MOVE_HISTORY_COMMAND = 'reticle sync --move-history';
 /**
@@ -154,6 +128,10 @@ function stateForLink(deps) {
     }
     return { link, ...(true === deps.moveHistory ? {} : { linkedAt: deps.now() }) };
 }
+/**
+ * Run one cycle. Never throws: the local record is already safe on disk, and a sync that can take
+ * down the thing it is backing up is worse than no sync.
+ */
 export async function runSyncCycle(given) {
     const deps = { ...given, state: stateForLink(given) };
     const empty = {
@@ -231,6 +209,11 @@ export async function runSyncCycle(given) {
         // 2. SEND — only what the server does not already have, and only what it can read.
         const allRuns = deps.source.runs();
         const acceptsHash = hashPayload(held.accepts ?? null);
+        const stamp = {
+            acceptsHash,
+            ...(deps.clientVersion === undefined ? {} : { version: deps.clientVersion }),
+            at: deps.now(),
+        };
         const priorRefusals = isRecord(deps.state.refusedRuns) ? deps.state.refusedRuns : {};
         const refusedRuns = {};
         const notRetried = [];
@@ -253,7 +236,7 @@ export async function runSyncCycle(given) {
             const prior = priorRefusals[run.runId];
             const unchanged = prior !== undefined &&
                 'string' === typeof prior.reason &&
-                prior.acceptsHash === acceptsHash &&
+                refusalHolds(prior, stamp) &&
                 prior.payloadHash === hashPayload(run.payload);
             if (!unchanged)
                 return true;
@@ -332,7 +315,19 @@ export async function runSyncCycle(given) {
             return false;
         });
         const capsulesAll = deps.source.capsules();
-        const flowsHash = hashPayload(flows);
+        const named = flows.map((flow, index) => ({ flow, name: flowName(flow, index) }));
+        const priorFlowHashes = {};
+        if (isRecord(deps.state.sentFlowHashes)) {
+            for (const [name, hash] of Object.entries(deps.state.sentFlowHashes))
+                if ('string' === typeof hash)
+                    priorFlowHashes[name] = hash;
+        }
+        else if (deps.state.sentFlowsHash === hashPayload(flows))
+            for (const { flow, name } of named)
+                priorFlowHashes[name] = hashPayload(flow);
+        // Only the flows that changed since the platform last took them; the platform upserts by name.
+        const pendingFlows = named.filter(({ flow, name }) => priorFlowHashes[name] !== hashPayload(flow));
+        const flowsHash = hashPayload(pendingFlows.map(({ flow }) => flow));
         const capsulesHash = hashPayload(capsulesAll);
         const priorSets = isRecord(deps.state.refusedSets) ? deps.state.refusedSets : {};
         const setsNotRetried = [];
@@ -343,7 +338,7 @@ export async function runSyncCycle(given) {
                 return false;
             const prior = priorSets[part];
             if (prior !== undefined && prior.hash === hash) {
-                if (prior.acceptsHash !== acceptsHash)
+                if (!refusalHolds(prior, stamp))
                     return true;
                 refusedSets[part] = prior;
                 setsNotRetried.push({ part, count: prior.count, reason: prior.reason });
@@ -351,10 +346,26 @@ export async function runSyncCycle(given) {
             }
             return hash !== sentHash;
         };
-        const flowsChanged = due('flow', flowsHash, flows.length, deps.state.sentFlowsHash);
+        const flowsChanged = due('flow', flowsHash, pendingFlows.length, undefined);
         const capsulesChanged = due('capsule', capsulesHash, capsulesAll.length, deps.state.sentCapsulesHash);
-        if (flowsChanged)
-            bundle['flows'] = flows;
+        // Each flow names the version it was based on; a stale base is answered with a conflict.
+        const priorVersions = versionsIn(deps.state.flowVersions);
+        const taken = {};
+        if (flowsChanged) {
+            bundle['flows'] = pendingFlows.map(({ flow }) => flow);
+            bundle['flowBases'] = flowBases(pendingFlows, priorVersions, priorFlowHashes);
+        }
+        const conflictCopies = [];
+        // What the platform holds from here once this cycle is answered: kept for the flows still here.
+        const flowHashes = {};
+        for (const { name } of named) {
+            const hash = priorFlowHashes[name];
+            if (hash !== undefined)
+                flowHashes[name] = hash;
+            const v = priorVersions[name];
+            if (v !== undefined)
+                taken[name] = v;
+        }
         const capsules = capsulesChanged ? capsulesAll : [];
         if (capsules.length > 0)
             bundle['capsules'] = capsules;
@@ -406,7 +417,7 @@ export async function runSyncCycle(given) {
                     recorded[run.runId] = hashPayload(run.payload);
                 }
                 else
-                    refusedRuns[run.runId] = { reason, payloadHash: hashPayload(run.payload), acceptsHash };
+                    refusedRuns[run.runId] = { reason, payloadHash: hashPayload(run.payload), ...stamp };
             });
             runsRejected.push(...runs.rejected.map((r) => ({ index: r.index + offset, reason: r.reason })));
             offset += batch.length;
@@ -418,8 +429,33 @@ export async function runSyncCycle(given) {
             const capsulesPart = partResult(answer, 'capsules');
             flowsSent = flowsPart.accepted;
             capsulesSent = capsulesPart.accepted;
+            const refusedFlows = new Set(flowsPart.rejected.map((r) => r.index));
+            const answered = isRecord(answer['flows']) ? answer['flows']['versions'] : undefined;
+            if (flowsChanged)
+                pendingFlows.forEach(({ flow, name }, index) => {
+                    if (refusedFlows.has(index))
+                        return;
+                    flowHashes[name] = hashPayload(flow);
+                    const v = isRecord(answered) ? answered[name] : undefined;
+                    if ('string' === typeof v)
+                        taken[name] = v;
+                });
+            // Another machine saved over the version this edit was based on: keep both, take theirs.
+            flowsPart.rejected = flowsPart.rejected.filter((r) => {
+                const name = pendingFlows[r.index]?.name;
+                const theirs = conflictOf(r);
+                if (name === undefined || theirs === undefined)
+                    return true;
+                const copy = deps.sink.resolveFlowConflict?.(name, theirs.flow);
+                if (copy === undefined)
+                    return true;
+                conflictCopies.push(copy);
+                flowHashes[name] = hashPayload(theirs.flow);
+                taken[name] = theirs.version;
+                return false;
+            });
             for (const r of flowsPart.rejected) {
-                const name = flowName(flows[r.index], r.index);
+                const name = pendingFlows[r.index]?.name ?? flowName(undefined, r.index);
                 refused.push({
                     part: 'flow',
                     reason: FlowErrorCode.WRONG_VERSION === r.code && 'number' === typeof r.version
@@ -433,14 +469,14 @@ export async function runSyncCycle(given) {
             if (flowsPart.rejected.length > 0)
                 refusedSets.flow = {
                     hash: flowsHash,
-                    acceptsHash,
+                    ...stamp,
                     count: flowsPart.rejected.length,
                     reason: firstOf('flow'),
                 };
             if (capsulesPart.rejected.length > 0)
                 refusedSets.capsule = {
                     hash: capsulesHash,
-                    acceptsHash,
+                    ...stamp,
                     count: capsulesPart.rejected.length,
                     reason: firstOf('capsule'),
                 };
@@ -474,12 +510,17 @@ export async function runSyncCycle(given) {
         }
         if (deps.projectId !== undefined)
             nextState.sentFlowNames = kept;
+        // Every cycle, so a flow gone from here is forgotten too: saved again, it goes again.
+        nextState.sentFlowHashes = flowHashes;
+        nextState.flowVersions = taken;
+        // Open until the person deletes the copy (or merges it back): counted on every cycle till then.
+        const flowConflicts = [...(deps.state.flowConflicts ?? []), ...conflictCopies].filter((copy, i, all) => all.indexOf(copy) === i && (onDisk.has(copy) || conflictCopies.includes(copy)));
+        nextState.flowConflicts = flowConflicts;
+        delete nextState.sentFlowsHash;
         if (requests > 0 && pushError === undefined) {
             nextState.lastPushAt = deps.now();
             // The platform answered for these sets, refusals included (reported above, and not re-offered
             // until the set changes — the same rule as a refused run).
-            if (flowsChanged)
-                nextState.sentFlowsHash = flowsHash;
             if (flowsChanged && deps.projectId !== undefined)
                 nextState.sentFlowNames = [...new Set([...kept, ...flows.map(flowName)])];
             if (capsules.length > 0)
@@ -578,6 +619,7 @@ export async function runSyncCycle(given) {
             ...(0 === screenshotsSent ? {} : { screenshotsSent }),
             runsRejected,
             flowsSent,
+            ...(0 === flowConflicts.length ? {} : { flowConflicts }),
             capsulesSent,
             derivedSent,
             refused,
@@ -613,92 +655,3 @@ export async function runSyncCycle(given) {
         return { ...empty, error: message };
     }
 }
-/**
- * Say WHICH host failed and WHY.
- *
- * Node's fetch rejects with the bare string "fetch failed" and hides the real reason — a refused
- * connection, a bad hostname, an expired certificate — one level down in `cause`. A machine that
- * has quietly stopped syncing shows that string in `reticle whoami`, and on its own it is
- * indistinguishable from every other network problem, so the person reading it learns nothing and
- * checks nothing. Naming the origin and the underlying code turns it into one thing to look at.
- */
-function describeTransportError(error, url) {
-    if (!(error instanceof Error))
-        return String(error);
-    const cause = error.cause;
-    if (!(cause instanceof Error))
-        return `${error.message} (${url})`;
-    const code = cause.code;
-    // The code and the message are each sometimes empty — an aggregate DNS failure carries a code and
-    // no text, a TLS failure the reverse. Joined only when both are there, so the line never trails
-    // off into a dangling separator.
-    const parts = ['string' === typeof code && code.length > 0 ? code : '', cause.message].filter((part) => part.length > 0);
-    const detail = parts.join(': ');
-    return 0 === detail.length
-        ? `${error.message} (${url})`
-        : `${error.message} — ${detail} (${url})`;
-}
-/** One line a human can read, for the CLI and the daemon log. */
-export function describeSync(report) {
-    if (report.error !== undefined)
-        return `sync failed — ${report.error}`;
-    const sent = [];
-    if (report.runsSent > 0)
-        sent.push(`${String(report.runsSent)} run(s)`);
-    if (report.flowsSent > 0)
-        sent.push(`${String(report.flowsSent)} flow(s)`);
-    if (report.capsulesSent > 0)
-        sent.push(`${String(report.capsulesSent)} capsule(s)`);
-    if ((report.screenshotsSent ?? 0) > 0)
-        sent.push(`${String(report.screenshotsSent)} screenshot(s)`);
-    if (report.derivedSent.length > 0)
-        sent.push(report.derivedSent.join(', '));
-    /*
-     * "Nothing to send" is a statement about the QUEUE, and it is false the moment the queue was full
-     * and the server threw it away. Built from what was accepted, the sentence used to read "nothing
-     * to send, 3 rejected" — something was very much sent, and the reader is told both that it was
-     * not and nothing about why.
-     */
-    const rejectedCount = report.runsRejected.length + report.refused.length;
-    const push = sent.length > 0
-        ? `sent ${sent.join(' + ')}`
-        : rejectedCount > 0
-            ? 'nothing accepted'
-            : true === report.localIsEmpty
-                ? // Not the same statement as "nothing to send", which describes a repo that is simply up
-                    // to date. This one has never recorded anything, which for a LINKED repo usually means
-                    // the app announces no projectId and its runs are landing under a different root.
-                    'nothing recorded here yet — if this app has been driven, it is reporting somewhere else'
-                : 'nothing to send';
-    const pull = 0 === report.pulled
-        ? ''
-        : `, pulled ${String(report.pulled)} decision(s)${report.morePending ? ' (more waiting)' : ''}`;
-    return `${push}${pull}${problemsOf(report)
-        .map((problem) => `, ${problem}`)
-        .join('')}`;
-}
-/**
- * What went wrong, one phrase per kind of problem.
- *
- * One reason, not a count. A rejection count tells somebody they have a problem and nothing about
- * which problem — and these arrive from a BACKGROUND daemon, so the summary line is often the only
- * place anybody ever sees it. Rejections in one cycle almost always share a cause (a version skew
- * refuses every payload the same way), so the first reason plus the count is the whole story
- * without printing a line per item; `reticle sync` still lists them all.
- */
-function problemsOf(report) {
-    const problems = [];
-    const count = report.runsRejected.length + report.refused.length;
-    if (count > 0) {
-        const first = report.runsRejected[0]?.reason ?? report.refused[0]?.reason ?? 'no reason given';
-        problems.push(`${String(count)} rejected — ${first}`);
-    }
-    const firstRefusal = report.notRetried[0];
-    if (firstRefusal !== undefined)
-        problems.push(`refused, not retried: ${String(report.notRetried.length)} run(s) (${firstRefusal.reason})`);
-    for (const set of report.setsNotRetried)
-        problems.push(`refused, not retried: ${String(set.count)} ${set.part}(s) (${set.reason})`);
-    if (report.held.length > 0)
-        problems.push(`not sent: ${report.held.join('; ')}`);
-    return problems;
-}

package/dist/memory/cloud/sync-daemon.d.ts

@@ -1,5 +1,7 @@
 import { type SyncReport } from './sync-cycle.js';
 import { type ProjectCloud } from './cloud-config.js';
+import { type FSWatcher } from 'node:fs';
+import { BackgroundSyncState, type BackgroundSyncGate } from './background-sync.js';
 import { type UnsyncedRoot } from './unsynced-roots.js';
 import type { FileSystemPort } from '../project/fs/fs-port.js';
 interface SyncDaemonDeps {
@@ -20,12 +22,15 @@ interface SyncDaemonDeps {
     /** Resolve the link for a root that is not this daemon's own. Required to use `otherRoots`. */
     cloudFor?: (root: string) => Promise<ProjectCloud>;
     now?: () => number;
+    /** This CLI's version, stamped on each refusal so a new one asks the platform again. */
+    clientVersion?: string;
     intervalMs?: number;
     /**
-     * Call `onChange` when this root's link file changes; returns how to stop. A re-link was read only
-     * on the next idle tick, so a daemon kept syncing to the old link for up to a minute.
+     * Call `onChange` with the path (relative to the root) of anything that changes under it; returns
+     * how to stop, or undefined when the root cannot be watched YET (it does not exist), in which case
+     * the daemon tries again after each cycle. A re-link is one of these changes.
      */
-    watchLink?: (onChange: () => void) => () => void;
+    watch?: (onChange: (path: string) => void) => (() => void) | undefined;
     /** Injected for the test; the real one is `fetch`. */
     request?: (url: string, init: {
         method: string;
@@ -34,7 +39,22 @@ interface SyncDaemonDeps {
     }) => Promise<{
         status: number;
         text: string;
+        etag?: string;
     }>;
+    /**
+     * May this root's work be sent now (opt-outs, link, sign-in)? Asked for every root on every tick.
+     * Absent: every resolved link may send, which is how `startSyncDaemon` behaved before.
+     */
+    gate?: (root: string, cloud: ProjectCloud) => BackgroundSyncGate;
+}
+/** Where the background sync of the daemon's own project stands, for `doctor` and the tests. */
+export interface BackgroundSyncReport {
+    state: BackgroundSyncState;
+    why: string;
+    /** Network failures in a row; the backoff grows with it. */
+    failures: number;
+    lastError?: string;
+    lastPushAt?: number;
 }
 export interface SyncDaemon {
     /** Run one cycle now, whatever the timer is doing. Returns undefined when not linked. */
@@ -69,12 +89,22 @@ export interface SyncDaemon {
      * or not. An unlinked one is the case nothing will ever send, and it used to be skipped silently.
      */
     unsyncedRoots: () => readonly UnsyncedRoot[];
+    /** Where the background sync of this daemon's own root stands. */
+    background: () => BackgroundSyncReport;
 }
 /**
  * Start the loop. Safe to call for an UNLINKED project: it resolves the link on every tick and does
  * nothing until one appears, which is what lets `reticle link` take effect without a restart.
  */
 export declare function startSyncDaemon(deps: SyncDaemonDeps): SyncDaemon;
+/**
+ * Flush, but return after `budgetMs` at the latest, and never throw.
+ *
+ * For a process on its way out. The cycle has per-call timeouts, but a stalled network still adds
+ * them up, and an exit must not wait on that. A cycle cut short loses nothing: the artifacts stay
+ * on disk, and the next daemon or `reticle sync` sends them.
+ */
+export declare function flushWithin(daemon: Pick<SyncDaemon, 'flush'>, budgetMs: number): Promise<void>;
 /**
  * Start cloud sync for a daemon and hand back the two hooks the daemon wires to it.
  *
@@ -93,5 +123,22 @@ export declare function attachCloudSync(deps: {
     homeDir: string;
     env: NodeJS.ProcessEnv;
     otherRoots: () => readonly string[];
+    clientVersion: string;
+    /**
+     * The signed-in session's token for a platform address, or undefined. See `signedInTo`. Given by
+     * the background daemon only: `start()`'s one cycle on close also serves CI, keyed by the env.
+     */
+    signedIn?: (url: string) => string | undefined;
+    /** Injected for the tests. */
+    request?: SyncDaemonDeps['request'];
+    intervalMs?: number;
 }): SyncDaemon;
+/**
+ * Watch a whole `.reticle` root, reporting each change by its path under the root.
+ *
+ * Returns undefined only when the root does not exist yet, so the caller tries again later. Where a
+ * recursive watch is unsupported (an old Node on Linux, a network filesystem) or fails later (too
+ * many files), it says so ONCE and returns a no-op: the backstop timer is the whole sync from then on.
+ */
+export declare function watchReticleRoot(reticleRoot: string, onChange: (path: string) => void, watchFn?: (root: string, listener: (event: string, name: string | null) => void) => FSWatcher): (() => void) | undefined;
 export {};