@osolmaz/pi-workflows 0.16.4 → 0.16.5

package/dist/server/server.d.ts

@@ -60,6 +60,12 @@ export declare class WorkflowServer {
     private decisionChannelConfig;
     private decisionChannelError;
     private channelReloading;
+    private automaticStatePruneTask;
+    private automaticStatePruneTimer;
+    private automaticStatePruneScheduled;
+    private automaticStatePruneDue;
+    private lastAutomaticStatePruneAt;
+    private nextAutomaticStatePruneAttemptAt;
     private nextTerminalMessageReconciliationAt;
     constructor(options?: WorkflowServerOptions);
     get endpoint(): string;
@@ -94,6 +100,13 @@ export declare class WorkflowServer {
     private executeResourceManagerOperation;
     private statusReceipt;
     private activeRunnerContentHashes;
+    private requestAutomaticStatePrune;
+    private resumeAutomaticStatePruneIfDue;
+    private startAutomaticStatePrune;
+    private runAutomaticStatePrune;
+    private scheduleAutomaticStatePruneTimer;
+    private automaticStatePruneCanContinue;
+    private waitForAutomaticStatePrune;
     private stateStatusReceipt;
     private answerCheckpoint;
     private answerDecision;

package/dist/server/server.js

@@ -15,7 +15,7 @@ import { ResourceManagerWorkflowCoordinator, } from "../resource-managers/workfl
 import { StateDatabase, workflowStatePath } from "../state/database.js";
 import { canonicalJson } from "../state/json.js";
 import { resourceIdFor } from "../state/mutation.js";
-import { pruneState } from "../state/prune.js";
+import { AUTOMATIC_STATE_PRUNE_INTERVAL_MS, pruneState, pruneStateAutomatically, } from "../state/prune.js";
 import { recordViewerDeltas } from "../state/viewer.js";
 import { workflowMessageIdFor } from "../state/workflow-messages.js";
 import { humanDecisionChannelRequest } from "../workflows/decision-presentation.js";
@@ -38,6 +38,7 @@ const SERVER_RENEW_MS = 10_000;
 const PACKAGE_VERSION = runtimePackageVersion();
 const CLAIM_POLL_MS = 2_000;
 const TERMINAL_MESSAGE_RECONCILE_MS = 1_000;
+const AUTOMATIC_STATE_PRUNE_IDLE_RETRY_MS = 5 * 60 * 1_000;
 const RUN_CLAIM_LEASE_MS = 30_000;
 const RESOURCE_MANAGER_CLAIM_LEASE_MS = 120_000;
 const RESOURCE_MANAGER_RENEW_MS = 30_000;
@@ -86,6 +87,12 @@ export class WorkflowServer {
     decisionChannelConfig = null;
     decisionChannelError = null;
     channelReloading = false;
+    automaticStatePruneTask = null;
+    automaticStatePruneTimer = null;
+    automaticStatePruneScheduled = false;
+    automaticStatePruneDue = true;
+    lastAutomaticStatePruneAt = null;
+    nextAutomaticStatePruneAttemptAt = 0;
     nextTerminalMessageReconciliationAt = 0;
     constructor(options = {}) {
         this.options = options;
@@ -159,13 +166,16 @@ export class WorkflowServer {
             this.startTimers();
             this.started = true;
             this.log(`ready on ${this.socketPath} at epoch ${this.claim.epoch}`);
+            this.requestAutomaticStatePrune();
             this.expireTimedOutInteraction();
-            void this.expireTimedOutDecision();
-            void this.claimOne();
-            void this.claimResourceManagerOne();
-            void this.reloadDecisionChannels().catch((error) => {
+            void this.expireTimedOutDecision().finally(() => this.resumeAutomaticStatePruneIfDue());
+            void this.claimOne().finally(() => this.resumeAutomaticStatePruneIfDue());
+            void this.claimResourceManagerOne().finally(() => this.resumeAutomaticStatePruneIfDue());
+            void this.reloadDecisionChannels()
+                .catch((error) => {
                 this.log(`decision channel startup failed: ${errorMessage(error)}`);
-            });
+            })
+                .finally(() => this.resumeAutomaticStatePruneIfDue());
         }
         catch (error) {
             const server = this.server;
@@ -196,9 +206,13 @@ export class WorkflowServer {
             clearInterval(this.pollTimer);
         if (this.viewTimer !== null)
             clearInterval(this.viewTimer);
+        if (this.automaticStatePruneTimer !== null)
+            clearTimeout(this.automaticStatePruneTimer);
         this.heartbeatTimer = null;
         this.pollTimer = null;
         this.viewTimer = null;
+        this.automaticStatePruneTimer = null;
+        this.automaticStatePruneScheduled = false;
         const server = this.server;
         this.server = null;
         this.detachSessionCoordinators();
@@ -227,6 +241,9 @@ export class WorkflowServer {
         this.activeChannels.clear();
         await Promise.allSettled(this.activationTasks.values());
         await Promise.allSettled(this.maintenanceCommands.values());
+        if (this.automaticStatePruneTask !== null) {
+            await Promise.allSettled([this.automaticStatePruneTask]);
+        }
         this.registry.killAll();
         if (this.claim !== null)
             this.serverState.releaseServer(this.claim);
@@ -524,6 +541,7 @@ export class WorkflowServer {
                     });
                 case "state.prune":
                     return await this.executeMaintenanceCommand(request, async () => {
+                        await this.waitForAutomaticStatePrune();
                         const payload = requireRecord(request.payload, "state.prune payload");
                         const before = requireString(payload.before, "before");
                         const apply = requireBoolean(payload.apply, "apply");
@@ -578,6 +596,7 @@ export class WorkflowServer {
             if (this.maintenanceCommands.get(key) === execution) {
                 this.maintenanceCommands.delete(key);
             }
+            this.requestAutomaticStatePrune();
         }
     }
     addSubscription(connection, request, kind, target) {
@@ -1521,6 +1540,116 @@ export class WorkflowServer {
         }
         return [...digests].map((digest) => Buffer.from(digest, "hex"));
     }
+    requestAutomaticStatePrune() {
+        this.automaticStatePruneDue = true;
+        if (!this.started ||
+            this.stopping ||
+            this.automaticStatePruneScheduled ||
+            this.automaticStatePruneTask !== null) {
+            return;
+        }
+        this.automaticStatePruneScheduled = true;
+        setImmediate(() => {
+            if (!this.automaticStatePruneScheduled)
+                return;
+            this.automaticStatePruneScheduled = false;
+            this.startAutomaticStatePrune();
+        });
+    }
+    resumeAutomaticStatePruneIfDue() {
+        if (this.automaticStatePruneDue)
+            this.requestAutomaticStatePrune();
+    }
+    startAutomaticStatePrune() {
+        if (this.automaticStatePruneTask !== null || this.stopping || !this.started)
+            return;
+        const task = this.runAutomaticStatePrune();
+        this.automaticStatePruneTask = task;
+        void task.finally(() => {
+            if (this.automaticStatePruneTask === task)
+                this.automaticStatePruneTask = null;
+        });
+    }
+    async runAutomaticStatePrune() {
+        if (!this.automaticStatePruneDue)
+            return;
+        if (!this.automaticStatePruneCanContinue()) {
+            this.scheduleAutomaticStatePruneTimer(AUTOMATIC_STATE_PRUNE_IDLE_RETRY_MS);
+            return;
+        }
+        const now = Date.now();
+        if (now < this.nextAutomaticStatePruneAttemptAt) {
+            this.scheduleAutomaticStatePruneTimer(this.nextAutomaticStatePruneAttemptAt - now);
+            return;
+        }
+        if (this.lastAutomaticStatePruneAt !== null &&
+            now - this.lastAutomaticStatePruneAt < AUTOMATIC_STATE_PRUNE_INTERVAL_MS) {
+            this.scheduleAutomaticStatePruneTimer(this.lastAutomaticStatePruneAt + AUTOMATIC_STATE_PRUNE_INTERVAL_MS - now);
+            return;
+        }
+        try {
+            const report = await pruneStateAutomatically(this.state, this.databasePath, {
+                now,
+                activeBlobHashes: () => this.activeRunnerContentHashes(),
+                shouldContinue: () => this.automaticStatePruneCanContinue(),
+            });
+            if (!report.completed) {
+                this.automaticStatePruneDue = true;
+                this.scheduleAutomaticStatePruneTimer(AUTOMATIC_STATE_PRUNE_IDLE_RETRY_MS);
+                return;
+            }
+            this.lastAutomaticStatePruneAt = now;
+            this.nextAutomaticStatePruneAttemptAt = 0;
+            this.automaticStatePruneDue = false;
+            this.scheduleAutomaticStatePruneTimer(AUTOMATIC_STATE_PRUNE_INTERVAL_MS);
+            this.log(`automatic state prune completed: ${report.selectedRuns} run(s) selected, ` +
+                `${report.blockedTrees} tree(s) blocked, ${report.deletedBlobs} blob(s) and ` +
+                `${report.deletedBlobBytes} byte(s) removed`);
+            if (report.compactionError !== undefined) {
+                this.log(`automatic state compaction failed: ${report.compactionError}`);
+            }
+        }
+        catch (error) {
+            this.automaticStatePruneDue = true;
+            this.nextAutomaticStatePruneAttemptAt = Date.now() + AUTOMATIC_STATE_PRUNE_IDLE_RETRY_MS;
+            this.scheduleAutomaticStatePruneTimer(AUTOMATIC_STATE_PRUNE_IDLE_RETRY_MS);
+            this.log(`automatic state prune failed: ${errorMessage(error)}`);
+        }
+    }
+    scheduleAutomaticStatePruneTimer(delayMs) {
+        if (this.stopping || !this.started)
+            return;
+        if (this.automaticStatePruneTimer !== null)
+            clearTimeout(this.automaticStatePruneTimer);
+        this.automaticStatePruneTimer = setTimeout(() => {
+            this.automaticStatePruneTimer = null;
+            this.requestAutomaticStatePrune();
+        }, Math.max(1, delayMs));
+        this.automaticStatePruneTimer.unref?.();
+    }
+    automaticStatePruneCanContinue() {
+        return (this.started &&
+            !this.stopping &&
+            this.activeRuns.size === 0 &&
+            this.activeResourceManagers.size === 0 &&
+            this.activationTasks.size === 0 &&
+            this.maintenanceCommands.size === 0 &&
+            this.pendingStarts.size === 0 &&
+            this.pendingRunClaims.size === 0 &&
+            this.pendingResumes.size === 0 &&
+            this.controlClaims.size === 0 &&
+            this.pendingTerminalMessageReconciliations.size === 0 &&
+            !this.resourceManagerPollActive &&
+            !this.decisionTimeoutActive &&
+            !this.channelReloading &&
+            [...this.activeChannels.values()].every((channel) => channel.inFlight.size === 0));
+    }
+    async waitForAutomaticStatePrune() {
+        this.automaticStatePruneScheduled = false;
+        const task = this.automaticStatePruneTask;
+        if (task !== null)
+            await task;
+    }
     stateStatusReceipt() {
         const count = (sql, ...params) => {
             const row = this.state.connection.prepare(sql).get(...params);
@@ -2485,6 +2614,7 @@ export class WorkflowServer {
         finally {
             clearInterval(renewTimer);
             this.activeResourceManagers.delete(key);
+            this.requestAutomaticStatePrune();
             if (!this.stopping)
                 setImmediate(() => void this.claimResourceManagerOne());
         }
@@ -3098,6 +3228,7 @@ export class WorkflowServer {
         finally {
             this.reapRunnerDescendants(envelope.runnerEpoch);
             this.activeRuns.delete(runId);
+            this.requestAutomaticStatePrune();
         }
     }
     async resumePendingRun(runId) {
@@ -3566,6 +3697,7 @@ export class WorkflowServer {
          WHERE resource_id = (SELECT resource_id FROM runs WHERE run_id = ?)
            AND generation = ?`)
             .run(context.runId, active.generation);
+        this.queue.settleRunEffect(context.runId, context.state.status === "waiting" ? "run.park_queue" : "run.settle_queue");
     }
     ensureTerminalWorkflowMessage(runId, now = Date.now(), stateOverride) {
         const queue = this.queue.getWorkflowRun(runId);