donobu 5.52.1 → 5.52.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/donobu-cli.js +58 -0
- package/dist/envVars.d.ts +2 -0
- package/dist/envVars.js +9 -0
- package/dist/esm/cli/donobu-cli.js +58 -0
- package/dist/esm/envVars.d.ts +2 -0
- package/dist/esm/envVars.js +9 -0
- package/dist/esm/lib/test/testExtension.js +5 -2
- package/dist/esm/lib/test/utils/donobuTestStack.js +1 -1
- package/dist/esm/main.d.ts +2 -0
- package/dist/esm/main.js +1 -1
- package/dist/esm/managers/AdminApiController.d.ts +1 -1
- package/dist/esm/managers/DonobuStack.d.ts +2 -0
- package/dist/esm/managers/DonobuStack.js +1 -1
- package/dist/esm/managers/PageInspector.js +43 -15
- package/dist/esm/persistence/files/FileUploadCache.d.ts +16 -0
- package/dist/esm/persistence/files/FileUploadCache.js +30 -0
- package/dist/esm/persistence/files/FileUploadWorker.d.ts +28 -4
- package/dist/esm/persistence/files/FileUploadWorker.js +31 -7
- package/dist/esm/persistence/files/fileUploadWorkerRegistry.d.ts +9 -0
- package/dist/esm/persistence/files/fileUploadWorkerRegistry.js +16 -0
- package/dist/esm/persistence/flows/FlowsPersistenceDonobuApi.d.ts +49 -13
- package/dist/esm/persistence/flows/FlowsPersistenceDonobuApi.js +106 -39
- package/dist/esm/persistence/flows/FlowsPersistenceRegistry.d.ts +17 -1
- package/dist/esm/persistence/flows/FlowsPersistenceRegistry.js +33 -10
- package/dist/lib/test/testExtension.js +5 -2
- package/dist/lib/test/utils/donobuTestStack.js +1 -1
- package/dist/main.d.ts +2 -0
- package/dist/main.js +1 -1
- package/dist/managers/AdminApiController.d.ts +1 -1
- package/dist/managers/DonobuStack.d.ts +2 -0
- package/dist/managers/DonobuStack.js +1 -1
- package/dist/managers/PageInspector.js +43 -15
- package/dist/persistence/files/FileUploadCache.d.ts +16 -0
- package/dist/persistence/files/FileUploadCache.js +30 -0
- package/dist/persistence/files/FileUploadWorker.d.ts +28 -4
- package/dist/persistence/files/FileUploadWorker.js +31 -7
- package/dist/persistence/files/fileUploadWorkerRegistry.d.ts +9 -0
- package/dist/persistence/files/fileUploadWorkerRegistry.js +16 -0
- package/dist/persistence/flows/FlowsPersistenceDonobuApi.d.ts +49 -13
- package/dist/persistence/flows/FlowsPersistenceDonobuApi.js +106 -39
- package/dist/persistence/flows/FlowsPersistenceRegistry.d.ts +17 -1
- package/dist/persistence/flows/FlowsPersistenceRegistry.js +33 -10
- package/package.json +1 -1
|
@@ -11,6 +11,15 @@ export interface FileUploadWorkerOptions {
|
|
|
11
11
|
readonly upload: UploadFn;
|
|
12
12
|
readonly platformLabel: string;
|
|
13
13
|
readonly maxAttempts?: number;
|
|
14
|
+
/**
|
|
15
|
+
* When set, the idle loop re-scans the cache for new `.pending` files every
|
|
16
|
+
* `pollIntervalMs` even without a {@link notify} call. Required when the
|
|
17
|
+
* producer runs in a DIFFERENT process than this worker (the parent-owned
|
|
18
|
+
* upload daemon draining a cache that Playwright worker processes write to),
|
|
19
|
+
* because `notify()` is in-process only. Same-process workers leave this
|
|
20
|
+
* unset and stay purely notify-driven.
|
|
21
|
+
*/
|
|
22
|
+
readonly pollIntervalMs?: number;
|
|
14
23
|
}
|
|
15
24
|
/**
|
|
16
25
|
* Drains pending file uploads from a {@link FileUploadCache}, uploading each
|
|
@@ -26,6 +35,7 @@ export declare class FileUploadWorker {
|
|
|
26
35
|
private readonly upload;
|
|
27
36
|
private readonly platformLabel;
|
|
28
37
|
private readonly maxAttempts;
|
|
38
|
+
private readonly pollIntervalMs?;
|
|
29
39
|
private readonly token;
|
|
30
40
|
private workerLoop;
|
|
31
41
|
private staleScanTimer;
|
|
@@ -37,6 +47,13 @@ export declare class FileUploadWorker {
|
|
|
37
47
|
constructor(options: FileUploadWorkerOptions);
|
|
38
48
|
start(): void;
|
|
39
49
|
stop(): Promise<void>;
|
|
50
|
+
/**
|
|
51
|
+
* Number of uploads THIS worker still owes before its process can exit:
|
|
52
|
+
* pending files plus this worker's own in-flight claims, excluding claims
|
|
53
|
+
* orphaned by other/crashed processes. Mirrors what {@link flush} waits on,
|
|
54
|
+
* so callers can log an accurate "N still syncing" count at shutdown.
|
|
55
|
+
*/
|
|
56
|
+
countOwnedActive(): Promise<number>;
|
|
40
57
|
/**
|
|
41
58
|
* Returns a snapshot of the worker's current state for surfacing to the
|
|
42
59
|
* UI (e.g. "N uploads still pending — wait or quit anyway?"). Reads the
|
|
@@ -50,10 +67,17 @@ export declare class FileUploadWorker {
|
|
|
50
67
|
*/
|
|
51
68
|
notify(): void;
|
|
52
69
|
/**
|
|
53
|
-
* Returns once
|
|
54
|
-
* use this on graceful shutdown to push outstanding uploads before
|
|
55
|
-
*
|
|
56
|
-
*
|
|
70
|
+
* Returns once THIS worker's queue has drained or the timeout has elapsed.
|
|
71
|
+
* Callers use this on graceful shutdown to push outstanding uploads before
|
|
72
|
+
* the process exits.
|
|
73
|
+
*
|
|
74
|
+
* "Drained" means zero files this worker still owes: nothing in `.pending`
|
|
75
|
+
* (which this worker will claim and upload) and none of this worker's own
|
|
76
|
+
* `.uploading.<token>` claims (a file mid-retry is still active work).
|
|
77
|
+
* Foreign `.uploading.<otherToken>` claims left by other or crashed
|
|
78
|
+
* processes are NOT waited on — see {@link FileUploadCache.countOwnedActive}
|
|
79
|
+
* — because this process can neither upload nor release them; doing so
|
|
80
|
+
* would block teardown for the full timeout on a dead peer's orphaned claim.
|
|
57
81
|
*/
|
|
58
82
|
flush(timeoutMs: number): Promise<{
|
|
59
83
|
drained: boolean;
|
|
@@ -27,6 +27,7 @@ class FileUploadWorker {
|
|
|
27
27
|
this.upload = options.upload;
|
|
28
28
|
this.platformLabel = options.platformLabel;
|
|
29
29
|
this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
30
|
+
this.pollIntervalMs = options.pollIntervalMs;
|
|
30
31
|
let wakeResolver;
|
|
31
32
|
this.wakePromise = new Promise((resolve) => {
|
|
32
33
|
wakeResolver = resolve;
|
|
@@ -75,6 +76,15 @@ class FileUploadWorker {
|
|
|
75
76
|
await this.workerLoop;
|
|
76
77
|
this.workerLoop = null;
|
|
77
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* Number of uploads THIS worker still owes before its process can exit:
|
|
81
|
+
* pending files plus this worker's own in-flight claims, excluding claims
|
|
82
|
+
* orphaned by other/crashed processes. Mirrors what {@link flush} waits on,
|
|
83
|
+
* so callers can log an accurate "N still syncing" count at shutdown.
|
|
84
|
+
*/
|
|
85
|
+
async countOwnedActive() {
|
|
86
|
+
return this.cache.countOwnedActive(this.token);
|
|
87
|
+
}
|
|
78
88
|
/**
|
|
79
89
|
* Returns a snapshot of the worker's current state for surfacing to the
|
|
80
90
|
* UI (e.g. "N uploads still pending — wait or quit anyway?"). Reads the
|
|
@@ -102,21 +112,28 @@ class FileUploadWorker {
|
|
|
102
112
|
resolve?.();
|
|
103
113
|
}
|
|
104
114
|
/**
|
|
105
|
-
* Returns once
|
|
106
|
-
* use this on graceful shutdown to push outstanding uploads before
|
|
107
|
-
*
|
|
108
|
-
*
|
|
115
|
+
* Returns once THIS worker's queue has drained or the timeout has elapsed.
|
|
116
|
+
* Callers use this on graceful shutdown to push outstanding uploads before
|
|
117
|
+
* the process exits.
|
|
118
|
+
*
|
|
119
|
+
* "Drained" means zero files this worker still owes: nothing in `.pending`
|
|
120
|
+
* (which this worker will claim and upload) and none of this worker's own
|
|
121
|
+
* `.uploading.<token>` claims (a file mid-retry is still active work).
|
|
122
|
+
* Foreign `.uploading.<otherToken>` claims left by other or crashed
|
|
123
|
+
* processes are NOT waited on — see {@link FileUploadCache.countOwnedActive}
|
|
124
|
+
* — because this process can neither upload nor release them; doing so
|
|
125
|
+
* would block teardown for the full timeout on a dead peer's orphaned claim.
|
|
109
126
|
*/
|
|
110
127
|
async flush(timeoutMs) {
|
|
111
128
|
const deadline = Date.now() + timeoutMs;
|
|
112
129
|
while (Date.now() < deadline) {
|
|
113
|
-
const active = await this.cache.
|
|
130
|
+
const active = await this.cache.countOwnedActive(this.token);
|
|
114
131
|
if (active === 0) {
|
|
115
132
|
return { drained: true, remaining: 0 };
|
|
116
133
|
}
|
|
117
134
|
await this.sleep(Math.min(200, Math.max(50, deadline - Date.now())));
|
|
118
135
|
}
|
|
119
|
-
const remaining = await this.cache.
|
|
136
|
+
const remaining = await this.cache.countOwnedActive(this.token);
|
|
120
137
|
return { drained: remaining === 0, remaining };
|
|
121
138
|
}
|
|
122
139
|
// ── internals ────────────────────────────────────────────────────
|
|
@@ -124,7 +141,14 @@ class FileUploadWorker {
|
|
|
124
141
|
while (!this.stopping) {
|
|
125
142
|
const pending = await this.cache.listPending();
|
|
126
143
|
if (pending.length === 0) {
|
|
127
|
-
|
|
144
|
+
// Wake on an in-process notify() or on stop. When draining a cache
|
|
145
|
+
// that OTHER processes write to (the parent-owned daemon), also wake
|
|
146
|
+
// on a poll timer, since notify() can't cross the process boundary.
|
|
147
|
+
const idleWaiters = [this.wakePromise, this.stopSignal];
|
|
148
|
+
if (this.pollIntervalMs !== undefined) {
|
|
149
|
+
idleWaiters.push(this.sleep(this.pollIntervalMs));
|
|
150
|
+
}
|
|
151
|
+
await Promise.race(idleWaiters);
|
|
128
152
|
continue;
|
|
129
153
|
}
|
|
130
154
|
for (const entry of pending) {
|
|
@@ -30,6 +30,15 @@ export declare function unregisterFileUploadWorker(worker: FileUploadWorker): vo
|
|
|
30
30
|
* time.
|
|
31
31
|
*/
|
|
32
32
|
export declare function getFileUploadAggregateStatus(): Promise<FileUploadAggregateStatus>;
|
|
33
|
+
/**
|
|
34
|
+
* Total uploads still owed by every registered worker — pending files plus
|
|
35
|
+
* each worker's own in-flight claims, excluding claims orphaned by other or
|
|
36
|
+
* crashed processes (which this process can neither upload nor release). This
|
|
37
|
+
* is what a graceful shutdown actually waits on, so use it (not the global
|
|
38
|
+
* {@link getFileUploadAggregateStatus}, which counts foreign claims too) when
|
|
39
|
+
* deciding whether a drain is needed and reporting how much is left.
|
|
40
|
+
*/
|
|
41
|
+
export declare function getOwnedActiveUploadCount(): Promise<number>;
|
|
33
42
|
/**
|
|
34
43
|
* Drains and stops every registered worker. Used during graceful shutdown.
|
|
35
44
|
* Returns a summary across all workers so the caller can log how many
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.registerFileUploadWorker = registerFileUploadWorker;
|
|
4
4
|
exports.unregisterFileUploadWorker = unregisterFileUploadWorker;
|
|
5
5
|
exports.getFileUploadAggregateStatus = getFileUploadAggregateStatus;
|
|
6
|
+
exports.getOwnedActiveUploadCount = getOwnedActiveUploadCount;
|
|
6
7
|
exports.shutdownFileUploadWorkers = shutdownFileUploadWorkers;
|
|
7
8
|
const Logger_1 = require("../../utils/Logger");
|
|
8
9
|
const workers = new Set();
|
|
@@ -38,6 +39,21 @@ async function getFileUploadAggregateStatus() {
|
|
|
38
39
|
}
|
|
39
40
|
return { totalPending, totalInFlight, totalFailed, perPlatform };
|
|
40
41
|
}
|
|
42
|
+
/**
|
|
43
|
+
* Total uploads still owed by every registered worker — pending files plus
|
|
44
|
+
* each worker's own in-flight claims, excluding claims orphaned by other or
|
|
45
|
+
* crashed processes (which this process can neither upload nor release). This
|
|
46
|
+
* is what a graceful shutdown actually waits on, so use it (not the global
|
|
47
|
+
* {@link getFileUploadAggregateStatus}, which counts foreign claims too) when
|
|
48
|
+
* deciding whether a drain is needed and reporting how much is left.
|
|
49
|
+
*/
|
|
50
|
+
async function getOwnedActiveUploadCount() {
|
|
51
|
+
let total = 0;
|
|
52
|
+
for (const worker of workers) {
|
|
53
|
+
total += await worker.countOwnedActive();
|
|
54
|
+
}
|
|
55
|
+
return total;
|
|
56
|
+
}
|
|
41
57
|
/**
|
|
42
58
|
* Drains and stops every registered worker. Used during graceful shutdown.
|
|
43
59
|
* Returns a summary across all workers so the caller can log how many
|
|
@@ -4,6 +4,7 @@ import type { FlowMetadata, FlowsQuery } from '../../models/FlowMetadata';
|
|
|
4
4
|
import type { PaginatedResult } from '../../models/PaginatedResult';
|
|
5
5
|
import type { ToolCall } from '../../models/ToolCall';
|
|
6
6
|
import type { VideoSegment } from '../../models/VideoSegment';
|
|
7
|
+
import { FileUploadCache } from '../files/FileUploadCache';
|
|
7
8
|
import type { FlowsPersistence } from './FlowsPersistence';
|
|
8
9
|
/**
|
|
9
10
|
* A {@link FlowsPersistence} implementation that persists flow data via the
|
|
@@ -24,8 +25,20 @@ export declare class FlowsPersistenceDonobuApi implements FlowsPersistence {
|
|
|
24
25
|
private readonly baseUrl;
|
|
25
26
|
private readonly apiKey;
|
|
26
27
|
private readonly fileCache;
|
|
27
|
-
private readonly
|
|
28
|
-
|
|
28
|
+
private readonly notifyUpload;
|
|
29
|
+
/**
|
|
30
|
+
* Prefer {@link createDonobuFlowsPersistence} over calling this directly — it
|
|
31
|
+
* builds the cache and, when this process owns draining, the background
|
|
32
|
+
* uploader, and wires `notifyUpload` for you.
|
|
33
|
+
*
|
|
34
|
+
* @param fileCache shared local upload cache (bytes + `.pending` markers).
|
|
35
|
+
* @param notifyUpload invoked after each cache write to wake a co-located
|
|
36
|
+
* uploader. A no-op in producer-only mode (Playwright workers under
|
|
37
|
+
* `donobu test`), where a different process owns draining the shared cache;
|
|
38
|
+
* cross-process wake-ups are impossible anyway, so the parent's uploader
|
|
39
|
+
* polls instead.
|
|
40
|
+
*/
|
|
41
|
+
constructor(baseUrl: string, apiKey: string, fileCache: FileUploadCache, notifyUpload?: () => void);
|
|
29
42
|
private request;
|
|
30
43
|
private jsonRequest;
|
|
31
44
|
setFlowMetadata(flowMetadata: FlowMetadata): Promise<void>;
|
|
@@ -50,20 +63,43 @@ export declare class FlowsPersistenceDonobuApi implements FlowsPersistence {
|
|
|
50
63
|
*/
|
|
51
64
|
getFlowFile(flowId: string, fileId: string): Promise<Buffer | null>;
|
|
52
65
|
/**
|
|
53
|
-
* Writes bytes to the local file cache and returns immediately. The
|
|
54
|
-
*
|
|
55
|
-
* {@link FileUploadWorker}
|
|
56
|
-
*
|
|
66
|
+
* Writes bytes to the local file cache and returns immediately. The actual
|
|
67
|
+
* HTTP upload to the Donobu API runs asynchronously in a
|
|
68
|
+
* {@link FileUploadWorker} (owned by this process or, under `donobu test`, by
|
|
69
|
+
* the CLI parent) with retry + backoff. Same-machine reads during the upload
|
|
70
|
+
* window hit the local cache (no 404).
|
|
57
71
|
*/
|
|
58
72
|
setFlowFile(flowId: string, fileId: string, fileBytes: Buffer): Promise<void>;
|
|
59
|
-
/**
|
|
60
|
-
* Performs the actual HTTP PUT against the Donobu API. Called by the
|
|
61
|
-
* {@link FileUploadWorker} when draining the upload queue. Not part of
|
|
62
|
-
* the public {@link FlowsPersistence} surface — callers go through
|
|
63
|
-
* {@link setFlowFile}.
|
|
64
|
-
*/
|
|
65
|
-
private uploadFlowFileViaHttp;
|
|
66
73
|
setBrowserState(flowId: string, browserState: BrowserStorageState): Promise<void>;
|
|
67
74
|
getBrowserState(flowId: string): Promise<BrowserStorageState | null>;
|
|
68
75
|
}
|
|
76
|
+
export interface CreateDonobuFlowsPersistenceOptions {
|
|
77
|
+
readonly baseUrl: string;
|
|
78
|
+
readonly apiKey: string;
|
|
79
|
+
readonly baseWorkingDirectory?: string;
|
|
80
|
+
/**
|
|
81
|
+
* Whether THIS process owns draining uploads to the cloud. True for a
|
|
82
|
+
* standalone process (Studio, or a raw `playwright test` worker with no CLI
|
|
83
|
+
* parent): start the background uploader here. False for a Playwright worker
|
|
84
|
+
* under `donobu test`: producer-only — write bytes to the shared cache and
|
|
85
|
+
* let the CLI parent drain them.
|
|
86
|
+
*/
|
|
87
|
+
readonly ownsUploadDraining: boolean;
|
|
88
|
+
/** Idle re-scan interval for the uploader, used by the cross-process parent
|
|
89
|
+
* daemon whose producers live in other processes (no in-process notify). */
|
|
90
|
+
readonly pollIntervalMs?: number;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Composition root for the Donobu persistence client. Builds the shared upload
|
|
94
|
+
* cache and, when this process owns draining, the background uploader; returns
|
|
95
|
+
* a {@link FlowsPersistenceDonobuApi} wired to notify that uploader on writes.
|
|
96
|
+
*/
|
|
97
|
+
export declare function createDonobuFlowsPersistence(options: CreateDonobuFlowsPersistenceOptions): FlowsPersistenceDonobuApi;
|
|
98
|
+
/**
|
|
99
|
+
* Starts ONLY the Donobu cloud uploader (no read/write client) for a process
|
|
100
|
+
* that owns draining but never reads or writes flows itself — the long-lived
|
|
101
|
+
* `donobu` CLI parent. The worker registers globally; drain it later via
|
|
102
|
+
* `shutdownFileUploadWorkers()`.
|
|
103
|
+
*/
|
|
104
|
+
export declare function startDonobuParentUploader(baseUrl: string, apiKey: string, pollIntervalMs: number, baseWorkingDirectory?: string): void;
|
|
69
105
|
//# sourceMappingURL=FlowsPersistenceDonobuApi.d.ts.map
|
|
@@ -4,6 +4,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.FlowsPersistenceDonobuApi = void 0;
|
|
7
|
+
exports.createDonobuFlowsPersistence = createDonobuFlowsPersistence;
|
|
8
|
+
exports.startDonobuParentUploader = startDonobuParentUploader;
|
|
7
9
|
const path_1 = __importDefault(require("path"));
|
|
8
10
|
const FlowNotFoundException_1 = require("../../exceptions/FlowNotFoundException");
|
|
9
11
|
const BrowserStorageState_1 = require("../../models/BrowserStorageState");
|
|
@@ -29,29 +31,27 @@ const VIDEO_FILE_ID = 'video.webm';
|
|
|
29
31
|
* remain synchronous — they're small JSON, no bandwidth concern.
|
|
30
32
|
*/
|
|
31
33
|
class FlowsPersistenceDonobuApi {
|
|
32
|
-
|
|
34
|
+
/**
|
|
35
|
+
* Prefer {@link createDonobuFlowsPersistence} over calling this directly — it
|
|
36
|
+
* builds the cache and, when this process owns draining, the background
|
|
37
|
+
* uploader, and wires `notifyUpload` for you.
|
|
38
|
+
*
|
|
39
|
+
* @param fileCache shared local upload cache (bytes + `.pending` markers).
|
|
40
|
+
* @param notifyUpload invoked after each cache write to wake a co-located
|
|
41
|
+
* uploader. A no-op in producer-only mode (Playwright workers under
|
|
42
|
+
* `donobu test`), where a different process owns draining the shared cache;
|
|
43
|
+
* cross-process wake-ups are impossible anyway, so the parent's uploader
|
|
44
|
+
* polls instead.
|
|
45
|
+
*/
|
|
46
|
+
constructor(baseUrl, apiKey, fileCache, notifyUpload = () => { }) {
|
|
33
47
|
this.baseUrl = baseUrl;
|
|
34
48
|
this.apiKey = apiKey;
|
|
35
|
-
this.fileCache =
|
|
36
|
-
this.
|
|
37
|
-
cache: this.fileCache,
|
|
38
|
-
platformLabel: PLATFORM_LABEL,
|
|
39
|
-
upload: (flowId, fileId, bytes) => this.uploadFlowFileViaHttp(flowId, fileId, bytes),
|
|
40
|
-
});
|
|
41
|
-
(0, fileUploadWorkerRegistry_1.registerFileUploadWorker)(this.fileWorker);
|
|
42
|
-
this.fileWorker.start();
|
|
49
|
+
this.fileCache = fileCache;
|
|
50
|
+
this.notifyUpload = notifyUpload;
|
|
43
51
|
}
|
|
44
52
|
// -- helpers --------------------------------------------------------
|
|
45
53
|
async request(path, init) {
|
|
46
|
-
|
|
47
|
-
const response = await fetch(url, {
|
|
48
|
-
...init,
|
|
49
|
-
headers: {
|
|
50
|
-
Authorization: `Bearer ${this.apiKey}`,
|
|
51
|
-
...init.headers,
|
|
52
|
-
},
|
|
53
|
-
});
|
|
54
|
-
return response;
|
|
54
|
+
return donobuApiFetch(this.baseUrl, this.apiKey, path, init);
|
|
55
55
|
}
|
|
56
56
|
async jsonRequest(path, method, body) {
|
|
57
57
|
return this.request(path, {
|
|
@@ -264,30 +264,15 @@ class FlowsPersistenceDonobuApi {
|
|
|
264
264
|
return buffer;
|
|
265
265
|
}
|
|
266
266
|
/**
|
|
267
|
-
* Writes bytes to the local file cache and returns immediately. The
|
|
268
|
-
*
|
|
269
|
-
* {@link FileUploadWorker}
|
|
270
|
-
*
|
|
267
|
+
* Writes bytes to the local file cache and returns immediately. The actual
|
|
268
|
+
* HTTP upload to the Donobu API runs asynchronously in a
|
|
269
|
+
* {@link FileUploadWorker} (owned by this process or, under `donobu test`, by
|
|
270
|
+
* the CLI parent) with retry + backoff. Same-machine reads during the upload
|
|
271
|
+
* window hit the local cache (no 404).
|
|
271
272
|
*/
|
|
272
273
|
async setFlowFile(flowId, fileId, fileBytes) {
|
|
273
274
|
await this.fileCache.writePending(flowId, fileId, fileBytes);
|
|
274
|
-
this.
|
|
275
|
-
}
|
|
276
|
-
/**
|
|
277
|
-
* Performs the actual HTTP PUT against the Donobu API. Called by the
|
|
278
|
-
* {@link FileUploadWorker} when draining the upload queue. Not part of
|
|
279
|
-
* the public {@link FlowsPersistence} surface — callers go through
|
|
280
|
-
* {@link setFlowFile}.
|
|
281
|
-
*/
|
|
282
|
-
async uploadFlowFileViaHttp(flowId, fileId, fileBytes) {
|
|
283
|
-
const response = await this.request(`/v1/flows/${encodeURIComponent(flowId)}/files/${encodeURIComponent(fileId)}`, {
|
|
284
|
-
method: 'PUT',
|
|
285
|
-
headers: { 'Content-Type': 'application/octet-stream' },
|
|
286
|
-
body: new Uint8Array(fileBytes),
|
|
287
|
-
});
|
|
288
|
-
if (!response.ok) {
|
|
289
|
-
throw new Error(`Failed to set flow file: ${response.status} ${response.statusText}`);
|
|
290
|
-
}
|
|
275
|
+
this.notifyUpload();
|
|
291
276
|
}
|
|
292
277
|
// -- Browser state -------------------------------------------------
|
|
293
278
|
async setBrowserState(flowId, browserState) {
|
|
@@ -312,4 +297,86 @@ class FlowsPersistenceDonobuApi {
|
|
|
312
297
|
}
|
|
313
298
|
}
|
|
314
299
|
exports.FlowsPersistenceDonobuApi = FlowsPersistenceDonobuApi;
|
|
300
|
+
// ---------------------------------------------------------------------------
|
|
301
|
+
// Donobu API transport + upload subsystem (composition root)
|
|
302
|
+
// ---------------------------------------------------------------------------
|
|
303
|
+
// These free functions own all construction of the Donobu cloud upload
|
|
304
|
+
// subsystem (cache + background uploader) and of the persistence client
|
|
305
|
+
// itself. Keeping them out of the class means the persistence is a plain
|
|
306
|
+
// read/write client over an injected cache + notify hook, with no knowledge of
|
|
307
|
+
// ownership policy, env, or worker lifecycle.
|
|
308
|
+
/** Authenticated fetch against the Donobu API. Shared by the persistence
|
|
309
|
+
* client (reads/writes) and the standalone upload transport below. */
|
|
310
|
+
async function donobuApiFetch(baseUrl, apiKey, path, init) {
|
|
311
|
+
return fetch(`${baseUrl}${path}`, {
|
|
312
|
+
...init,
|
|
313
|
+
headers: {
|
|
314
|
+
Authorization: `Bearer ${apiKey}`,
|
|
315
|
+
...init.headers,
|
|
316
|
+
},
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* PUTs file bytes to the Donobu API. This is the upload transport handed to the
|
|
321
|
+
* {@link FileUploadWorker}; it's standalone (takes creds, not a persistence
|
|
322
|
+
* instance) precisely so a worker can be built without a
|
|
323
|
+
* {@link FlowsPersistenceDonobuApi} — which is what lets the CLI parent own an
|
|
324
|
+
* uploader without constructing a read/write client it never uses.
|
|
325
|
+
*/
|
|
326
|
+
async function uploadDonobuFlowFile(baseUrl, apiKey, flowId, fileId, fileBytes) {
|
|
327
|
+
const response = await donobuApiFetch(baseUrl, apiKey, `/v1/flows/${encodeURIComponent(flowId)}/files/${encodeURIComponent(fileId)}`, {
|
|
328
|
+
method: 'PUT',
|
|
329
|
+
headers: { 'Content-Type': 'application/octet-stream' },
|
|
330
|
+
body: new Uint8Array(fileBytes),
|
|
331
|
+
});
|
|
332
|
+
if (!response.ok) {
|
|
333
|
+
throw new Error(`Failed to set flow file: ${response.status} ${response.statusText}`);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
/** The local upload cache for the Donobu platform, rooted under the shared
|
|
337
|
+
* working directory so every process (CLI parent + Playwright workers) agrees
|
|
338
|
+
* on one location. */
|
|
339
|
+
function donobuUploadCache(baseWorkingDirectory = MiscUtils_1.MiscUtils.baseWorkingDirectory()) {
|
|
340
|
+
return new FileUploadCache_1.FileUploadCache(path_1.default.join(baseWorkingDirectory, 'uploads', PLATFORM_LABEL));
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Builds, registers, and starts the background uploader that drains `cache` to
|
|
344
|
+
* the Donobu API. Whatever process is responsible for pushing artifacts to the
|
|
345
|
+
* cloud calls this exactly once; drain + stop it later via
|
|
346
|
+
* `shutdownFileUploadWorkers()`.
|
|
347
|
+
*/
|
|
348
|
+
function startDonobuCloudUploader(cache, baseUrl, apiKey, pollIntervalMs) {
|
|
349
|
+
const worker = new FileUploadWorker_1.FileUploadWorker({
|
|
350
|
+
cache,
|
|
351
|
+
platformLabel: PLATFORM_LABEL,
|
|
352
|
+
upload: (flowId, fileId, bytes) => uploadDonobuFlowFile(baseUrl, apiKey, flowId, fileId, bytes),
|
|
353
|
+
pollIntervalMs,
|
|
354
|
+
});
|
|
355
|
+
(0, fileUploadWorkerRegistry_1.registerFileUploadWorker)(worker);
|
|
356
|
+
worker.start();
|
|
357
|
+
return worker;
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Composition root for the Donobu persistence client. Builds the shared upload
|
|
361
|
+
* cache and, when this process owns draining, the background uploader; returns
|
|
362
|
+
* a {@link FlowsPersistenceDonobuApi} wired to notify that uploader on writes.
|
|
363
|
+
*/
|
|
364
|
+
function createDonobuFlowsPersistence(options) {
|
|
365
|
+
const cache = donobuUploadCache(options.baseWorkingDirectory);
|
|
366
|
+
let notifyUpload = () => { };
|
|
367
|
+
if (options.ownsUploadDraining) {
|
|
368
|
+
const worker = startDonobuCloudUploader(cache, options.baseUrl, options.apiKey, options.pollIntervalMs);
|
|
369
|
+
notifyUpload = () => worker.notify();
|
|
370
|
+
}
|
|
371
|
+
return new FlowsPersistenceDonobuApi(options.baseUrl, options.apiKey, cache, notifyUpload);
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* Starts ONLY the Donobu cloud uploader (no read/write client) for a process
|
|
375
|
+
* that owns draining but never reads or writes flows itself — the long-lived
|
|
376
|
+
* `donobu` CLI parent. The worker registers globally; drain it later via
|
|
377
|
+
* `shutdownFileUploadWorkers()`.
|
|
378
|
+
*/
|
|
379
|
+
function startDonobuParentUploader(baseUrl, apiKey, pollIntervalMs, baseWorkingDirectory) {
|
|
380
|
+
startDonobuCloudUploader(donobuUploadCache(baseWorkingDirectory), baseUrl, apiKey, pollIntervalMs);
|
|
381
|
+
}
|
|
315
382
|
//# sourceMappingURL=FlowsPersistenceDonobuApi.js.map
|
|
@@ -2,6 +2,22 @@ import type { EnvPick } from 'env-struct';
|
|
|
2
2
|
import type { env } from '../../envVars';
|
|
3
3
|
import { PersistencePluginRegistry } from '../PersistencePlugin';
|
|
4
4
|
import type { FlowsPersistence } from './FlowsPersistence';
|
|
5
|
+
/** Credentials for the DONOBU cloud layer, resolved from the environment. */
|
|
6
|
+
export interface DonobuCloudConfig {
|
|
7
|
+
readonly baseUrl: string;
|
|
8
|
+
readonly apiKey: string;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Resolves the DONOBU cloud persistence config, or null if it isn't active for
|
|
12
|
+
* this environment (missing credentials, or `DONOBU` not in
|
|
13
|
+
* `PERSISTENCE_PRIORITY`). Single source of truth for "is DONOBU cloud on?",
|
|
14
|
+
* shared by {@link FlowsPersistenceRegistryImpl.fromEnvironment} (which builds
|
|
15
|
+
* the read/write client) and the `donobu` CLI (which starts the parent-owned
|
|
16
|
+
* uploader). The credential precedence — primary `DONOBU_API_KEY`, then the
|
|
17
|
+
* persistence-only `DONOBU_PERSISTENCE_API_KEY` — matches the inference
|
|
18
|
+
* fallback chain in DonobuFlowsManager.createGptClient.
|
|
19
|
+
*/
|
|
20
|
+
export declare function resolveDonobuCloudConfig(environ: EnvPick<typeof env, 'DONOBU_API_BASE_URL' | 'DONOBU_API_KEY' | 'DONOBU_PERSISTENCE_API_KEY' | 'PERSISTENCE_PRIORITY'>): DonobuCloudConfig | null;
|
|
5
21
|
/**
|
|
6
22
|
* A persistence layer paired with the `PERSISTENCE_PRIORITY` key it was
|
|
7
23
|
* created from. The key is shared across the flows/tests/suites registries:
|
|
@@ -46,7 +62,7 @@ export declare class FlowsPersistenceRegistryImpl implements FlowsPersistenceReg
|
|
|
46
62
|
* Creates an instance by reading environment variables and eagerly constructing
|
|
47
63
|
* all applicable persistence layers.
|
|
48
64
|
*/
|
|
49
|
-
static fromEnvironment(environ: EnvPick<typeof env, 'DONOBU_API_BASE_URL' | 'DONOBU_API_KEY' | 'DONOBU_PERSISTENCE_API_KEY' | 'PERSISTENCE_PRIORITY'>, persistencePlugins?: PersistencePluginRegistry): Promise<FlowsPersistenceRegistryImpl>;
|
|
65
|
+
static fromEnvironment(environ: EnvPick<typeof env, 'DONOBU_API_BASE_URL' | 'DONOBU_API_KEY' | 'DONOBU_PERSISTENCE_API_KEY' | 'DONOBU_UPLOADS_OWNED_BY_PARENT' | 'PERSISTENCE_PRIORITY'>, persistencePlugins?: PersistencePluginRegistry): Promise<FlowsPersistenceRegistryImpl>;
|
|
50
66
|
/**
|
|
51
67
|
* Returns the primary persistence layer.
|
|
52
68
|
*/
|
|
@@ -1,11 +1,32 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.FlowsPersistenceRegistryImpl = void 0;
|
|
4
|
+
exports.resolveDonobuCloudConfig = resolveDonobuCloudConfig;
|
|
4
5
|
const DonobuSqliteDb_1 = require("../DonobuSqliteDb");
|
|
5
6
|
const PersistencePlugin_1 = require("../PersistencePlugin");
|
|
6
7
|
const FlowsPersistenceDonobuApi_1 = require("./FlowsPersistenceDonobuApi");
|
|
7
8
|
const FlowsPersistenceSqlite_1 = require("./FlowsPersistenceSqlite");
|
|
8
9
|
const FlowsPersistenceVolatile_1 = require("./FlowsPersistenceVolatile");
|
|
10
|
+
/**
|
|
11
|
+
* Resolves the DONOBU cloud persistence config, or null if it isn't active for
|
|
12
|
+
* this environment (missing credentials, or `DONOBU` not in
|
|
13
|
+
* `PERSISTENCE_PRIORITY`). Single source of truth for "is DONOBU cloud on?",
|
|
14
|
+
* shared by {@link FlowsPersistenceRegistryImpl.fromEnvironment} (which builds
|
|
15
|
+
* the read/write client) and the `donobu` CLI (which starts the parent-owned
|
|
16
|
+
* uploader). The credential precedence — primary `DONOBU_API_KEY`, then the
|
|
17
|
+
* persistence-only `DONOBU_PERSISTENCE_API_KEY` — matches the inference
|
|
18
|
+
* fallback chain in DonobuFlowsManager.createGptClient.
|
|
19
|
+
*/
|
|
20
|
+
function resolveDonobuCloudConfig(environ) {
|
|
21
|
+
const baseUrl = environ.data.DONOBU_API_BASE_URL;
|
|
22
|
+
const apiKey = environ.data.DONOBU_API_KEY ?? environ.data.DONOBU_PERSISTENCE_API_KEY;
|
|
23
|
+
if (!baseUrl ||
|
|
24
|
+
!apiKey ||
|
|
25
|
+
!environ.data.PERSISTENCE_PRIORITY.includes('DONOBU')) {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
return { baseUrl, apiKey };
|
|
29
|
+
}
|
|
9
30
|
/**
|
|
10
31
|
* A factory class for creating FlowsPersistence instances. Persistence layers are constructed
|
|
11
32
|
* eagerly at creation time and reused across all subsequent calls.
|
|
@@ -25,22 +46,24 @@ class FlowsPersistenceRegistryImpl {
|
|
|
25
46
|
* all applicable persistence layers.
|
|
26
47
|
*/
|
|
27
48
|
static async fromEnvironment(environ, persistencePlugins = new PersistencePlugin_1.PersistencePluginRegistry()) {
|
|
28
|
-
const
|
|
29
|
-
// Persistence credential resolution: prefer the primary DONOBU_API_KEY
|
|
30
|
-
// (which also drives the inference fallback chain in
|
|
31
|
-
// DonobuFlowsManager.createGptClient), then fall back to the
|
|
32
|
-
// persistence-only DONOBU_PERSISTENCE_API_KEY for hosts that route
|
|
33
|
-
// inference through an explicit gpt-config and don't want their
|
|
34
|
-
// persistence credential to short-circuit the agent lookup.
|
|
35
|
-
const donobuApiKey = environ.data.DONOBU_API_KEY ?? environ.data.DONOBU_PERSISTENCE_API_KEY;
|
|
49
|
+
const donobuCloud = resolveDonobuCloudConfig(environ);
|
|
36
50
|
const layers = [];
|
|
37
51
|
for (const key of environ.data.PERSISTENCE_PRIORITY) {
|
|
38
52
|
switch (key) {
|
|
39
53
|
case 'DONOBU':
|
|
40
|
-
if (
|
|
54
|
+
if (donobuCloud) {
|
|
41
55
|
layers.push({
|
|
42
56
|
key,
|
|
43
|
-
|
|
57
|
+
// Read the internal coordination flag here, at the env boundary,
|
|
58
|
+
// rather than inside the persistence client. When `donobu test`
|
|
59
|
+
// set it on this (worker) process, the worker runs producer-only
|
|
60
|
+
// and the CLI parent owns draining; otherwise this process owns
|
|
61
|
+
// its own uploader (Studio, or raw `playwright test`).
|
|
62
|
+
persistence: (0, FlowsPersistenceDonobuApi_1.createDonobuFlowsPersistence)({
|
|
63
|
+
baseUrl: donobuCloud.baseUrl,
|
|
64
|
+
apiKey: donobuCloud.apiKey,
|
|
65
|
+
ownsUploadDraining: !environ.data.DONOBU_UPLOADS_OWNED_BY_PARENT,
|
|
66
|
+
}),
|
|
44
67
|
});
|
|
45
68
|
}
|
|
46
69
|
break;
|