donobu 5.52.1 → 5.52.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/cli/donobu-cli.js +58 -0
  2. package/dist/envVars.d.ts +2 -0
  3. package/dist/envVars.js +9 -0
  4. package/dist/esm/cli/donobu-cli.js +58 -0
  5. package/dist/esm/envVars.d.ts +2 -0
  6. package/dist/esm/envVars.js +9 -0
  7. package/dist/esm/lib/test/testExtension.js +5 -2
  8. package/dist/esm/lib/test/utils/donobuTestStack.js +1 -1
  9. package/dist/esm/main.d.ts +2 -0
  10. package/dist/esm/main.js +1 -1
  11. package/dist/esm/managers/AdminApiController.d.ts +1 -1
  12. package/dist/esm/managers/DonobuStack.d.ts +2 -0
  13. package/dist/esm/managers/DonobuStack.js +1 -1
  14. package/dist/esm/managers/PageInspector.js +43 -15
  15. package/dist/esm/persistence/files/FileUploadCache.d.ts +16 -0
  16. package/dist/esm/persistence/files/FileUploadCache.js +30 -0
  17. package/dist/esm/persistence/files/FileUploadWorker.d.ts +28 -4
  18. package/dist/esm/persistence/files/FileUploadWorker.js +31 -7
  19. package/dist/esm/persistence/files/fileUploadWorkerRegistry.d.ts +9 -0
  20. package/dist/esm/persistence/files/fileUploadWorkerRegistry.js +16 -0
  21. package/dist/esm/persistence/flows/FlowsPersistenceDonobuApi.d.ts +49 -13
  22. package/dist/esm/persistence/flows/FlowsPersistenceDonobuApi.js +106 -39
  23. package/dist/esm/persistence/flows/FlowsPersistenceRegistry.d.ts +17 -1
  24. package/dist/esm/persistence/flows/FlowsPersistenceRegistry.js +33 -10
  25. package/dist/lib/test/testExtension.js +5 -2
  26. package/dist/lib/test/utils/donobuTestStack.js +1 -1
  27. package/dist/main.d.ts +2 -0
  28. package/dist/main.js +1 -1
  29. package/dist/managers/AdminApiController.d.ts +1 -1
  30. package/dist/managers/DonobuStack.d.ts +2 -0
  31. package/dist/managers/DonobuStack.js +1 -1
  32. package/dist/managers/PageInspector.js +43 -15
  33. package/dist/persistence/files/FileUploadCache.d.ts +16 -0
  34. package/dist/persistence/files/FileUploadCache.js +30 -0
  35. package/dist/persistence/files/FileUploadWorker.d.ts +28 -4
  36. package/dist/persistence/files/FileUploadWorker.js +31 -7
  37. package/dist/persistence/files/fileUploadWorkerRegistry.d.ts +9 -0
  38. package/dist/persistence/files/fileUploadWorkerRegistry.js +16 -0
  39. package/dist/persistence/flows/FlowsPersistenceDonobuApi.d.ts +49 -13
  40. package/dist/persistence/flows/FlowsPersistenceDonobuApi.js +106 -39
  41. package/dist/persistence/flows/FlowsPersistenceRegistry.d.ts +17 -1
  42. package/dist/persistence/flows/FlowsPersistenceRegistry.js +33 -10
  43. package/package.json +1 -1
@@ -64,6 +64,9 @@ const envVars_1 = require("../envVars");
64
64
  const gptClients_1 = require("../lib/test/fixtures/gptClients");
65
65
  const donobuTestStack_1 = require("../lib/test/utils/donobuTestStack");
66
66
  const triageTestFailure_1 = require("../lib/test/utils/triageTestFailure");
67
+ const fileUploadWorkerRegistry_1 = require("../persistence/files/fileUploadWorkerRegistry");
68
+ const FlowsPersistenceDonobuApi_1 = require("../persistence/flows/FlowsPersistenceDonobuApi");
69
+ const FlowsPersistenceRegistry_1 = require("../persistence/flows/FlowsPersistenceRegistry");
67
70
  const merge_1 = require("../reporter/merge");
68
71
  const model_1 = require("../reporter/model");
69
72
  const render_1 = require("../reporter/render");
@@ -1749,6 +1752,19 @@ function parseHealArgs(args) {
1749
1752
  * 3. Collects failure evidence and generates treatment plans.
1750
1753
  * 4. Optionally runs an auto-heal retry if plans suggest it.
1751
1754
  */
1755
+ /**
1756
+ * Idle re-scan interval for the parent-owned upload daemon. The producers are
1757
+ * the Playwright worker processes, so the daemon can't be woken by an
1758
+ * in-process notify() — it polls the shared cache instead.
1759
+ */
1760
+ const PARENT_UPLOAD_POLL_INTERVAL_MS = 1_000;
1761
+ /**
1762
+ * How long the parent waits for artifact uploads to flush after the Playwright
1763
+ * run finishes. Generous because — unlike the old per-worker drain — this runs
1764
+ * in the long-lived CLI parent, free of Playwright's fixture-teardown timeout,
1765
+ * so it blocks nothing critical and truncates no traces.
1766
+ */
1767
+ const PARENT_UPLOAD_DRAIN_TIMEOUT_MS = 60_000;
1752
1768
  async function runTestCommand(cliArgs) {
1753
1769
  const { options, playwrightArgs } = parseTestCommandArgs(cliArgs);
1754
1770
  const playwrightOutputDir = resolvePlaywrightOutputDir(playwrightArgs);
@@ -1793,6 +1809,27 @@ async function runTestCommand(cliArgs) {
1793
1809
  if (effectiveOptions.autoHeal) {
1794
1810
  envOverrides.DONOBU_AUTO_HEAL_ORCHESTRATED = '1';
1795
1811
  }
1812
+ // Own artifact uploads from THIS long-lived parent rather than the ephemeral,
1813
+ // force-killable Playwright workers. When a cloud uploader starts here, flip
1814
+ // the workers to producer-only mode: they cache artifact bytes locally but
1815
+ // run no uploader, so worker teardown can never block on an in-flight upload.
1816
+ // The parent's uploader drains the shared cache continuously during the run
1817
+ // and flushes whatever's left in the finally below.
1818
+ const donobuCloud = (0, FlowsPersistenceRegistry_1.resolveDonobuCloudConfig)(envVars_1.env.pick('DONOBU_API_BASE_URL', 'DONOBU_API_KEY', 'DONOBU_PERSISTENCE_API_KEY', 'PERSISTENCE_PRIORITY'));
1819
+ const parentOwnsUploads = donobuCloud !== null;
1820
+ if (donobuCloud) {
1821
+ // Start just the uploader — no read/write client, since the parent never
1822
+ // reads or writes flows itself.
1823
+ (0, FlowsPersistenceDonobuApi_1.startDonobuParentUploader)(donobuCloud.baseUrl, donobuCloud.apiKey, PARENT_UPLOAD_POLL_INTERVAL_MS);
1824
+ // Set on process.env (not just this run's overrides) so EVERY spawned
1825
+ // Playwright child inherits it — the initial run and the auto-heal rerun
1826
+ // alike — via runPlaywright's `{ ...process.env, ...overrides }`. Our own
1827
+ // parent uploader was already started above, so this late mutation (which
1828
+ // only affects freshly-parsed child envs) doesn't touch it.
1829
+ process.env.DONOBU_UPLOADS_OWNED_BY_PARENT = '1';
1830
+ Logger_1.appLogger.debug(`donobu test: this process owns cloud artifact uploads; Playwright ` +
1831
+ `workers will cache bytes and exit without waiting on uploads.`);
1832
+ }
1796
1833
  const reporterSetup = await ensureJsonReporter(playwrightArgs, {
1797
1834
  playwrightOutputDir,
1798
1835
  });
@@ -1895,6 +1932,27 @@ async function runTestCommand(cliArgs) {
1895
1932
  return exitCode;
1896
1933
  }
1897
1934
  finally {
1935
+ // Flush any artifact uploads this parent owns before returning. main()
1936
+ // calls process.exit() right after runTestCommand resolves, which would
1937
+ // otherwise abandon in-flight uploads. Runs on every return path. Skipped
1938
+ // when no parent uploader was started (local-only runs).
1939
+ if (parentOwnsUploads) {
1940
+ try {
1941
+ const result = await (0, fileUploadWorkerRegistry_1.shutdownFileUploadWorkers)(PARENT_UPLOAD_DRAIN_TIMEOUT_MS);
1942
+ if (result.drained) {
1943
+ Logger_1.appLogger.debug('donobu test: all artifact uploads flushed.');
1944
+ }
1945
+ else {
1946
+ Logger_1.appLogger.warn(`donobu test: ${result.totalRemaining} artifact upload(s) did not ` +
1947
+ `finish within ` +
1948
+ `${Math.round(PARENT_UPLOAD_DRAIN_TIMEOUT_MS / 1000)}s; they ` +
1949
+ `remain on disk and resume on the next \`donobu\` run.`);
1950
+ }
1951
+ }
1952
+ catch (error) {
1953
+ Logger_1.appLogger.warn('donobu test: error draining parent uploads.', error);
1954
+ }
1955
+ }
1898
1956
  // Clean up the JSON artifacts Donobu created for its own internal use.
1899
1957
  // Runs after every return path (early-exit when tests pass / auto-heal
1900
1958
  // disabled, and the post-heal path) so we don't leave the state file or
package/dist/envVars.d.ts CHANGED
@@ -75,6 +75,7 @@ export declare const env: Env<{
75
75
  EXPERIMENTAL_FEATURES_ENABLED: z.ZodDefault<z.ZodBoolean>;
76
76
  DONOBU_API_KEY: z.ZodOptional<z.ZodString>;
77
77
  DONOBU_PERSISTENCE_API_KEY: z.ZodOptional<z.ZodString>;
78
+ DONOBU_UPLOADS_OWNED_BY_PARENT: z.ZodOptional<z.ZodCodec<z.ZodString, z.ZodBoolean>>;
78
79
  GOOGLE_CLOUD_STORAGE_BUCKET: z.ZodOptional<z.ZodString>;
79
80
  SCREENSHOT_TIMEOUT_MS: z.ZodDefault<z.ZodNumber>;
80
81
  SELF_HEAL_TESTS_ENABLED: z.ZodOptional<z.ZodString>;
@@ -129,6 +130,7 @@ export declare const env: Env<{
129
130
  AWS_SECRET_ACCESS_KEY?: string | undefined;
130
131
  DONOBU_API_KEY?: string | undefined;
131
132
  DONOBU_PERSISTENCE_API_KEY?: string | undefined;
133
+ DONOBU_UPLOADS_OWNED_BY_PARENT?: boolean | undefined;
132
134
  GOOGLE_CLOUD_STORAGE_BUCKET?: string | undefined;
133
135
  SELF_HEAL_TESTS_ENABLED?: string | undefined;
134
136
  PROXY_SERVER?: string | undefined;
package/dist/envVars.js CHANGED
@@ -200,6 +200,15 @@ Intended for hosts (e.g. the Donobu Studio desktop app) that drive AI
200
200
  inference through an explicit gpt-config / flow-runner agent and want
201
201
  to enable Donobu Cloud persistence without their persistence credential
202
202
  silently overriding the user's flow-runner pick.`),
203
+ DONOBU_UPLOADS_OWNED_BY_PARENT: v4_1.z.stringbool().optional()
204
+ .describe(`Internal coordination flag set by the \`donobu test\` CLI on the Playwright
205
+ processes it spawns. When true, cloud persistence runs in producer-only
206
+ mode inside the Playwright workers: artifact bytes are still written to the
207
+ local upload cache, but the background upload-to-cloud worker is NOT started
208
+ there. Instead the long-lived CLI parent owns a single uploader that drains
209
+ the shared cache continuously and flushes it after the run. This keeps the
210
+ ephemeral, force-killable Playwright workers out of the upload critical path
211
+ so worker teardown never blocks on in-flight uploads.`),
203
212
  GOOGLE_CLOUD_STORAGE_BUCKET: v4_1.z.string().optional()
204
213
  .describe(`Directs Donobu flows to be persisted using this Google Cloud Storage
205
214
  bucket.`),
@@ -64,6 +64,9 @@ const envVars_1 = require("../envVars");
64
64
  const gptClients_1 = require("../lib/test/fixtures/gptClients");
65
65
  const donobuTestStack_1 = require("../lib/test/utils/donobuTestStack");
66
66
  const triageTestFailure_1 = require("../lib/test/utils/triageTestFailure");
67
+ const fileUploadWorkerRegistry_1 = require("../persistence/files/fileUploadWorkerRegistry");
68
+ const FlowsPersistenceDonobuApi_1 = require("../persistence/flows/FlowsPersistenceDonobuApi");
69
+ const FlowsPersistenceRegistry_1 = require("../persistence/flows/FlowsPersistenceRegistry");
67
70
  const merge_1 = require("../reporter/merge");
68
71
  const model_1 = require("../reporter/model");
69
72
  const render_1 = require("../reporter/render");
@@ -1749,6 +1752,19 @@ function parseHealArgs(args) {
1749
1752
  * 3. Collects failure evidence and generates treatment plans.
1750
1753
  * 4. Optionally runs an auto-heal retry if plans suggest it.
1751
1754
  */
1755
+ /**
1756
+ * Idle re-scan interval for the parent-owned upload daemon. The producers are
1757
+ * the Playwright worker processes, so the daemon can't be woken by an
1758
+ * in-process notify() — it polls the shared cache instead.
1759
+ */
1760
+ const PARENT_UPLOAD_POLL_INTERVAL_MS = 1_000;
1761
+ /**
1762
+ * How long the parent waits for artifact uploads to flush after the Playwright
1763
+ * run finishes. Generous because — unlike the old per-worker drain — this runs
1764
+ * in the long-lived CLI parent, free of Playwright's fixture-teardown timeout,
1765
+ * so it blocks nothing critical and truncates no traces.
1766
+ */
1767
+ const PARENT_UPLOAD_DRAIN_TIMEOUT_MS = 60_000;
1752
1768
  async function runTestCommand(cliArgs) {
1753
1769
  const { options, playwrightArgs } = parseTestCommandArgs(cliArgs);
1754
1770
  const playwrightOutputDir = resolvePlaywrightOutputDir(playwrightArgs);
@@ -1793,6 +1809,27 @@ async function runTestCommand(cliArgs) {
1793
1809
  if (effectiveOptions.autoHeal) {
1794
1810
  envOverrides.DONOBU_AUTO_HEAL_ORCHESTRATED = '1';
1795
1811
  }
1812
+ // Own artifact uploads from THIS long-lived parent rather than the ephemeral,
1813
+ // force-killable Playwright workers. When a cloud uploader starts here, flip
1814
+ // the workers to producer-only mode: they cache artifact bytes locally but
1815
+ // run no uploader, so worker teardown can never block on an in-flight upload.
1816
+ // The parent's uploader drains the shared cache continuously during the run
1817
+ // and flushes whatever's left in the finally below.
1818
+ const donobuCloud = (0, FlowsPersistenceRegistry_1.resolveDonobuCloudConfig)(envVars_1.env.pick('DONOBU_API_BASE_URL', 'DONOBU_API_KEY', 'DONOBU_PERSISTENCE_API_KEY', 'PERSISTENCE_PRIORITY'));
1819
+ const parentOwnsUploads = donobuCloud !== null;
1820
+ if (donobuCloud) {
1821
+ // Start just the uploader — no read/write client, since the parent never
1822
+ // reads or writes flows itself.
1823
+ (0, FlowsPersistenceDonobuApi_1.startDonobuParentUploader)(donobuCloud.baseUrl, donobuCloud.apiKey, PARENT_UPLOAD_POLL_INTERVAL_MS);
1824
+ // Set on process.env (not just this run's overrides) so EVERY spawned
1825
+ // Playwright child inherits it — the initial run and the auto-heal rerun
1826
+ // alike — via runPlaywright's `{ ...process.env, ...overrides }`. Our own
1827
+ // parent uploader was already started above, so this late mutation (which
1828
+ // only affects freshly-parsed child envs) doesn't touch it.
1829
+ process.env.DONOBU_UPLOADS_OWNED_BY_PARENT = '1';
1830
+ Logger_1.appLogger.debug(`donobu test: this process owns cloud artifact uploads; Playwright ` +
1831
+ `workers will cache bytes and exit without waiting on uploads.`);
1832
+ }
1796
1833
  const reporterSetup = await ensureJsonReporter(playwrightArgs, {
1797
1834
  playwrightOutputDir,
1798
1835
  });
@@ -1895,6 +1932,27 @@ async function runTestCommand(cliArgs) {
1895
1932
  return exitCode;
1896
1933
  }
1897
1934
  finally {
1935
+ // Flush any artifact uploads this parent owns before returning. main()
1936
+ // calls process.exit() right after runTestCommand resolves, which would
1937
+ // otherwise abandon in-flight uploads. Runs on every return path. Skipped
1938
+ // when no parent uploader was started (local-only runs).
1939
+ if (parentOwnsUploads) {
1940
+ try {
1941
+ const result = await (0, fileUploadWorkerRegistry_1.shutdownFileUploadWorkers)(PARENT_UPLOAD_DRAIN_TIMEOUT_MS);
1942
+ if (result.drained) {
1943
+ Logger_1.appLogger.debug('donobu test: all artifact uploads flushed.');
1944
+ }
1945
+ else {
1946
+ Logger_1.appLogger.warn(`donobu test: ${result.totalRemaining} artifact upload(s) did not ` +
1947
+ `finish within ` +
1948
+ `${Math.round(PARENT_UPLOAD_DRAIN_TIMEOUT_MS / 1000)}s; they ` +
1949
+ `remain on disk and resume on the next \`donobu\` run.`);
1950
+ }
1951
+ }
1952
+ catch (error) {
1953
+ Logger_1.appLogger.warn('donobu test: error draining parent uploads.', error);
1954
+ }
1955
+ }
1898
1956
  // Clean up the JSON artifacts Donobu created for its own internal use.
1899
1957
  // Runs after every return path (early-exit when tests pass / auto-heal
1900
1958
  // disabled, and the post-heal path) so we don't leave the state file or
@@ -75,6 +75,7 @@ export declare const env: Env<{
75
75
  EXPERIMENTAL_FEATURES_ENABLED: z.ZodDefault<z.ZodBoolean>;
76
76
  DONOBU_API_KEY: z.ZodOptional<z.ZodString>;
77
77
  DONOBU_PERSISTENCE_API_KEY: z.ZodOptional<z.ZodString>;
78
+ DONOBU_UPLOADS_OWNED_BY_PARENT: z.ZodOptional<z.ZodCodec<z.ZodString, z.ZodBoolean>>;
78
79
  GOOGLE_CLOUD_STORAGE_BUCKET: z.ZodOptional<z.ZodString>;
79
80
  SCREENSHOT_TIMEOUT_MS: z.ZodDefault<z.ZodNumber>;
80
81
  SELF_HEAL_TESTS_ENABLED: z.ZodOptional<z.ZodString>;
@@ -129,6 +130,7 @@ export declare const env: Env<{
129
130
  AWS_SECRET_ACCESS_KEY?: string | undefined;
130
131
  DONOBU_API_KEY?: string | undefined;
131
132
  DONOBU_PERSISTENCE_API_KEY?: string | undefined;
133
+ DONOBU_UPLOADS_OWNED_BY_PARENT?: boolean | undefined;
132
134
  GOOGLE_CLOUD_STORAGE_BUCKET?: string | undefined;
133
135
  SELF_HEAL_TESTS_ENABLED?: string | undefined;
134
136
  PROXY_SERVER?: string | undefined;
@@ -200,6 +200,15 @@ Intended for hosts (e.g. the Donobu Studio desktop app) that drive AI
200
200
  inference through an explicit gpt-config / flow-runner agent and want
201
201
  to enable Donobu Cloud persistence without their persistence credential
202
202
  silently overriding the user's flow-runner pick.`),
203
+ DONOBU_UPLOADS_OWNED_BY_PARENT: v4_1.z.stringbool().optional()
204
+ .describe(`Internal coordination flag set by the \`donobu test\` CLI on the Playwright
205
+ processes it spawns. When true, cloud persistence runs in producer-only
206
+ mode inside the Playwright workers: artifact bytes are still written to the
207
+ local upload cache, but the background upload-to-cloud worker is NOT started
208
+ there. Instead the long-lived CLI parent owns a single uploader that drains
209
+ the shared cache continuously and flushes it after the run. This keeps the
210
+ ephemeral, force-killable Playwright workers out of the upload critical path
211
+ so worker teardown never blocks on in-flight uploads.`),
203
212
  GOOGLE_CLOUD_STORAGE_BUCKET: v4_1.z.string().optional()
204
213
  .describe(`Directs Donobu flows to be persisted using this Google Cloud Storage
205
214
  bucket.`),
@@ -333,8 +333,11 @@ exports.test = test_1.test.extend({
333
333
  }
334
334
  let initialActive;
335
335
  try {
336
- const status = await (0, fileUploadWorkerRegistry_1.getFileUploadAggregateStatus)();
337
- initialActive = status.totalPending + status.totalInFlight;
336
+ // Owned count only: pending + this process's own in-flight claims.
337
+ // Claims orphaned by other/crashed processes are excluded — this
338
+ // process can neither upload nor release them, so waiting on them
339
+ // would wedge worker teardown until the grace timeout force-kills it.
340
+ initialActive = await (0, fileUploadWorkerRegistry_1.getOwnedActiveUploadCount)();
338
341
  }
339
342
  catch (err) {
340
343
  Logger_1.appLogger.warn(`donobuFileUploadDrainGuard: failed to read upload status; ` +
@@ -20,7 +20,7 @@ function getEnvSnapshot() {
20
20
  }
21
21
  async function getOrCreateDonobuStack() {
22
22
  if (!donobuStack) {
23
- donobuStack = await (0, DonobuStack_1.setupDonobuStack)('LOCAL', ControlPanel_1.NoOpControlPanelFactory, new EnvPersistenceVolatile_1.EnvPersistenceVolatile(getEnvSnapshot()), envVars_1.env.pick('ANTHROPIC_API_KEY', 'ANTHROPIC_MODEL_NAME', 'AWS_ACCESS_KEY_ID', 'AWS_BEDROCK_MODEL_NAME', 'AWS_SECRET_ACCESS_KEY', 'BASE64_GPT_CONFIG', 'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'DONOBU_API_BASE_URL', 'DONOBU_API_KEY', 'DONOBU_PERSISTENCE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY', 'GOOGLE_GENERATIVE_AI_MODEL_NAME', 'OLLAMA_API_URL', 'OLLAMA_MODEL_NAME', 'OPENAI_API_KEY', 'OPENAI_API_MODEL_NAME', 'PERSISTENCE_PRIORITY'));
23
+ donobuStack = await (0, DonobuStack_1.setupDonobuStack)('LOCAL', ControlPanel_1.NoOpControlPanelFactory, new EnvPersistenceVolatile_1.EnvPersistenceVolatile(getEnvSnapshot()), envVars_1.env.pick('ANTHROPIC_API_KEY', 'ANTHROPIC_MODEL_NAME', 'AWS_ACCESS_KEY_ID', 'AWS_BEDROCK_MODEL_NAME', 'AWS_SECRET_ACCESS_KEY', 'BASE64_GPT_CONFIG', 'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'DONOBU_API_BASE_URL', 'DONOBU_API_KEY', 'DONOBU_PERSISTENCE_API_KEY', 'DONOBU_UPLOADS_OWNED_BY_PARENT', 'GOOGLE_GENERATIVE_AI_API_KEY', 'GOOGLE_GENERATIVE_AI_MODEL_NAME', 'OLLAMA_API_URL', 'OLLAMA_MODEL_NAME', 'OPENAI_API_KEY', 'OPENAI_API_MODEL_NAME', 'PERSISTENCE_PRIORITY'));
24
24
  return donobuStack;
25
25
  }
26
26
  else {
@@ -105,6 +105,7 @@ export declare function startDonobuServer({ port, controlPanelHost, environ, }?:
105
105
  AWS_SECRET_ACCESS_KEY: z.ZodOptional<z.ZodString>;
106
106
  DONOBU_API_KEY: z.ZodOptional<z.ZodString>;
107
107
  DONOBU_PERSISTENCE_API_KEY: z.ZodOptional<z.ZodString>;
108
+ DONOBU_UPLOADS_OWNED_BY_PARENT: z.ZodOptional<z.ZodCodec<z.ZodString, z.ZodBoolean>>;
108
109
  }, {
109
110
  BASE64_GPT_CONFIG?: string | undefined;
110
111
  BROWSERBASE_API_KEY?: string | undefined;
@@ -125,6 +126,7 @@ export declare function startDonobuServer({ port, controlPanelHost, environ, }?:
125
126
  AWS_SECRET_ACCESS_KEY?: string | undefined;
126
127
  DONOBU_API_KEY?: string | undefined;
127
128
  DONOBU_PERSISTENCE_API_KEY?: string | undefined;
129
+ DONOBU_UPLOADS_OWNED_BY_PARENT?: boolean | undefined;
128
130
  }> | undefined;
129
131
  }): Promise<void>;
130
132
  //# sourceMappingURL=main.d.ts.map
package/dist/esm/main.js CHANGED
@@ -111,7 +111,7 @@ const DEFAULT_PORT = 31000;
111
111
  * Starts a Donobu API server at the given port. The server assumes that the
112
112
  * Playwright browsers have been installed.
113
113
  */
114
- async function startDonobuServer({ port = DEFAULT_PORT, controlPanelHost = ControlPanel_1.NoOpControlPanelFactory, environ = envVars_1.env.pick('ANTHROPIC_API_KEY', 'ANTHROPIC_MODEL_NAME', 'AWS_ACCESS_KEY_ID', 'AWS_BEDROCK_MODEL_NAME', 'AWS_SECRET_ACCESS_KEY', 'BASE64_GPT_CONFIG', 'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'DONOBU_DEPLOYMENT_ENVIRONMENT', 'DONOBU_API_BASE_URL', 'DONOBU_API_KEY', 'DONOBU_PERSISTENCE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY', 'GOOGLE_GENERATIVE_AI_MODEL_NAME', 'OLLAMA_API_URL', 'OLLAMA_MODEL_NAME', 'OPENAI_API_KEY', 'OPENAI_API_MODEL_NAME', 'PERSISTENCE_PRIORITY'), } = {}) {
114
+ async function startDonobuServer({ port = DEFAULT_PORT, controlPanelHost = ControlPanel_1.NoOpControlPanelFactory, environ = envVars_1.env.pick('ANTHROPIC_API_KEY', 'ANTHROPIC_MODEL_NAME', 'AWS_ACCESS_KEY_ID', 'AWS_BEDROCK_MODEL_NAME', 'AWS_SECRET_ACCESS_KEY', 'BASE64_GPT_CONFIG', 'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'DONOBU_DEPLOYMENT_ENVIRONMENT', 'DONOBU_API_BASE_URL', 'DONOBU_API_KEY', 'DONOBU_PERSISTENCE_API_KEY', 'DONOBU_UPLOADS_OWNED_BY_PARENT', 'GOOGLE_GENERATIVE_AI_API_KEY', 'GOOGLE_GENERATIVE_AI_MODEL_NAME', 'OLLAMA_API_URL', 'OLLAMA_MODEL_NAME', 'OPENAI_API_KEY', 'OPENAI_API_MODEL_NAME', 'PERSISTENCE_PRIORITY'), } = {}) {
115
115
  try {
116
116
  const adminController = await AdminApiController_1.AdminApiController.create(environ.data.DONOBU_DEPLOYMENT_ENVIRONMENT ?? 'LOCAL', controlPanelHost, environ);
117
117
  await adminController.start(port);
@@ -59,7 +59,7 @@ export declare class AdminApiController {
59
59
  * - no checking for flow ownership is performed, as all flows are considered owned by the
60
60
  * local environment.
61
61
  */
62
- static create(donobuDeploymentEnvironment: DonobuDeploymentEnvironment, controlPanelFactory: ControlPanelFactory, environ: EnvPick<typeof env, 'ANTHROPIC_API_KEY' | 'ANTHROPIC_MODEL_NAME' | 'AWS_ACCESS_KEY_ID' | 'AWS_BEDROCK_MODEL_NAME' | 'AWS_SECRET_ACCESS_KEY' | 'BASE64_GPT_CONFIG' | 'BROWSERBASE_API_KEY' | 'BROWSERBASE_PROJECT_ID' | 'DONOBU_API_BASE_URL' | 'DONOBU_API_KEY' | 'DONOBU_PERSISTENCE_API_KEY' | 'GOOGLE_GENERATIVE_AI_API_KEY' | 'GOOGLE_GENERATIVE_AI_MODEL_NAME' | 'OLLAMA_API_URL' | 'OLLAMA_MODEL_NAME' | 'OPENAI_API_KEY' | 'OPENAI_API_MODEL_NAME' | 'PERSISTENCE_PRIORITY'>): Promise<AdminApiController>;
62
+ static create(donobuDeploymentEnvironment: DonobuDeploymentEnvironment, controlPanelFactory: ControlPanelFactory, environ: EnvPick<typeof env, 'ANTHROPIC_API_KEY' | 'ANTHROPIC_MODEL_NAME' | 'AWS_ACCESS_KEY_ID' | 'AWS_BEDROCK_MODEL_NAME' | 'AWS_SECRET_ACCESS_KEY' | 'BASE64_GPT_CONFIG' | 'BROWSERBASE_API_KEY' | 'BROWSERBASE_PROJECT_ID' | 'DONOBU_API_BASE_URL' | 'DONOBU_API_KEY' | 'DONOBU_PERSISTENCE_API_KEY' | 'DONOBU_UPLOADS_OWNED_BY_PARENT' | 'GOOGLE_GENERATIVE_AI_API_KEY' | 'GOOGLE_GENERATIVE_AI_MODEL_NAME' | 'OLLAMA_API_URL' | 'OLLAMA_MODEL_NAME' | 'OPENAI_API_KEY' | 'OPENAI_API_MODEL_NAME' | 'PERSISTENCE_PRIORITY'>): Promise<AdminApiController>;
63
63
  private constructor();
64
64
  /**
65
65
  * Binds the API/web-asset server to `port` and resolves once the socket is
@@ -61,6 +61,7 @@ export declare function setupDonobuStack(donobuDeploymentEnvironment: DonobuDepl
61
61
  AWS_SECRET_ACCESS_KEY: import("zod/v4").ZodOptional<import("zod/v4").ZodString>;
62
62
  DONOBU_API_KEY: import("zod/v4").ZodOptional<import("zod/v4").ZodString>;
63
63
  DONOBU_PERSISTENCE_API_KEY: import("zod/v4").ZodOptional<import("zod/v4").ZodString>;
64
+ DONOBU_UPLOADS_OWNED_BY_PARENT: import("zod/v4").ZodOptional<import("zod/v4").ZodCodec<import("zod/v4").ZodString, import("zod/v4").ZodBoolean>>;
64
65
  }, {
65
66
  BASE64_GPT_CONFIG?: string | undefined;
66
67
  BROWSERBASE_API_KEY?: string | undefined;
@@ -80,5 +81,6 @@ export declare function setupDonobuStack(donobuDeploymentEnvironment: DonobuDepl
80
81
  AWS_SECRET_ACCESS_KEY?: string | undefined;
81
82
  DONOBU_API_KEY?: string | undefined;
82
83
  DONOBU_PERSISTENCE_API_KEY?: string | undefined;
84
+ DONOBU_UPLOADS_OWNED_BY_PARENT?: boolean | undefined;
83
85
  }>): Promise<DonobuStack>;
84
86
  //# sourceMappingURL=DonobuStack.d.ts.map
@@ -33,7 +33,7 @@ const ToolRegistry_1 = require("./ToolRegistry");
33
33
  * then having this snapshot is relevant so that tests can use normal
34
34
  * environment variables.
35
35
  */
36
- async function setupDonobuStack(donobuDeploymentEnvironment, controlPanelFactory, envPersistenceVolatile, environ = envVars_1.env.pick('ANTHROPIC_API_KEY', 'ANTHROPIC_MODEL_NAME', 'AWS_ACCESS_KEY_ID', 'AWS_BEDROCK_MODEL_NAME', 'AWS_SECRET_ACCESS_KEY', 'BASE64_GPT_CONFIG', 'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'DONOBU_API_BASE_URL', 'DONOBU_API_KEY', 'DONOBU_PERSISTENCE_API_KEY', 'GOOGLE_GENERATIVE_AI_API_KEY', 'GOOGLE_GENERATIVE_AI_MODEL_NAME', 'OLLAMA_API_URL', 'OLLAMA_MODEL_NAME', 'OPENAI_API_KEY', 'OPENAI_API_MODEL_NAME', 'PERSISTENCE_PRIORITY')) {
36
+ async function setupDonobuStack(donobuDeploymentEnvironment, controlPanelFactory, envPersistenceVolatile, environ = envVars_1.env.pick('ANTHROPIC_API_KEY', 'ANTHROPIC_MODEL_NAME', 'AWS_ACCESS_KEY_ID', 'AWS_BEDROCK_MODEL_NAME', 'AWS_SECRET_ACCESS_KEY', 'BASE64_GPT_CONFIG', 'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'DONOBU_API_BASE_URL', 'DONOBU_API_KEY', 'DONOBU_PERSISTENCE_API_KEY', 'DONOBU_UPLOADS_OWNED_BY_PARENT', 'GOOGLE_GENERATIVE_AI_API_KEY', 'GOOGLE_GENERATIVE_AI_MODEL_NAME', 'OLLAMA_API_URL', 'OLLAMA_MODEL_NAME', 'OPENAI_API_KEY', 'OPENAI_API_MODEL_NAME', 'PERSISTENCE_PRIORITY')) {
37
37
  const loadedPlugins = await loadDefaultPlugins();
38
38
  const resolvedToolRegistry = await (0, ToolRegistry_1.createDefaultToolRegistry)(loadedPlugins.tools);
39
39
  const persistencePlugins = new PersistencePlugin_1.PersistencePluginRegistry(loadedPlugins.persistencePlugins);
@@ -933,24 +933,22 @@ class PageInspector {
933
933
  if (document.scrollingElement) {
934
934
  maybeAddScrollable(document.scrollingElement);
935
935
  }
936
- // 2) Iterate and assign numbers
937
- uniqueElements.forEach((element) => {
938
- if (element === document.scrollingElement) {
939
- // Special-case: always keep the root scrolling element
940
- element.setAttribute(interactableAttribute, offset.toString());
941
- offset++;
942
- return; // skip the usual checks
943
- }
944
- else if (element.hasAttribute(interactableAttribute)) {
945
- // Skip if this element already carries a value (e.g. assigned via <label>)
946
- return;
936
+ /**
937
+ * Run the visibility / enabled / top-most checks on a single element and,
938
+ * if they pass, assign it the next interactable number. Returns `true` if
939
+ * the element (or, via the <label htmlFor> mapping, its associated control)
940
+ * was attributed.
941
+ */
942
+ function tryAttributeElement(element) {
943
+ if (element.hasAttribute(interactableAttribute)) {
944
+ return false;
947
945
  }
948
946
  const rect = element.getBoundingClientRect();
949
947
  const style = window.getComputedStyle(element);
950
948
  const visible = isElementVisible(rect, style) && isElementMoreThanHalfInViewport(rect);
951
949
  const enabled = isElementEnabled(element, style);
952
950
  if (!visible || !enabled) {
953
- return;
951
+ return false;
954
952
  }
955
953
  // Check a few probe points to make sure the element is top-most
956
954
  for (const pt of getPointsToCheck(rect)) {
@@ -959,9 +957,9 @@ class PageInspector {
959
957
  if (elToCheck === element) {
960
958
  element.setAttribute(interactableAttribute, offset.toString());
961
959
  offset++;
962
- return; // this element done
960
+ return true; // this element done
963
961
  }
964
- // Handle <label> -> control mapping
962
+ // Handle <label> -> control mapping (explicit `for`/`htmlFor`)
965
963
  if (elToCheck.tagName.toLowerCase() === 'label' &&
966
964
  elToCheck.htmlFor) {
967
965
  const forId = elToCheck.htmlFor;
@@ -972,11 +970,41 @@ class PageInspector {
972
970
  control.setAttribute(interactableAttribute, offset.toString());
973
971
  offset++;
974
972
  }
975
- return;
973
+ return true;
976
974
  }
977
975
  elToCheck = elToCheck.parentElement;
978
976
  }
979
977
  }
978
+ return false;
979
+ }
980
+ // 2) Iterate and assign numbers
981
+ uniqueElements.forEach((element) => {
982
+ if (element === document.scrollingElement) {
983
+ // Special-case: always keep the root scrolling element
984
+ element.setAttribute(interactableAttribute, offset.toString());
985
+ offset++;
986
+ return; // skip the usual checks
987
+ }
988
+ else if (element.hasAttribute(interactableAttribute)) {
989
+ // Skip if this element already carries a value (e.g. assigned via <label>)
990
+ return;
991
+ }
992
+ if (tryAttributeElement(element)) {
993
+ return;
994
+ }
995
+ // Fallback: the element is a visually-hidden native control (e.g. a 0x0,
996
+ // opacity-0, pointer-events:none <input>) wrapped in a styled <label>.
997
+ // This is the standard pattern for Ant Design Segmented/Radio/Checkbox/
998
+ // Switch and many other component libraries: the native input is hidden
999
+ // and the surrounding <label> is the real clickable surface. Since the
1000
+ // hidden control fails the visibility/enabled checks above, attribute the
1001
+ // wrapping <label> instead so the toggle is still annotated.
1002
+ const wrappingLabel = element.closest('label');
1003
+ if (wrappingLabel &&
1004
+ wrappingLabel !== element &&
1005
+ !wrappingLabel.hasAttribute(interactableAttribute)) {
1006
+ tryAttributeElement(wrappingLabel);
1007
+ }
980
1008
  });
981
1009
  return offset;
982
1010
  }
@@ -94,6 +94,22 @@ export declare class FileUploadCache {
94
94
  * active work even though `listPending` excludes it.
95
95
  */
96
96
  countActive(): Promise<number>;
97
+ /**
98
+ * Like {@link countActive}, but scoped to the work THIS worker is
99
+ * responsible for finishing before its process exits: files still
100
+ * `.pending` (which this worker will claim and upload) plus this worker's
101
+ * own `.uploading.<ownToken>` claims.
102
+ *
103
+ * Foreign `.uploading.<otherToken>` markers — in-flight claims owned by
104
+ * OTHER (possibly long-dead) processes — are deliberately EXCLUDED. A live
105
+ * peer is responsible for finishing its own claims, and claims orphaned by
106
+ * a crashed peer are recovered by {@link reclaimStaleClaims} on the next
107
+ * long-lived process. Counting them here is exactly what made an
108
+ * end-of-worker drain block for the full timeout on a crashed peer's
109
+ * orphaned claim (which this process can neither upload nor release),
110
+ * wedging Playwright worker teardown until it was force-killed.
111
+ */
112
+ countOwnedActive(ownToken: string): Promise<number>;
97
113
  /**
98
114
  * Returns a breakdown of marker counts on disk. Used by the admin
99
115
  * "uploads status" endpoint so the desktop app can show the user how
@@ -178,6 +178,36 @@ class FileUploadCache {
178
178
  const counts = await this.getCounts();
179
179
  return counts.pending + counts.inFlight;
180
180
  }
181
+ /**
182
+ * Like {@link countActive}, but scoped to the work THIS worker is
183
+ * responsible for finishing before its process exits: files still
184
+ * `.pending` (which this worker will claim and upload) plus this worker's
185
+ * own `.uploading.<ownToken>` claims.
186
+ *
187
+ * Foreign `.uploading.<otherToken>` markers — in-flight claims owned by
188
+ * OTHER (possibly long-dead) processes — are deliberately EXCLUDED. A live
189
+ * peer is responsible for finishing its own claims, and claims orphaned by
190
+ * a crashed peer are recovered by {@link reclaimStaleClaims} on the next
191
+ * long-lived process. Counting them here is exactly what made an
192
+ * end-of-worker drain block for the full timeout on a crashed peer's
193
+ * orphaned claim (which this process can neither upload nor release),
194
+ * wedging Playwright worker teardown until it was force-killed.
195
+ */
196
+ async countOwnedActive(ownToken) {
197
+ const ownSuffix = `${UPLOADING_PREFIX}${ownToken}`;
198
+ let active = 0;
199
+ const flowDirs = await this.safeReaddir(this.platformDir);
200
+ for (const encFlow of flowDirs) {
201
+ const flowDir = path_1.default.join(this.platformDir, encFlow);
202
+ const names = await this.safeReaddir(flowDir);
203
+ for (const name of names) {
204
+ if (name.endsWith(PENDING_SUFFIX) || name.endsWith(ownSuffix)) {
205
+ active += 1;
206
+ }
207
+ }
208
+ }
209
+ return active;
210
+ }
181
211
  /**
182
212
  * Returns a breakdown of marker counts on disk. Used by the admin
183
213
  * "uploads status" endpoint so the desktop app can show the user how
@@ -11,6 +11,15 @@ export interface FileUploadWorkerOptions {
11
11
  readonly upload: UploadFn;
12
12
  readonly platformLabel: string;
13
13
  readonly maxAttempts?: number;
14
+ /**
15
+ * When set, the idle loop re-scans the cache for new `.pending` files every
16
+ * `pollIntervalMs` even without a {@link notify} call. Required when the
17
+ * producer runs in a DIFFERENT process than this worker (the parent-owned
18
+ * upload daemon draining a cache that Playwright worker processes write to),
19
+ * because `notify()` is in-process only. Same-process workers leave this
20
+ * unset and stay purely notify-driven.
21
+ */
22
+ readonly pollIntervalMs?: number;
14
23
  }
15
24
  /**
16
25
  * Drains pending file uploads from a {@link FileUploadCache}, uploading each
@@ -26,6 +35,7 @@ export declare class FileUploadWorker {
26
35
  private readonly upload;
27
36
  private readonly platformLabel;
28
37
  private readonly maxAttempts;
38
+ private readonly pollIntervalMs?;
29
39
  private readonly token;
30
40
  private workerLoop;
31
41
  private staleScanTimer;
@@ -37,6 +47,13 @@ export declare class FileUploadWorker {
37
47
  constructor(options: FileUploadWorkerOptions);
38
48
  start(): void;
39
49
  stop(): Promise<void>;
50
+ /**
51
+ * Number of uploads THIS worker still owes before its process can exit:
52
+ * pending files plus this worker's own in-flight claims, excluding claims
53
+ * orphaned by other/crashed processes. Mirrors what {@link flush} waits on,
54
+ * so callers can log an accurate "N still syncing" count at shutdown.
55
+ */
56
+ countOwnedActive(): Promise<number>;
40
57
  /**
41
58
  * Returns a snapshot of the worker's current state for surfacing to the
42
59
  * UI (e.g. "N uploads still pending — wait or quit anyway?"). Reads the
@@ -50,10 +67,17 @@ export declare class FileUploadWorker {
50
67
  */
51
68
  notify(): void;
52
69
  /**
53
- * Returns once the queue has drained or the timeout has elapsed. Callers
54
- * use this on graceful shutdown to push outstanding uploads before exit.
55
- * "Drained" means zero files in `.pending` AND zero in `.uploading.*` —
56
- * a file mid-retry is still active work.
70
+ * Returns once THIS worker's queue has drained or the timeout has elapsed.
71
+ * Callers use this on graceful shutdown to push outstanding uploads before
72
+ * the process exits.
73
+ *
74
+ * "Drained" means zero files this worker still owes: nothing in `.pending`
75
+ * (which this worker will claim and upload) and none of this worker's own
76
+ * `.uploading.<token>` claims (a file mid-retry is still active work).
77
+ * Foreign `.uploading.<otherToken>` claims left by other or crashed
78
+ * processes are NOT waited on — see {@link FileUploadCache.countOwnedActive}
79
+ * — because this process can neither upload nor release them; doing so
80
+ * would block teardown for the full timeout on a dead peer's orphaned claim.
57
81
  */
58
82
  flush(timeoutMs: number): Promise<{
59
83
  drained: boolean;
@@ -27,6 +27,7 @@ class FileUploadWorker {
27
27
  this.upload = options.upload;
28
28
  this.platformLabel = options.platformLabel;
29
29
  this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
30
+ this.pollIntervalMs = options.pollIntervalMs;
30
31
  let wakeResolver;
31
32
  this.wakePromise = new Promise((resolve) => {
32
33
  wakeResolver = resolve;
@@ -75,6 +76,15 @@ class FileUploadWorker {
75
76
  await this.workerLoop;
76
77
  this.workerLoop = null;
77
78
  }
79
+ /**
80
+ * Number of uploads THIS worker still owes before its process can exit:
81
+ * pending files plus this worker's own in-flight claims, excluding claims
82
+ * orphaned by other/crashed processes. Mirrors what {@link flush} waits on,
83
+ * so callers can log an accurate "N still syncing" count at shutdown.
84
+ */
85
+ async countOwnedActive() {
86
+ return this.cache.countOwnedActive(this.token);
87
+ }
78
88
  /**
79
89
  * Returns a snapshot of the worker's current state for surfacing to the
80
90
  * UI (e.g. "N uploads still pending — wait or quit anyway?"). Reads the
@@ -102,21 +112,28 @@ class FileUploadWorker {
102
112
  resolve?.();
103
113
  }
104
114
  /**
105
- * Returns once the queue has drained or the timeout has elapsed. Callers
106
- * use this on graceful shutdown to push outstanding uploads before exit.
107
- * "Drained" means zero files in `.pending` AND zero in `.uploading.*` —
108
- * a file mid-retry is still active work.
115
+ * Returns once THIS worker's queue has drained or the timeout has elapsed.
116
+ * Callers use this on graceful shutdown to push outstanding uploads before
117
+ * the process exits.
118
+ *
119
+ * "Drained" means zero files this worker still owes: nothing in `.pending`
120
+ * (which this worker will claim and upload) and none of this worker's own
121
+ * `.uploading.<token>` claims (a file mid-retry is still active work).
122
+ * Foreign `.uploading.<otherToken>` claims left by other or crashed
123
+ * processes are NOT waited on — see {@link FileUploadCache.countOwnedActive}
124
+ * — because this process can neither upload nor release them; doing so
125
+ * would block teardown for the full timeout on a dead peer's orphaned claim.
109
126
  */
110
127
  async flush(timeoutMs) {
111
128
  const deadline = Date.now() + timeoutMs;
112
129
  while (Date.now() < deadline) {
113
- const active = await this.cache.countActive();
130
+ const active = await this.cache.countOwnedActive(this.token);
114
131
  if (active === 0) {
115
132
  return { drained: true, remaining: 0 };
116
133
  }
117
134
  await this.sleep(Math.min(200, Math.max(50, deadline - Date.now())));
118
135
  }
119
- const remaining = await this.cache.countActive();
136
+ const remaining = await this.cache.countOwnedActive(this.token);
120
137
  return { drained: remaining === 0, remaining };
121
138
  }
122
139
  // ── internals ────────────────────────────────────────────────────
@@ -124,7 +141,14 @@ class FileUploadWorker {
124
141
  while (!this.stopping) {
125
142
  const pending = await this.cache.listPending();
126
143
  if (pending.length === 0) {
127
- await Promise.race([this.wakePromise, this.stopSignal]);
144
+ // Wake on an in-process notify() or on stop. When draining a cache
145
+ // that OTHER processes write to (the parent-owned daemon), also wake
146
+ // on a poll timer, since notify() can't cross the process boundary.
147
+ const idleWaiters = [this.wakePromise, this.stopSignal];
148
+ if (this.pollIntervalMs !== undefined) {
149
+ idleWaiters.push(this.sleep(this.pollIntervalMs));
150
+ }
151
+ await Promise.race(idleWaiters);
128
152
  continue;
129
153
  }
130
154
  for (const entry of pending) {
@@ -30,6 +30,15 @@ export declare function unregisterFileUploadWorker(worker: FileUploadWorker): vo
30
30
  * time.
31
31
  */
32
32
  export declare function getFileUploadAggregateStatus(): Promise<FileUploadAggregateStatus>;
33
+ /**
34
+ * Total uploads still owed by every registered worker — pending files plus
35
+ * each worker's own in-flight claims, excluding claims orphaned by other or
36
+ * crashed processes (which this process can neither upload nor release). This
37
+ * is what a graceful shutdown actually waits on, so use it (not the global
38
+ * {@link getFileUploadAggregateStatus}, which counts foreign claims too) when
39
+ * deciding whether a drain is needed and reporting how much is left.
40
+ */
41
+ export declare function getOwnedActiveUploadCount(): Promise<number>;
33
42
  /**
34
43
  * Drains and stops every registered worker. Used during graceful shutdown.
35
44
  * Returns a summary across all workers so the caller can log how many