humanish 0.101.0 → 0.103.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +4 -2
  2. package/dist/affordance.d.ts +2 -0
  3. package/dist/affordance.js +3 -0
  4. package/dist/affordance.js.map +1 -1
  5. package/dist/browser-control-client.d.ts +5 -1
  6. package/dist/browser-control-client.js +45 -5
  7. package/dist/browser-control-client.js.map +1 -1
  8. package/dist/browser-control-dispatcher.d.ts +6 -1
  9. package/dist/browser-control-dispatcher.js +41 -3
  10. package/dist/browser-control-dispatcher.js.map +1 -1
  11. package/dist/browser-control-protocol.d.ts +101 -0
  12. package/dist/browser-control-protocol.js +38 -8
  13. package/dist/browser-control-protocol.js.map +1 -1
  14. package/dist/browser-control-recording-transfer.d.ts +4 -0
  15. package/dist/browser-control-recording-transfer.js +42 -0
  16. package/dist/browser-control-recording-transfer.js.map +1 -0
  17. package/dist/browser-control-transport.d.ts +3 -0
  18. package/dist/browser-control-transport.js +23 -2
  19. package/dist/browser-control-transport.js.map +1 -1
  20. package/dist/computer-use.d.ts +9 -0
  21. package/dist/computer-use.js +36 -8
  22. package/dist/computer-use.js.map +1 -1
  23. package/dist/cua-actor-lab.d.ts +6 -2
  24. package/dist/cua-actor-lab.js +19 -7
  25. package/dist/cua-actor-lab.js.map +1 -1
  26. package/dist/cua-desktop-lane.d.ts +1 -1
  27. package/dist/cua-speech.d.ts +13 -0
  28. package/dist/cua-speech.js +2 -0
  29. package/dist/cua-speech.js.map +1 -0
  30. package/dist/desktop-recorder.d.ts +39 -0
  31. package/dist/desktop-recorder.js +220 -0
  32. package/dist/desktop-recorder.js.map +1 -0
  33. package/dist/desktop-recording-artifact.d.ts +5 -0
  34. package/dist/desktop-recording-artifact.js +54 -0
  35. package/dist/desktop-recording-artifact.js.map +1 -0
  36. package/dist/desktop-recording-types.d.ts +24 -0
  37. package/dist/desktop-recording-types.js +14 -0
  38. package/dist/desktop-recording-types.js.map +1 -0
  39. package/dist/desktop-session.js +1 -0
  40. package/dist/desktop-session.js.map +1 -1
  41. package/dist/doctor-lab.js +1 -1
  42. package/dist/doctor-lab.js.map +1 -1
  43. package/dist/e2b-cua-desktop.js +77 -44
  44. package/dist/e2b-cua-desktop.js.map +1 -1
  45. package/dist/e2b-cua-provisioning.d.ts +4 -1
  46. package/dist/e2b-cua-provisioning.js +8 -2
  47. package/dist/e2b-cua-provisioning.js.map +1 -1
  48. package/dist/e2b-desktop-executor.js +3 -0
  49. package/dist/e2b-desktop-executor.js.map +1 -1
  50. package/dist/e2b-desktop-launch.d.ts +19 -1
  51. package/dist/e2b-desktop-launch.js.map +1 -1
  52. package/dist/e2b-desktop-media.d.ts +14 -0
  53. package/dist/e2b-desktop-media.js +58 -0
  54. package/dist/e2b-desktop-media.js.map +1 -0
  55. package/dist/e2b-desktop-recording.d.ts +16 -0
  56. package/dist/e2b-desktop-recording.js +162 -0
  57. package/dist/e2b-desktop-recording.js.map +1 -0
  58. package/dist/export-bundle.js +2 -0
  59. package/dist/export-bundle.js.map +1 -1
  60. package/dist/export.js +19 -3
  61. package/dist/export.js.map +1 -1
  62. package/dist/guest-bootstrap.d.ts +6 -2
  63. package/dist/guest-bootstrap.js +22 -9
  64. package/dist/guest-bootstrap.js.map +1 -1
  65. package/dist/guest-desktop-executor.js +1 -0
  66. package/dist/guest-desktop-executor.js.map +1 -1
  67. package/dist/guest-desktop-media.d.ts +33 -0
  68. package/dist/guest-desktop-media.js +281 -0
  69. package/dist/guest-desktop-media.js.map +1 -0
  70. package/dist/guest-media-config.d.ts +15 -0
  71. package/dist/guest-media-config.js +8 -0
  72. package/dist/guest-media-config.js.map +1 -0
  73. package/dist/guest-media-worker.d.ts +1 -0
  74. package/dist/guest-media-worker.js +319 -0
  75. package/dist/guest-media-worker.js.map +1 -0
  76. package/dist/guest-runtime-desktop.d.ts +5 -1
  77. package/dist/guest-runtime-desktop.js +62 -3
  78. package/dist/guest-runtime-desktop.js.map +1 -1
  79. package/dist/guest-runtime-main.js +3 -2
  80. package/dist/guest-runtime-main.js.map +1 -1
  81. package/dist/guest-runtime.d.ts +8 -2
  82. package/dist/guest-runtime.js +3 -2
  83. package/dist/guest-runtime.js.map +1 -1
  84. package/dist/lab-config.d.ts +6 -2
  85. package/dist/lab-config.js +26 -4
  86. package/dist/lab-config.js.map +1 -1
  87. package/dist/lab-summary.js +1 -1
  88. package/dist/lab-summary.js.map +1 -1
  89. package/dist/local-firecracker-desktop.d.ts +10 -1
  90. package/dist/local-firecracker-desktop.js +16 -2
  91. package/dist/local-firecracker-desktop.js.map +1 -1
  92. package/dist/local-firecracker-study.js +29 -3
  93. package/dist/local-firecracker-study.js.map +1 -1
  94. package/dist/local-runtime-config.js +5 -2
  95. package/dist/local-runtime-config.js.map +1 -1
  96. package/dist/local-runtime-release.d.ts +2 -0
  97. package/dist/local-runtime-release.js +15 -0
  98. package/dist/local-runtime-release.js.map +1 -1
  99. package/dist/local-runtime.d.ts +1 -0
  100. package/dist/local-runtime.js +11 -9
  101. package/dist/local-runtime.js.map +1 -1
  102. package/dist/observer-app.html +9 -9
  103. package/dist/observer-serve.js +1 -1
  104. package/dist/observer-serve.js.map +1 -1
  105. package/dist/observer.d.ts +2 -2
  106. package/dist/observer.js +106 -27
  107. package/dist/observer.js.map +1 -1
  108. package/dist/program.js +3 -2
  109. package/dist/program.js.map +1 -1
  110. package/dist/restricted-codex-participant-policy.d.ts +2 -1
  111. package/dist/restricted-codex-participant-policy.js +9 -6
  112. package/dist/restricted-codex-participant-policy.js.map +1 -1
  113. package/dist/restricted-codex-participant-run.js +2 -1
  114. package/dist/restricted-codex-participant-run.js.map +1 -1
  115. package/dist/restricted-codex-participant.d.ts +2 -0
  116. package/dist/restricted-codex-participant.js +20 -7
  117. package/dist/restricted-codex-participant.js.map +1 -1
  118. package/dist/run.d.ts +12 -7
  119. package/dist/run.js +55 -15
  120. package/dist/run.js.map +1 -1
  121. package/dist/selected-output-paths.d.ts +3 -0
  122. package/dist/selected-output-paths.js +29 -17
  123. package/dist/selected-output-paths.js.map +1 -1
  124. package/dist/study-analysis-engine.d.ts +1 -1
  125. package/dist/study-analysis-engine.js +53 -10
  126. package/dist/study-analysis-engine.js.map +1 -1
  127. package/docs/architecture/desktop-recording.md +65 -0
  128. package/docs/architecture/local-browser-runtime.md +3 -2
  129. package/docs/architecture/participant-media.md +90 -0
  130. package/docs/contracts/run-bundle.md +27 -0
  131. package/docs/contracts/schemas.md +1 -1
  132. package/docs/contracts/study-analysis.md +14 -0
  133. package/docs/goals/current.md +13 -9
  134. package/docs/ramp/README.md +11 -1
  135. package/docs/release/0.102.0-participant-media.md +43 -0
  136. package/docs/release/0.103.0-desktop-recording.md +29 -0
  137. package/package.json +5 -4
  138. package/skills/humanish/SKILL.md +14 -5
@@ -1,7 +1,13 @@
1
- import type { Duplex } from "node:stream";
1
+ import type { Duplex, Readable } from "node:stream";
2
2
  import type { CuaExecutor } from "./computer-use.js";
3
+ import type { GuestMediaConfig } from "./guest-media-config.js";
4
+ import type { DesktopRecordingConfig, DesktopRecordingMetadata } from "./desktop-recording-types.js";
3
5
  export interface GuestRuntimeDesktop {
4
6
  executor: CuaExecutor;
7
+ finishRecording?(): Promise<{
8
+ metadata: DesktopRecordingMetadata;
9
+ stream: Readable;
10
+ }>;
5
11
  close(): Promise<{
6
12
  complete: boolean;
7
13
  }>;
@@ -12,7 +18,7 @@ export interface GuestRuntimeOptions {
12
18
  signal: AbortSignal;
13
19
  /** Fixed private supervision channel, not browser-control output. */
14
20
  marker(value: "A" | "R"): void;
15
- createDesktop(signal: AbortSignal, onTerminal: () => void, initialUrl?: string): Promise<GuestRuntimeDesktop>;
21
+ createDesktop(signal: AbortSignal, onTerminal: () => void, initialUrl?: string, media?: GuestMediaConfig, recording?: DesktopRecordingConfig): Promise<GuestRuntimeDesktop>;
16
22
  }
17
23
  /** One admitted desktop and one dispatcher. The owner retains physical teardown. */
18
24
  export declare function runGuestRuntime(options: GuestRuntimeOptions): Promise<{
@@ -67,7 +67,7 @@ export async function runGuestRuntime(options) {
67
67
  clearTimeout(timer);
68
68
  timer = setTimeout(() => { void close(); }, guestReadyTimeoutMs(reader.initialUrl));
69
69
  options.marker("A");
70
- preparing = options.createDesktop(authority, terminal, reader.initialUrl);
70
+ preparing = options.createDesktop(authority, terminal, reader.initialUrl, reader.media, reader.recording);
71
71
  desktop = await Promise.race([preparing, new Promise((_, reject) => {
72
72
  const stop = () => reject(new CuaExecutorError("session_revoked", "not_dispatched"));
73
73
  authority.addEventListener("abort", stop, { once: true });
@@ -79,7 +79,8 @@ export async function runGuestRuntime(options) {
79
79
  throw new CuaExecutorError("session_revoked", "not_dispatched");
80
80
  reader.handoff();
81
81
  dispatcher = attachBrowserControlDispatcher({ transport: options.transport, identity, executor: desktop.executor,
82
- authoritySignal: authority, isAuthorized: () => !authority.aborted });
82
+ authoritySignal: authority, isAuthorized: () => !authority.aborted,
83
+ ...(desktop.finishRecording ? { finishRecording: desktop.finishRecording } : {}) });
83
84
  // No await in this handoff: an immediate HELLO already has its sole receiver.
84
85
  options.transport.write(encodeGuestBootstrap(identity, true), error => { if (error)
85
86
  terminal(); });
@@ -1 +1 @@
1
- {"version":3,"file":"guest-runtime.js","sourceRoot":"","sources":["../src/guest-runtime.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,8BAA8B,EAAE,MAAM,iCAAiC,CAAC;AACjF,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC/H,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAY3D,oFAAoF;AACpF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAA4B;IAChE,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,IAAI,OAAwC,CAAC;IAC7C,IAAI,UAAyE,CAAC;IAC9E,IAAI,MAAwC,CAAC;IAC7C,IAAI,OAAmD,CAAC;IACxD,IAAI,SAAmD,CAAC;IACxD,IAAI,aAAsD,CAAC;IAC3D,MAAM,MAAM,GAAG,IAAI,OAAO,CAAwB,OAAO,CAAC,EAAE,GAAG,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3F,IAAI,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAChF,SAAS,QAAQ,KAAW,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC;IAC3C,SAAS,KAAK;QACZ,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;QAC5B,sEAAsE;QACtE,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC1C,MAAM,EAAE,KAAK,EAAE,CAAC;YAAC,UAAU,EAAE,KAAK,EAAE,CAAC;YAAC,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YAClE,IAAI,QAAQ,GAAG,IAAI,CAAC;YACpB,IAAI,QAAoC,CAAC;YACzC,MAAM,OAAO,GAAG,KAAK,IAAmB,EAAE;gBACxC,IAAI,CAAC;oBACH,IAAI,CAAC,OAAO,IAAI,SAAS;wBAAE,OAAO,GAAG,MAAM,SAAS,CAAC;oBACrD,IAAI,OAAO;wBAAE,QAAQ,GAAG,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC;gBAC3D,CAAC;gBAAC,MAAM,CAAC;oBAAC,QAAQ,GAAG,KAAK,CAAC;gBAAC,CAAC;YAC/B,CAAC,CAAC;YACF,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,GAAG,QAAQ,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1I,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,QAAQ,CAAC,CAAC;gBACvB,OAAO,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACtD,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACjF,2EAA2E;gBAC3E,IAAI,OAAO,CAAC,SAAS,CAAC,MAAM;oBAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACvE,MAAM,MAAM,GAAG,EAAE,QAAQ,EAAE,CAAC;gBAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YACrD,CAAC;YACD,OAAO,EAAE,QAAQ,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QACnC,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACxH,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,oBAAoB,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QACnG,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;QACvC,IAAI,SAAS,CAAC,OAAO;YAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;QACvF,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;QACpF,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACpB,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QAC1E,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBACxE,MAAM,IAAI,GAAG,GAAS,EAAE,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC,CAAC;gBAC3F,SAAS,CAAC,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1D,KAAK,SAAU,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAC5F,IAAI,SAAS,CAAC,OAAO;oBAAE,IAAI,EAAE,CAAC;YAChC,CAAC,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,SAAS,CAAC,OAAO;YAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;QACvF,MAAM,CAAC,OAAO,EAAE,CAAC;QACjB,UAAU,GAAG,8BAA8B,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC9G,eAAe,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,8EAA8E;QAC9E,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,KAAK,CAAC,EAAE,GAAG,IAAI,KAAK;YAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACnG,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QAC3B,IAAI,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,SAAS,CAAC,SAAS;YAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;QACtH,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,MAAM,KAAK,EAAE,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;AACjD,CAAC"}
1
+ {"version":3,"file":"guest-runtime.js","sourceRoot":"","sources":["../src/guest-runtime.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,8BAA8B,EAAE,MAAM,iCAAiC,CAAC;AACjF,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC/H,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAmB3D,oFAAoF;AACpF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAA4B;IAChE,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,IAAI,OAAwC,CAAC;IAC7C,IAAI,UAAyE,CAAC;IAC9E,IAAI,MAAwC,CAAC;IAC7C,IAAI,OAAmD,CAAC;IACxD,IAAI,SAAmD,CAAC;IACxD,IAAI,aAAsD,CAAC;IAC3D,MAAM,MAAM,GAAG,IAAI,OAAO,CAAwB,OAAO,CAAC,EAAE,GAAG,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3F,IAAI,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAChF,SAAS,QAAQ,KAAW,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC;IAC3C,SAAS,KAAK;QACZ,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;QAC5B,sEAAsE;QACtE,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC1C,MAAM,EAAE,KAAK,EAAE,CAAC;YAAC,UAAU,EAAE,KAAK,EAAE,CAAC;YAAC,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;YAClE,IAAI,QAAQ,GAAG,IAAI,CAAC;YACpB,IAAI,QAAoC,CAAC;YACzC,MAAM,OAAO,GAAG,KAAK,IAAmB,EAAE;gBACxC,IAAI,CAAC;oBACH,IAAI,CAAC,OAAO,IAAI,SAAS;wBAAE,OAAO,GAAG,MAAM,SAAS,CAAC;oBACrD,IAAI,OAAO;wBAAE,QAAQ,GAAG,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC;gBAC3D,CAAC;gBAAC,MAAM,CAAC;oBAAC,QAAQ,GAAG,KAAK,CAAC;gBAAC,CAAC;YAC/B,CAAC,CAAC;YACF,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,GAAG,QAAQ,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1I,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,QAAQ,CAAC,CAAC;gBACvB,OAAO,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACtD,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACjF,2EAA2E;gBAC3E,IAAI,OAAO,CAAC,SAAS,CAAC,MAAM;oBAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACvE,MAAM,MAAM,GAAG,EAAE,QAAQ,EAAE,CAAC;gBAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YACrD,CAAC;YACD,OAAO,EAAE,QAAQ,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QACnC,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACxH,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,oBAAoB,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QACnG,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;QACvC,IAAI,SAAS,CAAC,OAAO;YAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;QACvF,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;QACpF,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACpB,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1G,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;gBACxE,MAAM,IAAI,GAAG,GAAS,EAAE,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC,CAAC;gBAC3F,SAAS,CAAC,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1D,KAAK,SAAU,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAC5F,IAAI,SAAS,CAAC,OAAO;oBAAE,IAAI,EAAE,CAAC;YAChC,CAAC,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,SAAS,CAAC,OAAO;YAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;QACvF,MAAM,CAAC,OAAO,EAAE,CAAC;QACjB,UAAU,GAAG,8BAA8B,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC9G,eAAe,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC,SAAS,CAAC,OAAO;YAClE,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACtF,8EAA8E;QAC9E,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,KAAK,CAAC,EAAE,GAAG,IAAI,KAAK;YAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACnG,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QAC3B,IAAI,SAAS,CAAC,OAAO,IAAI,OAAO,CAAC,SAAS,CAAC,SAAS;YAAE,MAAM,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,CAAC;QACtH,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,MAAM,KAAK,EAAE,CAAC;QAAC,MAAM,KAAK,CAAC;IAAC,CAAC;AACjD,CAAC"}
@@ -495,13 +495,17 @@ export interface LabExecutionDesktop {
495
495
  fidelity?: LabDesktopFidelity;
496
496
  /** Synthetic media devices behind the browser's own permission prompt (#509). */
497
497
  media?: LabDesktopMedia;
498
+ /** Optional retained screen video. Capture is independent of participant media input. */
499
+ recording?: {
500
+ audio: boolean;
501
+ };
498
502
  }
499
503
  /**
500
504
  * A participant with a camera (#509): a property of the ENVIRONMENT, like the screen preset and
501
505
  * the browser, never support for any conferencing product. `camera.source: synthetic` generates
502
506
  * a test pattern in the sandbox with the image's own ffmpeg; a `.y4m` path on the host is
503
- * uploaded instead. Microphone source-file injection is not implemented and is rejected before
504
- * spend, including on custom templates. A template's own audio capabilities are separate.
507
+ * uploaded instead on hosted desktops. `microphone.source: speech` enables the participant's
508
+ * own spoken replies and listening through native audio devices. File microphones are unsupported.
505
509
  */
506
510
  export interface LabDesktopMedia {
507
511
  camera?: {
@@ -643,21 +643,33 @@ export function effectiveComputerUseLaneIds(config) {
643
643
  * topology may not identify the backend that is actually executing.
644
644
  */
645
645
  export function desktopMediaValidationReason(config, supportsMedia = routesToComputerUse(config)) {
646
+ if (config.execution?.desktop?.recording !== undefined
647
+ && (!supportsMedia || config.subject.topology === "shared-world" || config.subject.source === "local-app")) {
648
+ return "execution.desktop.recording is supported only on independent computer-use desktop lanes. Remove the declaration or select a supported route.";
649
+ }
646
650
  const media = config.execution?.desktop?.media;
647
651
  if (media === undefined)
648
652
  return undefined;
649
- if (media.microphone !== undefined) {
650
- return "execution.desktop.media.microphone.source injection is unsupported, including on custom templates. Humanish cannot deliver the declared microphone file; remove this declaration.";
653
+ if (media.microphone !== undefined && media.microphone.source !== "speech") {
654
+ return "execution.desktop.media.microphone.source must be speech. Microphone source-file injection is unsupported.";
651
655
  }
652
656
  if (config.subject.topology === "shared-world") {
653
657
  return "execution.desktop.media is unsupported on shared-world routes; declared capture devices would not be provisioned. Use independent computer-use browser lanes or remove the declaration.";
654
658
  }
655
659
  if (!supportsMedia || config.subject.source === "desktop-cli" || config.subject.source === "local-app") {
656
- return "execution.desktop.media is supported only on hosted computer-use browser lanes (app-url, clone or local-tree), not this execution route. Remove the declaration or use a supported route.";
660
+ return "execution.desktop.media is supported only on computer-use browser lanes (app-url, clone or local-tree), not this execution route. Remove the declaration or use a supported route.";
657
661
  }
658
662
  if (config.execution?.desktop?.browser === "firefox") {
659
663
  return "execution.desktop.media requires Chrome or Chromium; Firefox cannot receive the declared synthetic capture device. Set execution.desktop.browser: chrome or chromium.";
660
664
  }
665
+ if (media.microphone !== undefined) {
666
+ if (config.actors.some(actor => actor.type !== "local-agent" || (actor.localAgent !== undefined && actor.localAgent !== "codex"))) {
667
+ return "Participant speech currently requires local-agent with Codex, on a local or hosted desktop.";
668
+ }
669
+ if (config.execution?.target !== "local" && media.camera !== undefined) {
670
+ return "Hosted synthetic cameras replace Chromium's microphone and cannot be combined with speech. Use a local media desktop for camera and speech together, or omit the hosted camera.";
671
+ }
672
+ }
661
673
  return undefined;
662
674
  }
663
675
  export function routesToComputerUse(config) {
@@ -2221,6 +2233,13 @@ function parseDesktop(raw) {
2221
2233
  return { ok: true, value: undefined };
2222
2234
  }
2223
2235
  const desktop = {};
2236
+ if (raw.recording !== undefined) {
2237
+ if (!isRecord(raw.recording) || typeof raw.recording.audio !== "boolean"
2238
+ || Object.keys(raw.recording).some(key => key !== "audio")) {
2239
+ return invalid("`execution.desktop.recording` must be { audio: true|false }; omit it for snapshots only.");
2240
+ }
2241
+ desktop.recording = { audio: raw.recording.audio };
2242
+ }
2224
2243
  if (raw.device !== undefined) {
2225
2244
  const device = str(raw.device);
2226
2245
  if (!device || !isDevicePresetName(device)) {
@@ -2297,7 +2316,10 @@ function parseDesktop(raw) {
2297
2316
  media.camera = { source };
2298
2317
  }
2299
2318
  if (raw.media.microphone !== undefined) {
2300
- return invalid("`execution.desktop.media.microphone.source` injection is unsupported, including on custom templates. Humanish cannot deliver the declared microphone file; remove this declaration. A custom template's own audio devices are a separate capability.");
2319
+ const source = isRecord(raw.media.microphone) ? str(raw.media.microphone.source) : undefined;
2320
+ if (source !== "speech")
2321
+ return invalid("`execution.desktop.media.microphone.source` must be `speech`. Microphone source-file injection is unsupported.");
2322
+ media.microphone = { source };
2301
2323
  }
2302
2324
  if (media.camera === undefined && media.microphone === undefined) {
2303
2325
  return invalid("`execution.desktop.media` declares neither `camera` nor `microphone`.");