humanish 0.101.0 → 0.102.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/README.md +3 -2
  2. package/dist/affordance.d.ts +2 -0
  3. package/dist/affordance.js +3 -0
  4. package/dist/affordance.js.map +1 -1
  5. package/dist/browser-control-client.d.ts +2 -0
  6. package/dist/browser-control-client.js +4 -0
  7. package/dist/browser-control-client.js.map +1 -1
  8. package/dist/browser-control-dispatcher.js +3 -0
  9. package/dist/browser-control-dispatcher.js.map +1 -1
  10. package/dist/browser-control-protocol.d.ts +77 -0
  11. package/dist/browser-control-protocol.js +31 -4
  12. package/dist/browser-control-protocol.js.map +1 -1
  13. package/dist/computer-use.d.ts +9 -0
  14. package/dist/computer-use.js +36 -8
  15. package/dist/computer-use.js.map +1 -1
  16. package/dist/cua-actor-lab.d.ts +1 -0
  17. package/dist/cua-actor-lab.js +10 -6
  18. package/dist/cua-actor-lab.js.map +1 -1
  19. package/dist/cua-speech.d.ts +13 -0
  20. package/dist/cua-speech.js +2 -0
  21. package/dist/cua-speech.js.map +1 -0
  22. package/dist/desktop-session.js +1 -0
  23. package/dist/desktop-session.js.map +1 -1
  24. package/dist/doctor-lab.js +1 -1
  25. package/dist/doctor-lab.js.map +1 -1
  26. package/dist/e2b-cua-desktop.js +51 -42
  27. package/dist/e2b-cua-desktop.js.map +1 -1
  28. package/dist/e2b-cua-provisioning.d.ts +4 -1
  29. package/dist/e2b-cua-provisioning.js +8 -2
  30. package/dist/e2b-cua-provisioning.js.map +1 -1
  31. package/dist/e2b-desktop-executor.js +3 -0
  32. package/dist/e2b-desktop-executor.js.map +1 -1
  33. package/dist/e2b-desktop-launch.d.ts +11 -1
  34. package/dist/e2b-desktop-launch.js.map +1 -1
  35. package/dist/e2b-desktop-media.d.ts +14 -0
  36. package/dist/e2b-desktop-media.js +58 -0
  37. package/dist/e2b-desktop-media.js.map +1 -0
  38. package/dist/guest-bootstrap.d.ts +4 -2
  39. package/dist/guest-bootstrap.js +16 -9
  40. package/dist/guest-bootstrap.js.map +1 -1
  41. package/dist/guest-desktop-executor.js +1 -0
  42. package/dist/guest-desktop-executor.js.map +1 -1
  43. package/dist/guest-desktop-media.d.ts +33 -0
  44. package/dist/guest-desktop-media.js +281 -0
  45. package/dist/guest-desktop-media.js.map +1 -0
  46. package/dist/guest-media-config.d.ts +15 -0
  47. package/dist/guest-media-config.js +8 -0
  48. package/dist/guest-media-config.js.map +1 -0
  49. package/dist/guest-media-worker.d.ts +1 -0
  50. package/dist/guest-media-worker.js +319 -0
  51. package/dist/guest-media-worker.js.map +1 -0
  52. package/dist/guest-runtime-desktop.d.ts +3 -1
  53. package/dist/guest-runtime-desktop.js +20 -3
  54. package/dist/guest-runtime-desktop.js.map +1 -1
  55. package/dist/guest-runtime-main.js +2 -2
  56. package/dist/guest-runtime-main.js.map +1 -1
  57. package/dist/guest-runtime.d.ts +2 -1
  58. package/dist/guest-runtime.js +1 -1
  59. package/dist/guest-runtime.js.map +1 -1
  60. package/dist/lab-config.d.ts +2 -2
  61. package/dist/lab-config.js +15 -4
  62. package/dist/lab-config.js.map +1 -1
  63. package/dist/lab-summary.js +1 -1
  64. package/dist/lab-summary.js.map +1 -1
  65. package/dist/local-firecracker-desktop.d.ts +3 -0
  66. package/dist/local-firecracker-desktop.js +7 -1
  67. package/dist/local-firecracker-desktop.js.map +1 -1
  68. package/dist/local-firecracker-study.js +16 -2
  69. package/dist/local-firecracker-study.js.map +1 -1
  70. package/dist/local-runtime-config.js +5 -2
  71. package/dist/local-runtime-config.js.map +1 -1
  72. package/dist/local-runtime-release.d.ts +2 -0
  73. package/dist/local-runtime-release.js +15 -0
  74. package/dist/local-runtime-release.js.map +1 -1
  75. package/dist/local-runtime.d.ts +1 -0
  76. package/dist/local-runtime.js +11 -9
  77. package/dist/local-runtime.js.map +1 -1
  78. package/dist/program.js +3 -2
  79. package/dist/program.js.map +1 -1
  80. package/dist/restricted-codex-participant-policy.d.ts +2 -1
  81. package/dist/restricted-codex-participant-policy.js +9 -6
  82. package/dist/restricted-codex-participant-policy.js.map +1 -1
  83. package/dist/restricted-codex-participant-run.js +2 -1
  84. package/dist/restricted-codex-participant-run.js.map +1 -1
  85. package/dist/restricted-codex-participant.d.ts +2 -0
  86. package/dist/restricted-codex-participant.js +20 -7
  87. package/dist/restricted-codex-participant.js.map +1 -1
  88. package/dist/run.d.ts +8 -5
  89. package/dist/run.js +6 -6
  90. package/dist/run.js.map +1 -1
  91. package/dist/study-analysis-engine.d.ts +1 -1
  92. package/dist/study-analysis-engine.js +53 -10
  93. package/dist/study-analysis-engine.js.map +1 -1
  94. package/docs/architecture/local-browser-runtime.md +3 -2
  95. package/docs/architecture/participant-media.md +90 -0
  96. package/docs/contracts/schemas.md +1 -1
  97. package/docs/contracts/study-analysis.md +14 -0
  98. package/docs/goals/current.md +13 -9
  99. package/docs/ramp/README.md +7 -1
  100. package/docs/release/0.102.0-participant-media.md +43 -0
  101. package/package.json +5 -4
  102. package/skills/humanish/SKILL.md +8 -5
@@ -10,6 +10,7 @@ export declare const PARTICIPANT_LIMITS: Readonly<{
10
10
  requestMs: 180000;
11
11
  cleanupMs: 5000;
12
12
  }>;
13
+ export declare function participantToolSchema(speechEnabled?: boolean): Record<string, unknown>;
13
14
  export declare const PARTICIPANT_TOOL_SCHEMA: Record<string, unknown>;
14
15
  export declare const PARTICIPANT_FINAL_SCHEMA: {
15
16
  type: string;
@@ -37,7 +38,7 @@ export declare const PARTICIPANT_FINAL_SCHEMA: {
37
38
  };
38
39
  };
39
40
  /** Humanish tool arguments, validated before the shared executor sees a batch. */
40
- export declare function parseParticipantTool(value: unknown): CuaTurn;
41
+ export declare function parseParticipantTool(value: unknown, speechEnabled?: boolean): CuaTurn;
41
42
  export declare function parseParticipantFinal(value: unknown): CuaTurn & {
42
43
  closingReport: ParticipantClosingReport;
43
44
  };
@@ -1,5 +1,5 @@
1
1
  import { z } from "zod";
2
- import { browserControlActionSchema, validateBrowserControlAction } from "./browser-control-protocol.js";
2
+ import { browserControlActionSchema, browserOnlyControlActionSchema, validateBrowserControlAction } from "./browser-control-protocol.js";
3
3
  import { validClosingReport } from "./computer-use.js";
4
4
  export const PARTICIPANT_PROFILE = Object.freeze({
5
5
  schema: "humanish.actor-execution-profile.v1", transport: "codex-app-server", authentication: "chatgpt-account",
@@ -7,9 +7,12 @@ export const PARTICIPANT_PROFILE = Object.freeze({
7
7
  toolPolicy: "codex-ui-tools-v1", participantSchema: "humanish.codex-ui-tool.v1", memoryPolicy: "continuing-thread-v1"
8
8
  });
9
9
  export const PARTICIPANT_LIMITS = Object.freeze({ instructions: 64 * 1024, hint: 8 * 1024, narration: 2000, output: 256 * 1024, actions: 4, requestMs: 180_000, cleanupMs: 5000 });
10
- const toolInput = z.strictObject({ narration: z.string().max(PARTICIPANT_LIMITS.narration),
11
- actions: z.array(browserControlActionSchema).min(1).max(PARTICIPANT_LIMITS.actions) });
12
- export const PARTICIPANT_TOOL_SCHEMA = z.toJSONSchema(toolInput, { io: "input" });
10
+ const toolInput = (speechEnabled) => z.strictObject({ narration: z.string().max(PARTICIPANT_LIMITS.narration),
11
+ actions: z.array(speechEnabled ? browserControlActionSchema : browserOnlyControlActionSchema).min(1).max(PARTICIPANT_LIMITS.actions) });
12
+ export function participantToolSchema(speechEnabled = false) {
13
+ return z.toJSONSchema(toolInput(speechEnabled), { io: "input" });
14
+ }
15
+ export const PARTICIPANT_TOOL_SCHEMA = participantToolSchema();
13
16
  export const PARTICIPANT_FINAL_SCHEMA = {
14
17
  type: "object", additionalProperties: false, required: ["outcome", "summary", "frictionReports"], properties: {
15
18
  outcome: { type: "string", enum: ["reached", "not_reached", "blocked"] },
@@ -18,10 +21,10 @@ export const PARTICIPANT_FINAL_SCHEMA = {
18
21
  }
19
22
  };
20
23
  /** Humanish tool arguments, validated before the shared executor sees a batch. */
21
- export function parseParticipantTool(value) {
24
+ export function parseParticipantTool(value, speechEnabled = false) {
22
25
  if (Buffer.byteLength(JSON.stringify(value) ?? "") > PARTICIPANT_LIMITS.output)
23
26
  throw new Error("invalid_response");
24
- const v = toolInput.parse(value);
27
+ const v = toolInput(speechEnabled).parse(value);
25
28
  if (Buffer.byteLength(v.narration) > PARTICIPANT_LIMITS.narration)
26
29
  throw new Error("invalid_response");
27
30
  return { actions: v.actions.map(validateBrowserControlAction), pendingSafetyChecks: [], done: false,
@@ -1 +1 @@
1
- {"version":3,"file":"restricted-codex-participant-policy.js","sourceRoot":"","sources":["../src/restricted-codex-participant-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AACzG,OAAO,EAAE,kBAAkB,EAAgB,MAAM,mBAAmB,CAAC;AAGrE,MAAM,CAAC,MAAM,mBAAmB,GAAoC,MAAM,CAAC,MAAM,CAAC;IAChF,MAAM,EAAE,qCAAqC,EAAE,SAAS,EAAE,kBAAkB,EAAE,cAAc,EAAE,iBAAiB;IAC/G,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,aAAa,EAAE,eAAe,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS;IACxG,UAAU,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,YAAY,EAAE,sBAAsB;CACtH,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,GAAG,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AACnL,MAAM,SAAS,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,kBAAkB,CAAC,SAAS,CAAC;IACxF,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;AACzF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,YAAY,CAAC,SAAS,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,CAA4B,CAAC;AAC7G,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,IAAI,EAAE,QAAQ,EAAE,oBAAoB,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,iBAAiB,CAAC,EAAE,UAAU,EAAE;QAC5G,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,aAAa,EAAE,SAAS,CAAC,EAAE;QACxE,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE;QAC1D,eAAe,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE;KAC1G;CACF,CAAC;AACF,kFAAkF;AAClF,MAAM,UAAU,oBAAoB,CAAC,KAAc;IACjD,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,GAAG,kBAAkB,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACpH,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,kBAAkB,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACvG,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,EAAE,mBAAmB,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK;QACjG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,sBAAsB,EAAE,IAAI,EAAE,CAAC;AACrF,CAAC;AACD,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACrG,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,EAAE,GAAG,KAAgC,CAAC;IAChE,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,aAAa,IAAI,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtJ,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO;QAChE,OAAO,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAC/D,aAAa,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC;AAC/F,CAAC"}
1
+ {"version":3,"file":"restricted-codex-participant-policy.js","sourceRoot":"","sources":["../src/restricted-codex-participant-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,0BAA0B,EAAE,8BAA8B,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AACzI,OAAO,EAAE,kBAAkB,EAAgB,MAAM,mBAAmB,CAAC;AAGrE,MAAM,CAAC,MAAM,mBAAmB,GAAoC,MAAM,CAAC,MAAM,CAAC;IAChF,MAAM,EAAE,qCAAqC,EAAE,SAAS,EAAE,kBAAkB,EAAE,cAAc,EAAE,iBAAiB;IAC/G,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,aAAa,EAAE,eAAe,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS;IACxG,UAAU,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,YAAY,EAAE,sBAAsB;CACtH,CAAC,CAAC;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,GAAG,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AACnL,MAAM,SAAS,GAAG,CAAC,aAAsB,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,kBAAkB,CAAC,SAAS,CAAC;IACpH,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;AAC1I,MAAM,UAAU,qBAAqB,CAAC,aAAa,GAAG,KAAK;IACzD,OAAO,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,CAA4B,CAAC;AAC9F,CAAC;AACD,MAAM,CAAC,MAAM,uBAAuB,GAAG,qBAAqB,EAAE,CAAC;AAC/D,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,IAAI,EAAE,QAAQ,EAAE,oBAAoB,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,iBAAiB,CAAC,EAAE,UAAU,EAAE;QAC5G,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,aAAa,EAAE,SAAS,CAAC,EAAE;QACxE,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE;QAC1D,eAAe,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE;KAC1G;CACF,CAAC;AACF,kFAAkF;AAClF,MAAM,UAAU,oBAAoB,CAAC,KAAc,EAAE,aAAa,GAAG,KAAK;IACxE,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,GAAG,kBAAkB,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACpH,MAAM,CAAC,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChD,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,kBAAkB,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACvG,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,EAAE,mBAAmB,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK;QACjG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,sBAAsB,EAAE,IAAI,EAAE,CAAC;AACrF,CAAC;AACD,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACrG,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,EAAE,GAAG,KAAgC,CAAC;IAChE,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,aAAa,IAAI,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACtJ,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO;QAChE,OAAO,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAC/D,aAAa,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,CAAC,GAAG,MAAM,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC;AAC/F,CAAC"}
@@ -47,7 +47,8 @@ export async function runRestrictedParticipantStudy(options) {
47
47
  (config.review?.analysis !== false && (!config.review?.analysis || config.review.analysis.provider !== "codex"))) {
48
48
  throw new CuaProviderError("request_rejected", { dispatched: false, usageComplete: false, cleanup: "confirmed" });
49
49
  }
50
- participant = createRestrictedCodexParticipant(options.participant);
50
+ participant = createRestrictedCodexParticipant({ ...options.participant,
51
+ ...(options.desktop.executor.speechEnabled === true ? { speechEnabled: true } : {}) });
51
52
  const outcome = await runLab(config, { cwd: options.cwd, dryRun: false, open: false,
52
53
  ...(options.runId === undefined ? {} : { runId: options.runId }),
53
54
  automaticAnalysis: { ...options.automaticAnalysis, onStart: () => {
@@ -1 +1 @@
1
- {"version":3,"file":"restricted-codex-participant-run.js","sourceRoot":"","sources":["../src/restricted-codex-participant-run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAmB,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAG7D,OAAO,EAAE,gCAAgC,EAA0E,MAAM,mCAAmC,CAAC;AAC7J,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE3D;gFACgF;AAChF,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,OAGnD;IACC,IAAI,WAA4E,CAAC;IACjF,IAAI,eAAe,GAAmC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;IAChF,IAAI,cAAc,GAAyB,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC;IACpG,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,UAAqC,CAAC;IAC1C,MAAM,MAAM,GAAG,GAAkB,EAAE,CAAC,UAAU,KAAK,CAAC,KAAK,IAAI,EAAE;QAC7D,IAAI,CAAC;YAAC,IAAI,WAAW;gBAAE,eAAe,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;QAAC,CAAC;QACrE,MAAM,CAAC;YAAC,eAAe,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;QAAC,CAAC;QACtD,IAAI,KAAgD,CAAC;QACrD,IAAI,CAAC;YACH,cAAc,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,OAAO,CAAuB,OAAO,CAAC,EAAE;oBACtI,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;gBACtG,CAAC,CAAC,CAAC,CAAC,CAAC;QACP,CAAC;QAAC,MAAM,CAAC;YAAC,cAAc,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;QAAC,CAAC;gBACzE,CAAC;YAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAChC,gBAAgB,KAAK,eAAe,CAAC,MAAM,KAAK,WAAW,IAAI,cAAc,CAAC,MAAM,KAAK,UAAU,CAAC;QACpG,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC,CAAC,EAAE,CAAC;IACL,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,qBAAqB;YACzH,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,aAAa,CAAC;YAClF,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,KAAK,KAAK,CAAC;YAC9F,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,IAAI,CAAC,CAAC,KAAK,CAAC;YAClF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,eAAe,KAAK,SAAS,IAAI,MAAM,CAAC,SAAS,EAAE,IAAI,EAAE,MAAM,KAAK,SAAS;YAC/F,MAAM,CAAC,SAAS,EAAE,IAAI,EAAE,WAAW,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,WAAW,KAAK,SAAS;YACpJ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,SAAU,CAAC,SAAU,GAAG,CAAC,IAAI,MAAM,CAAC,SAAU,CAAC,SAAU,GAAG,OAAO;YAC3H,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,KAAK,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC;YACnH,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;QACpH,CAAC;QACD,WAAW,GAAG,gCAAgC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK;YACjF,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;YAChE,iBAAiB,EAAE,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,OAAO,EAAE,GAAG,EAAE;oBAC/D,IAAI,CAAC,SAAS,IAAI,gBAAgB,IAAI,eAAe,CAAC,MAAM,KAAK,WAAW,IAAI,cAAc,CAAC,MAAM,KAAK,UAAU;wBAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;oBACzK,OAAO,OAAO,CAAC,iBAAiB,EAAE,OAAO,EAAE,EAAE,CAAC;gBAChD,CAAC,EAAE;YACH,QAAQ,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ;gBACtE,aAAa,EAAE,KAAK,IAAI,EAAE,CAAC,WAAY,CAAC,QAAQ;gBAChD,UAAU,EAAE,KAAK,EAAC,KAAK,EAAC,EAAE;oBACxB,IAAI,CAAC;wBACH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;wBAC3H,gBAAgB,KAAK,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,WAAW,CAAC,KAAK,IAAI,CAAC;wBAClG,OAAO,MAAM,CAAC;oBAChB,CAAC;4BAAS,CAAC;wBAAC,MAAM,MAAM,EAAE,CAAC;oBAAC,CAAC;gBAC/B,CAAC,EAAE;SACN,CAAC,CAAC;QACH,MAAM,MAAM,EAAE,CAAC;QACf,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC;IACtD,CAAC;YAAS,CAAC;QAAC,MAAM,MAAM,EAAE,CAAC;IAAC,CAAC;AAC/B,CAAC"}
1
+ {"version":3,"file":"restricted-codex-participant-run.js","sourceRoot":"","sources":["../src/restricted-codex-participant-run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAmB,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAG7D,OAAO,EAAE,gCAAgC,EAA0E,MAAM,mCAAmC,CAAC;AAC7J,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE3D;gFACgF;AAChF,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,OAGnD;IACC,IAAI,WAA4E,CAAC;IACjF,IAAI,eAAe,GAAmC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;IAChF,IAAI,cAAc,GAAyB,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC;IACpG,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,UAAqC,CAAC;IAC1C,MAAM,MAAM,GAAG,GAAkB,EAAE,CAAC,UAAU,KAAK,CAAC,KAAK,IAAI,EAAE;QAC7D,IAAI,CAAC;YAAC,IAAI,WAAW;gBAAE,eAAe,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;QAAC,CAAC;QACrE,MAAM,CAAC;YAAC,eAAe,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;QAAC,CAAC;QACtD,IAAI,KAAgD,CAAC;QACrD,IAAI,CAAC;YACH,cAAc,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,OAAO,CAAuB,OAAO,CAAC,EAAE;oBACtI,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;gBACtG,CAAC,CAAC,CAAC,CAAC,CAAC;QACP,CAAC;QAAC,MAAM,CAAC;YAAC,cAAc,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;QAAC,CAAC;gBACzE,CAAC;YAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAChC,gBAAgB,KAAK,eAAe,CAAC,MAAM,KAAK,WAAW,IAAI,cAAc,CAAC,MAAM,KAAK,UAAU,CAAC;QACpG,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC,CAAC,EAAE,CAAC;IACL,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,qBAAqB;YACzH,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,aAAa,CAAC;YAClF,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,KAAK,KAAK,CAAC;YAC9F,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,IAAI,CAAC,CAAC,KAAK,CAAC;YAClF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM;YAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,eAAe,KAAK,SAAS,IAAI,MAAM,CAAC,SAAS,EAAE,IAAI,EAAE,MAAM,KAAK,SAAS;YAC/F,MAAM,CAAC,SAAS,EAAE,IAAI,EAAE,WAAW,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,WAAW,KAAK,SAAS;YACpJ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,SAAU,CAAC,SAAU,GAAG,CAAC,IAAI,MAAM,CAAC,SAAU,CAAC,SAAU,GAAG,OAAO;YAC3H,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,KAAK,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,EAAE,CAAC;YACnH,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;QACpH,CAAC;QACD,WAAW,GAAG,gCAAgC,CAAC,EAAE,GAAG,OAAO,CAAC,WAAW;YACrE,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACzF,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK;YACjF,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;YAChE,iBAAiB,EAAE,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,OAAO,EAAE,GAAG,EAAE;oBAC/D,IAAI,CAAC,SAAS,IAAI,gBAAgB,IAAI,eAAe,CAAC,MAAM,KAAK,WAAW,IAAI,cAAc,CAAC,MAAM,KAAK,UAAU;wBAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;oBACzK,OAAO,OAAO,CAAC,iBAAiB,EAAE,OAAO,EAAE,EAAE,CAAC;gBAChD,CAAC,EAAE;YACH,QAAQ,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ;gBACtE,aAAa,EAAE,KAAK,IAAI,EAAE,CAAC,WAAY,CAAC,QAAQ;gBAChD,UAAU,EAAE,KAAK,EAAC,KAAK,EAAC,EAAE;oBACxB,IAAI,CAAC;wBACH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;wBAC3H,gBAAgB,KAAK,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,WAAW,CAAC,KAAK,IAAI,CAAC;wBAClG,OAAO,MAAM,CAAC;oBAChB,CAAC;4BAAS,CAAC;wBAAC,MAAM,MAAM,EAAE,CAAC;oBAAC,CAAC;gBAC/B,CAAC,EAAE;SACN,CAAC,CAAC;QACH,MAAM,MAAM,EAAE,CAAC;QACf,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC;IACtD,CAAC;YAAS,CAAC;QAAC,MAAM,MAAM,EAAE,CAAC;IAAC,CAAC;AAC/B,CAAC"}
@@ -10,6 +10,8 @@ export interface RestrictedParticipantOptions {
10
10
  authMode?: "operator";
11
11
  model?: string;
12
12
  reasoningEffort?: ReasoningEffort;
13
+ /** Admit speech actions and heard-speaker evidence for a speech-capable desktop only. */
14
+ speechEnabled?: boolean;
13
15
  }
14
16
  /** One native tool-calling conversation; the existing CUA loop owns every input. */
15
17
  export declare function createRestrictedCodexParticipant(options?: RestrictedParticipantOptions): {
@@ -1,7 +1,7 @@
1
1
  import { CuaProviderError, isCuaProviderError } from "./cua-provider-error.js";
2
- import { validateBrowserControlPng } from "./browser-control-protocol.js";
2
+ import { validateBrowserControlPng, validateHeardSpeech } from "./browser-control-protocol.js";
3
3
  import { createRestrictedCodexSession } from "./restricted-codex-session.js";
4
- import { PARTICIPANT_PROFILE, PARTICIPANT_LIMITS as L, PARTICIPANT_TOOL_SCHEMA, PARTICIPANT_FINAL_SCHEMA, parseParticipantTool, parseParticipantFinal } from "./restricted-codex-participant-policy.js";
4
+ import { PARTICIPANT_PROFILE, PARTICIPANT_LIMITS as L, PARTICIPANT_FINAL_SCHEMA, participantToolSchema, parseParticipantTool, parseParticipantFinal } from "./restricted-codex-participant-policy.js";
5
5
  const codeOf = (code) => {
6
6
  if (code === "cancelled" || code === "timeout" || code === "refusal")
7
7
  return code === "refusal" ? "refused" : code;
@@ -27,13 +27,14 @@ function deferred() {
27
27
  void promise.catch(() => undefined);
28
28
  return { promise, resolve, reject };
29
29
  }
30
- const TOOL_DESCRIPTION = `Act on the participant's browser through Humanish. Submit one to four UI actions and a short public comment; never private reasoning. Humanish returns a JSON STRING with execution acknowledgments and a fresh screenshot. In Code Mode use: const r = JSON.parse(await tools.humanish_ui({narration: "...", actions: [...]})); text({acknowledgments: r.acknowledgments, contextHint: r.contextHint, closing: r.closing}); image(r.imageUrl). Call serially and inspect each returned screenshot before deciding what to do next. Acknowledged input does not prove an application outcome. If closing is true, stop calling tools and give your final account.`;
30
+ const toolDescription = (speechEnabled) => `Act on the participant's browser through Humanish. Submit one to four UI actions${speechEnabled ? ", including speak when you need to reply aloud," : ""} and a short public comment; never private reasoning. Humanish returns a JSON STRING with execution acknowledgments${speechEnabled ? ", speech heard from the actual participant speaker sink," : ""} and a fresh screenshot. In Code Mode use: const r = JSON.parse(await tools.humanish_ui({narration: "...", actions: [...]})); text({acknowledgments: r.acknowledgments${speechEnabled ? ", heardSpeech: r.heardSpeech" : ""}, contextHint: r.contextHint, closing: r.closing}); image(r.imageUrl). Call serially and inspect each returned screenshot${speechEnabled ? " and heardSpeech array" : ""} before deciding what to do next.${speechEnabled ? " Use speak only after the visible UI shows that you joined the call and your microphone is unmuted; it sends audio into that call." : ""} Acknowledged input does not prove an application outcome. If closing is true, stop calling tools and give your final account.`;
31
31
  /** One native tool-calling conversation; the existing CUA loop owns every input. */
32
32
  export function createRestrictedCodexParticipant(options = {}) {
33
33
  const timeoutMs = options.requestTimeoutMs ?? L.requestMs;
34
34
  if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > L.requestMs)
35
35
  throw new CuaProviderError("request_rejected", noDispatch());
36
36
  const operator = options.authMode === "operator";
37
+ const speechEnabled = options.speechEnabled === true;
37
38
  const model = options.model ?? (operator ? undefined : PARTICIPANT_PROFILE.requestedModel);
38
39
  const effort = options.reasoningEffort ?? "low";
39
40
  let closed = false, failedCleanup = false, incompleteUsage = false, active = false, closingPhase = false;
@@ -62,11 +63,11 @@ export function createRestrictedCodexParticipant(options = {}) {
62
63
  let abortAt;
63
64
  const session = createRestrictedCodexSession({ ...options.session, participant: {
64
65
  ...(operator ? { authMode: "operator" } : {}), reasoningEffort: effort,
65
- tool: { name: "humanish_ui", description: TOOL_DESCRIPTION, inputSchema: PARTICIPANT_TOOL_SCHEMA,
66
+ tool: { name: "humanish_ui", description: toolDescription(speechEnabled), inputSchema: participantToolSchema(speechEnabled),
66
67
  async call(args) {
67
68
  if (closed || closingPhase || continuation || !active)
68
69
  throw new Error("Unexpected participant tool call");
69
- const turn = parseParticipantTool(args);
70
+ const turn = parseParticipantTool(args, speechEnabled);
70
71
  const reply = deferred();
71
72
  continuation = reply;
72
73
  lastActionCount = turn.actions.length;
@@ -94,12 +95,13 @@ export function createRestrictedCodexParticipant(options = {}) {
94
95
  let receipt = noDispatch(), usage;
95
96
  try {
96
97
  const result = await session.run({ ...(model === undefined ? {} : { model }),
97
- instructions: `${instructions}\n\nYou are the study participant throughout this conversation, including its closing account. Use only the supplied screenshots and humanish_ui tool to interact. The tool returns a JSON string: parse it, inspect acknowledgments, and display imageUrl with Code Mode image(). Do not print the image data URL as text. Keep your persona and earlier observations throughout the session. Speak publicly about your experience, never reveal private reasoning. Completed inputs do not prove application outcomes; verify on the next screenshot. When the task ends, return only the required final JSON with outcome, summary and frictionReports. Report observed confusion and recovered mistakes as well as blockers. Do not invent observations.`,
98
+ instructions: `${instructions}\n\nYou are the study participant throughout this conversation, including its closing account. Use only the supplied screenshots and humanish_ui tool to interact. The tool returns a JSON string: parse it, inspect acknowledgments${speechEnabled ? " and heardSpeech captured from the actual participant speaker sink" : ""}, and display imageUrl with Code Mode image(). Do not print the image data URL as text. Keep your persona and earlier observations throughout the session.${speechEnabled ? " A speak action plays into the participant microphone; use it only after the visible UI shows that you joined the call and the microphone is unmuted." : ""} Speak publicly about your experience, never reveal private reasoning. Completed inputs do not prove application outcomes; verify on the next screenshot. When the task ends, return only the required final JSON with outcome, summary and frictionReports. Report observed confusion and recovered mistakes as well as blockers. Do not invent observations.`,
98
99
  evidence: JSON.stringify({ phase: closingPhase ? "closing" : "interaction", contextHint: req.contextHint ?? null,
99
100
  instruction: closingPhase ? "Interaction has ended. Do not call tools; give your closing account."
100
101
  : "Use humanish_ui to act. Inspect each result before choosing the next batch. Finish when appropriate.",
101
102
  width: req.observation.screenshot.readUInt32BE(16), height: req.observation.screenshot.readUInt32BE(20),
102
- previousExecution: req.previousExecution ?? null }),
103
+ previousExecution: req.previousExecution ?? null,
104
+ ...(speechEnabled ? { heardSpeech: req.observation.heardSpeech ?? [] } : {}) }),
103
105
  images: [{ evidenceId: "current-frame", dataUrl: imageUrl }], schema: PARTICIPANT_FINAL_SCHEMA,
104
106
  maxOutputTokens: null, timeoutMs, signal: controller.signal });
105
107
  receipt = { dispatched: result.dispatched, usageComplete: result.usageComplete,
@@ -154,6 +156,16 @@ export function createRestrictedCodexParticipant(options = {}) {
154
156
  catch {
155
157
  throw new CuaProviderError("request_rejected", noDispatch());
156
158
  }
159
+ if (req.observation.heardSpeech !== undefined) {
160
+ if (!speechEnabled)
161
+ throw new CuaProviderError("request_rejected", noDispatch());
162
+ try {
163
+ validateHeardSpeech(req.observation.heardSpeech);
164
+ }
165
+ catch {
166
+ throw new CuaProviderError("request_rejected", noDispatch());
167
+ }
168
+ }
157
169
  const acknowledgments = req.previousExecution?.actions;
158
170
  if ((continuation && acknowledgments === undefined) || (acknowledgments !== undefined &&
159
171
  (lastActionCount === undefined || !Array.isArray(acknowledgments) || acknowledgments.length !== lastActionCount ||
@@ -170,6 +182,7 @@ export function createRestrictedCodexParticipant(options = {}) {
170
182
  const reply = continuation;
171
183
  continuation = undefined;
172
184
  reply.resolve(JSON.stringify({ acknowledgments: acknowledgments.map(({ index, status }) => ({ index, status })), imageUrl,
185
+ ...(speechEnabled ? { heardSpeech: req.observation.heardSpeech ?? [] } : {}),
173
186
  contextHint: req.contextHint ?? null, closing: debrief }));
174
187
  }
175
188
  else if (!active)
@@ -1 +1 @@
1
- {"version":3,"file":"restricted-codex-participant.js","sourceRoot":"","sources":["../src/restricted-codex-participant.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAA6B,MAAM,yBAAyB,CAAC;AAC1G,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,OAAO,EAAE,4BAA4B,EAAsC,MAAM,+BAA+B,CAAC;AAGjH,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,IAAI,CAAC,EAAE,uBAAuB,EAAE,wBAAwB,EACtG,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,0CAA0C,CAAC;AAUhG,MAAM,MAAM,GAAG,CAAC,IAA6C,EAAwB,EAAE;IACrF,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IACnH,IAAI,IAAI,KAAK,sBAAsB;QAAE,OAAO,qBAAqB,CAAC;IAClE,IAAI,IAAI,KAAK,YAAY;QAAE,OAAO,MAAM,CAAC;IACzC,IAAI,IAAI,KAAK,sBAAsB,IAAI,IAAI,KAAK,iBAAiB;QAAE,OAAO,gBAAgB,CAAC;IAC3F,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,KAAK,oBAAoB,IAAI,IAAI,KAAK,mBAAmB;QAAE,OAAO,kBAAkB,CAAC;IAC5H,IAAI,IAAI,KAAK,iBAAiB;QAAE,OAAO,kBAAkB,CAAC;IAC1D,IAAI,IAAI,KAAK,sBAAsB;QAAE,OAAO,gBAAgB,CAAC;IAC7D,OAAO,aAAa,CAAC;AACvB,CAAC,CAAC;AACF,MAAM,UAAU,GAAG,GAA2B,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;AACrH,SAAS,QAAQ;IACf,IAAI,OAA4B,EAAE,MAAiC,CAAC;IACpE,MAAM,OAAO,GAAG,IAAI,OAAO,CAAI,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,GAAG,OAAO,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,mFAAmF;IACnF,KAAK,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACpC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACtC,CAAC;AACD,MAAM,gBAAgB,GAAG,moBAAmoB,CAAC;AAE7pB,oFAAoF;AACpF,MAAM,UAAU,gCAAgC,CAAC,UAAwC,EAAE;IAGzF,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC,SAAS,CAAC;IAC1D,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,CAAC,SAAS;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC;IAC/I,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,KAAK,UAAU,CAAC;IACjD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC,cAAc,CAAC,CAAC;IAC3F,MAAM,MAAM,GAAG,OAAO,CAAC,eAAe,IAAI,KAAK,CAAC;IAChD,IAAI,MAAM,GAAG,KAAK,EAAE,aAAa,GAAG,KAAK,EAAE,eAAe,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,EAAE,YAAY,GAAG,KAAK,CAAC;IACzG,IAAI,YAAgC,EAAE,eAAmC,CAAC;IAC1E,IAAI,YAA6D,CAAC;IAElE,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,MAAsD,CAAC;IAC3D,MAAM,IAAI,GAAG,CAAC,KAAY,EAAQ,EAAE;QAClC,IAAI,MAAM,EAAE,CAAC;YAAC,MAAM,OAAO,GAAG,MAAM,CAAC;YAAC,MAAM,GAAG,SAAS,CAAC;YAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;;YAC9E,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,IAAsB,EAAE;QAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAS,CAAC,CAAC,OAAO,CAAC;QAC7E,IAAI,OAAO,IAAI,KAAK;YAAE,MAAM,KAAK,CAAC,KAAK,CAAC;QACxC,OAAO,KAAK,CAAC,IAAI,CAAC;IACpB,CAAC,CAAC;IACF,IAAI,UAAqC,EAAE,OAAqC,CAAC;IACjF,IAAI,UAAuC,CAAC;IAC5C,IAAI,cAA4C,EAAE,OAA4D,CAAC;IAC/G,IAAI,OAA2B,CAAC;IAChC,MAAM,OAAO,GAAG,4BAA4B,CAAC,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE;YAC9E,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,eAAe,EAAE,MAAM;YAC/E,IAAI,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,gBAAgB,EAAE,WAAW,EAAE,uBAAuB;gBAC9F,KAAK,CAAC,IAAI,CAAC,IAAI;oBACb,IAAI,MAAM,IAAI,YAAY,IAAI,YAAY,IAAI,CAAC,MAAM;wBAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;oBAC3G,MAAM,IAAI,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;oBACxC,MAAM,KAAK,GAAG,QAAQ,EAAU,CAAC;oBAAC,YAAY,GAAG,KAAK,CAAC;oBAAC,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;oBAC9F,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;oBACf,OAAO,KAAK,CAAC,OAAO,CAAC;gBACvB,CAAC,EAAE;SACN,EAAE,CAAC,CAAC;IACL,MAAM,YAAY,GAAG,GAAqB,EAAE,CAAC,cAAc,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE;QAChJ,IAAI,CAAC,SAAS;YAAE,aAAa,GAAG,IAAI,CAAC;QACrC,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,KAAK,WAAW,CAAC,GAAG,EAAE,CAAC;QAC7C,UAAU,EAAE,KAAK,EAAE,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAAC,YAAY,GAAG,SAAS,CAAC;QAChF,KAAK,YAAY,EAAE,CAAC;IACtB,CAAC,CAAC;IACF,SAAS,MAAM,CAAC,GAAmB,EAAE,QAAgB;QACnD,MAAM,GAAG,IAAI,CAAC;QAAC,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QAClD,UAAU,GAAG,CAAC,KAAK,IAAI,EAAE;YACvB,IAAI,OAAO,GAAG,UAAU,EAAE,EAAE,KAAkC,CAAC;YAC/D,IAAI,CAAC;gBACH,MAAM,MAAM,GAA0B,MAAM,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;oBACjG,YAAY,EAAE,GAAG,YAAY,8tBAA8tB;oBAC3vB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI;wBAC9G,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC,sEAAsE;4BAChG,CAAC,CAAC,sGAAsG;wBAC1G,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,UAAW,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC,UAAW,CAAC,YAAY,CAAC,EAAE,CAAC;wBACzG,iBAAiB,EAAE,GAAG,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;oBACrD,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,wBAAwB;oBAC9F,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,UAAW,CAAC,MAAM,EAAE,CAAC,CAAC;gBAClE,OAAO,GAAG,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa;oBAC5E,OAAO,EAAE,MAAM,CAAC,SAAS,KAAK,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBACvF,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;gBAClC,IAAI,MAAM,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,aAAa;oBAAE,eAAe,GAAG,IAAI,CAAC;gBACvE,IAAI,OAAO,CAAC,OAAO,KAAK,aAAa;oBAAE,aAAa,GAAG,IAAI,CAAC;gBAC5D,IAAI,MAAM,IAAI,UAAW,CAAC,MAAM,CAAC,OAAO;oBAAE,MAAM,IAAI,gBAAgB,CAAC,aAAa,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;gBAC/J,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,SAAS,KAAK,IAAI;oBAAE,MAAM,IAAI,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;gBAC1J,IAAI,IAAa,CAAC;gBAClB,IAAI,CAAC;oBAAC,IAAI,GAAG,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAAC,CAAC;gBACpD,MAAM,CAAC;oBAAC,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;gBAAC,CAAC;gBACrF,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,cAAc,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,eAAe,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;YACrJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,eAAe,KAAK,OAAO,CAAC,UAAU,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;gBAC3E,MAAM,EAAE,CAAC;gBACT,IAAI,CAAC,EAAE,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;wBAC7C,CAAC,CAAC,IAAI,gBAAgB,CAAC,gBAAgB,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;YAChI,CAAC;oBAAS,CAAC;gBAAC,MAAM,GAAG,KAAK,CAAC;YAAC,CAAC;QAC/B,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IACD,KAAK,UAAU,OAAO,CAAC,GAAmB,EAAE,MAAmB,EAAE,OAAgB;QAC/E,IAAI,MAAM,CAAC,MAAM;YAAE,OAAO,SAAS,EAAE,CAAC;QACtC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAAC,MAAM,EAAE,CAAC;YAAC,MAAM,IAAI,gBAAgB,CAAC,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;QAAC,CAAC;QACxF,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,YAAY;YAC9F,CAAC,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,CAAC,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YACvH,GAAG,CAAC,kBAAkB,KAAK,SAAS,IAAI,CAAC,GAAG,CAAC,wBAAwB,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;YACvF,CAAC,YAAY,KAAK,SAAS,IAAI,YAAY,KAAK,GAAG,CAAC,YAAY,CAAC;YAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC;QAClI,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,CAAC,UAAU,CAAC;QACzC,IAAI,CAAC;YAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,EAAE,CAAC;YAAC,yBAAyB,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QACzF,MAAM,CAAC;YAAC,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC;QAAC,CAAC;QACvE,MAAM,eAAe,GAAG,GAAG,CAAC,iBAAiB,EAAE,OAAO,CAAC;QACvD,IAAI,CAAC,YAAY,IAAI,eAAe,KAAK,SAAS,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS;YACnF,CAAC,eAAe,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,eAAe,CAAC,MAAM,KAAK,eAAe;gBAC7G,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,SAAS,EAAE,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACjJ,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,MAAM,OAAO,GAAG,GAAS,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC;YACH,YAAY,KAAK,GAAG,CAAC,YAAY,CAAC;YAAC,YAAY,GAAG,OAAO,CAAC;YAC1D,MAAM,QAAQ,GAAG,yBAAyB,KAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtE,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,YAAY,CAAC;gBAAC,YAAY,GAAG,SAAS,CAAC;gBACrD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,eAAe,EAAE,eAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,QAAQ;oBACxH,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;YAC/D,CAAC;iBAAM,IAAI,CAAC,MAAM;gBAAE,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;;gBACrC,MAAM,IAAI,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YACtD,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,2EAA2E;YAC3E,0EAA0E;YAC1E,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAmB,EAAE,MAAmB,EAAE,OAAgB,EAAoB,EAAE;QAC7F,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;QACjH,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;QAC/E,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,OAAO,GAAG,IAAI,CAAC;QAC3D,KAAK,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC/F,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,QAAQ,GAAgB;QAC5B,EAAE,EAAE,mBAAmB,EAAE,aAAa,EAAE,IAAI,EAAE,aAAa,EAAE,aAAa;QAC1E,IAAI,OAAO,KAAK,OAAO,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC,CAAC,CAAC;QACxD,IAAI,gBAAgB;YAClB,IAAI,CAAC,QAAQ;gBAAE,OAAO,mBAAmB,CAAC;YAC1C,OAAO,OAAO,CAAC,cAAc,KAAK,iBAAiB,IAAI,OAAO,CAAC,aAAa;gBAC1E,CAAC,CAAC,EAAE,GAAG,mBAAmB,EAAE,cAAc,EAAE,OAAO,CAAC,aAAa,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7G,CAAC;QACD,aAAa,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE;QAC1C,YAAY,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,cAAc,CAAC,EAAE,mBAAmB,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ;YAC3H,qBAAqB,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE;QAC/E,IAAI,mBAAmB;YACrB,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC;YACnC,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,qBAAqB,IAAI,EAAE,EAAE,CAAC;QACpG,CAAC;QACD,IAAI,0BAA0B,KAAK,OAAO,eAAe,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,mBAAmB,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/G,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC;KACzG,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE;YAC7B,IAAI,OAAO;gBAAE,OAAO,OAAO,CAAC;YAC5B,MAAM,EAAE,CAAC;YACT,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE;gBACpB,IAAI,KAAgD,CAAC;gBACrD,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;wBAClG,IAAI,OAAO,CAAQ,OAAO,CAAC,EAAE,GAAG,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,SAAS,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC5I,IAAI,CAAC,SAAS;wBAAE,aAAa,GAAG,IAAI,CAAC;oBACrC,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,EAAoC,CAAC;gBACnG,CAAC;wBAAS,CAAC;oBAAC,YAAY,CAAC,KAAK,CAAC,CAAC;gBAAC,CAAC;YACpC,CAAC,CAAC,EAAE,CAAC;YACL,OAAO,OAAO,CAAC;QACjB,CAAC,EAAE,CAAC;AACN,CAAC"}
1
+ {"version":3,"file":"restricted-codex-participant.js","sourceRoot":"","sources":["../src/restricted-codex-participant.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAA6B,MAAM,yBAAyB,CAAC;AAC1G,OAAO,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAC/F,OAAO,EAAE,4BAA4B,EAAsC,MAAM,+BAA+B,CAAC;AAGjH,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,IAAI,CAAC,EAAE,wBAAwB,EAC7E,qBAAqB,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,0CAA0C,CAAC;AAYvH,MAAM,MAAM,GAAG,CAAC,IAA6C,EAAwB,EAAE;IACrF,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IACnH,IAAI,IAAI,KAAK,sBAAsB;QAAE,OAAO,qBAAqB,CAAC;IAClE,IAAI,IAAI,KAAK,YAAY;QAAE,OAAO,MAAM,CAAC;IACzC,IAAI,IAAI,KAAK,sBAAsB,IAAI,IAAI,KAAK,iBAAiB;QAAE,OAAO,gBAAgB,CAAC;IAC3F,IAAI,IAAI,KAAK,kBAAkB,IAAI,IAAI,KAAK,oBAAoB,IAAI,IAAI,KAAK,mBAAmB;QAAE,OAAO,kBAAkB,CAAC;IAC5H,IAAI,IAAI,KAAK,iBAAiB;QAAE,OAAO,kBAAkB,CAAC;IAC1D,IAAI,IAAI,KAAK,sBAAsB;QAAE,OAAO,gBAAgB,CAAC;IAC7D,OAAO,aAAa,CAAC;AACvB,CAAC,CAAC;AACF,MAAM,UAAU,GAAG,GAA2B,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;AACrH,SAAS,QAAQ;IACf,IAAI,OAA4B,EAAE,MAAiC,CAAC;IACpE,MAAM,OAAO,GAAG,IAAI,OAAO,CAAI,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,GAAG,OAAO,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,mFAAmF;IACnF,KAAK,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACpC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACtC,CAAC;AACD,MAAM,eAAe,GAAG,CAAC,aAAsB,EAAU,EAAE,CAAC,mFAAmF,aAAa,CAAC,CAAC,CAAC,iDAAiD,CAAC,CAAC,CAAC,EAAE,sHAAsH,aAAa,CAAC,CAAC,CAAC,0DAA0D,CAAC,CAAC,CAAC,EAAE,yKAAyK,aAAa,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,EAAE,4HAA4H,aAAa,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,EAAE,oCAAoC,aAAa,CAAC,CAAC,CAAC,oIAAoI,CAAC,CAAC,CAAC,EAAE,gIAAgI,CAAC;AAE7lC,oFAAoF;AACpF,MAAM,UAAU,gCAAgC,CAAC,UAAwC,EAAE;IAGzF,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC,SAAS,CAAC;IAC1D,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,CAAC,SAAS;QAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC;IAC/I,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,KAAK,UAAU,CAAC;IACjD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,KAAK,IAAI,CAAC;IACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC,cAAc,CAAC,CAAC;IAC3F,MAAM,MAAM,GAAG,OAAO,CAAC,eAAe,IAAI,KAAK,CAAC;IAChD,IAAI,MAAM,GAAG,KAAK,EAAE,aAAa,GAAG,KAAK,EAAE,eAAe,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,EAAE,YAAY,GAAG,KAAK,CAAC;IACzG,IAAI,YAAgC,EAAE,eAAmC,CAAC;IAC1E,IAAI,YAA6D,CAAC;IAElE,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,MAAsD,CAAC;IAC3D,MAAM,IAAI,GAAG,CAAC,KAAY,EAAQ,EAAE;QAClC,IAAI,MAAM,EAAE,CAAC;YAAC,MAAM,OAAO,GAAG,MAAM,CAAC;YAAC,MAAM,GAAG,SAAS,CAAC;YAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;;YAC9E,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,IAAsB,EAAE;QAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAS,CAAC,CAAC,OAAO,CAAC;QAC7E,IAAI,OAAO,IAAI,KAAK;YAAE,MAAM,KAAK,CAAC,KAAK,CAAC;QACxC,OAAO,KAAK,CAAC,IAAI,CAAC;IACpB,CAAC,CAAC;IACF,IAAI,UAAqC,EAAE,OAAqC,CAAC;IACjF,IAAI,UAAuC,CAAC;IAC5C,IAAI,cAA4C,EAAE,OAA4D,CAAC;IAC/G,IAAI,OAA2B,CAAC;IAChC,MAAM,OAAO,GAAG,4BAA4B,CAAC,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE;YAC9E,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,eAAe,EAAE,MAAM;YAC/E,IAAI,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,eAAe,CAAC,aAAa,CAAC,EAAE,WAAW,EAAE,qBAAqB,CAAC,aAAa,CAAC;gBACzH,KAAK,CAAC,IAAI,CAAC,IAAI;oBACb,IAAI,MAAM,IAAI,YAAY,IAAI,YAAY,IAAI,CAAC,MAAM;wBAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;oBAC3G,MAAM,IAAI,GAAG,oBAAoB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;oBACvD,MAAM,KAAK,GAAG,QAAQ,EAAU,CAAC;oBAAC,YAAY,GAAG,KAAK,CAAC;oBAAC,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;oBAC9F,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;oBACf,OAAO,KAAK,CAAC,OAAO,CAAC;gBACvB,CAAC,EAAE;SACN,EAAE,CAAC,CAAC;IACL,MAAM,YAAY,GAAG,GAAqB,EAAE,CAAC,cAAc,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE;QAChJ,IAAI,CAAC,SAAS;YAAE,aAAa,GAAG,IAAI,CAAC;QACrC,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,MAAM,GAAG,IAAI,CAAC;QAAC,OAAO,KAAK,WAAW,CAAC,GAAG,EAAE,CAAC;QAC7C,UAAU,EAAE,KAAK,EAAE,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAAC,YAAY,GAAG,SAAS,CAAC;QAChF,KAAK,YAAY,EAAE,CAAC;IACtB,CAAC,CAAC;IACF,SAAS,MAAM,CAAC,GAAmB,EAAE,QAAgB;QACnD,MAAM,GAAG,IAAI,CAAC;QAAC,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QAClD,UAAU,GAAG,CAAC,KAAK,IAAI,EAAE;YACvB,IAAI,OAAO,GAAG,UAAU,EAAE,EAAE,KAAkC,CAAC;YAC/D,IAAI,CAAC;gBACH,MAAM,MAAM,GAA0B,MAAM,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;oBACjG,YAAY,EAAE,GAAG,YAAY,uOAAuO,aAAa,CAAC,CAAC,CAAC,oEAAoE,CAAC,CAAC,CAAC,EAAE,6JAA6J,aAAa,CAAC,CAAC,CAAC,uJAAuJ,CAAC,CAAC,CAAC,EAAE,gWAAgW;oBACtgC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI;wBAC9G,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC,sEAAsE;4BAChG,CAAC,CAAC,sGAAsG;wBAC1G,KAAK,EAAE,GAAG,CAAC,WAAW,CAAC,UAAW,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,WAAW,CAAC,UAAW,CAAC,YAAY,CAAC,EAAE,CAAC;wBACzG,iBAAiB,EAAE,GAAG,CAAC,iBAAiB,IAAI,IAAI;wBAChD,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;oBACjF,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,wBAAwB;oBAC9F,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,UAAW,CAAC,MAAM,EAAE,CAAC,CAAC;gBAClE,OAAO,GAAG,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa;oBAC5E,OAAO,EAAE,MAAM,CAAC,SAAS,KAAK,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBACvF,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;gBAClC,IAAI,MAAM,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,aAAa;oBAAE,eAAe,GAAG,IAAI,CAAC;gBACvE,IAAI,OAAO,CAAC,OAAO,KAAK,aAAa;oBAAE,aAAa,GAAG,IAAI,CAAC;gBAC5D,IAAI,MAAM,IAAI,UAAW,CAAC,MAAM,CAAC,OAAO;oBAAE,MAAM,IAAI,gBAAgB,CAAC,aAAa,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;gBAC/J,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,SAAS,KAAK,IAAI;oBAAE,MAAM,IAAI,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;gBAC1J,IAAI,IAAa,CAAC;gBAClB,IAAI,CAAC;oBAAC,IAAI,GAAG,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAAC,CAAC;gBACpD,MAAM,CAAC;oBAAC,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;gBAAC,CAAC;gBACrF,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,cAAc,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,eAAe,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;YACrJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,eAAe,KAAK,OAAO,CAAC,UAAU,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;gBAC3E,MAAM,EAAE,CAAC;gBACT,IAAI,CAAC,EAAE,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;wBAC7C,CAAC,CAAC,IAAI,gBAAgB,CAAC,gBAAgB,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;YAChI,CAAC;oBAAS,CAAC;gBAAC,MAAM,GAAG,KAAK,CAAC;YAAC,CAAC;QAC/B,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IACD,KAAK,UAAU,OAAO,CAAC,GAAmB,EAAE,MAAmB,EAAE,OAAgB;QAC/E,IAAI,MAAM,CAAC,MAAM;YAAE,OAAO,SAAS,EAAE,CAAC;QACtC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAAC,MAAM,EAAE,CAAC;YAAC,MAAM,IAAI,gBAAgB,CAAC,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;QAAC,CAAC;QACxF,IAAI,OAAO,GAAG,CAAC,YAAY,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,YAAY;YAC9F,CAAC,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,CAAC,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YACvH,GAAG,CAAC,kBAAkB,KAAK,SAAS,IAAI,CAAC,GAAG,CAAC,wBAAwB,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;YACvF,CAAC,YAAY,KAAK,SAAS,IAAI,YAAY,KAAK,GAAG,CAAC,YAAY,CAAC;YAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC;QAClI,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,CAAC,UAAU,CAAC;QACzC,IAAI,CAAC;YAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,EAAE,CAAC;YAAC,yBAAyB,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QACzF,MAAM,CAAC;YAAC,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC;QAAC,CAAC;QACvE,IAAI,GAAG,CAAC,WAAW,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9C,IAAI,CAAC,aAAa;gBAAE,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC;YACjF,IAAI,CAAC;gBAAC,mBAAmB,CAAC,GAAG,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;YAAC,CAAC;YACzD,MAAM,CAAC;gBAAC,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC;YAAC,CAAC;QACzE,CAAC;QACD,MAAM,eAAe,GAAG,GAAG,CAAC,iBAAiB,EAAE,OAAO,CAAC;QACvD,IAAI,CAAC,YAAY,IAAI,eAAe,KAAK,SAAS,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS;YACnF,CAAC,eAAe,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,eAAe,CAAC,MAAM,KAAK,eAAe;gBAC7G,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,SAAS,EAAE,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACjJ,MAAM,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,MAAM,OAAO,GAAG,GAAS,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC;YACH,YAAY,KAAK,GAAG,CAAC,YAAY,CAAC;YAAC,YAAY,GAAG,OAAO,CAAC;YAC1D,MAAM,QAAQ,GAAG,yBAAyB,KAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtE,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,YAAY,CAAC;gBAAC,YAAY,GAAG,SAAS,CAAC;gBACrD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,eAAe,EAAE,eAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,QAAQ;oBACxH,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC5E,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;YAC/D,CAAC;iBAAM,IAAI,CAAC,MAAM;gBAAE,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;;gBACrC,MAAM,IAAI,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YACtD,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;gBAAS,CAAC;YACT,2EAA2E;YAC3E,0EAA0E;YAC1E,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAmB,EAAE,MAAmB,EAAE,OAAgB,EAAoB,EAAE;QAC7F,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,kBAAkB,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;QACjH,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;QAC/E,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAAC,OAAO,GAAG,IAAI,CAAC;QAC3D,KAAK,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC/F,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,QAAQ,GAAgB;QAC5B,EAAE,EAAE,mBAAmB,EAAE,aAAa,EAAE,IAAI,EAAE,aAAa,EAAE,aAAa;QAC1E,IAAI,OAAO,KAAK,OAAO,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC,CAAC,CAAC;QACxD,IAAI,gBAAgB;YAClB,IAAI,CAAC,QAAQ;gBAAE,OAAO,mBAAmB,CAAC;YAC1C,OAAO,OAAO,CAAC,cAAc,KAAK,iBAAiB,IAAI,OAAO,CAAC,aAAa;gBAC1E,CAAC,CAAC,EAAE,GAAG,mBAAmB,EAAE,cAAc,EAAE,OAAO,CAAC,aAAa,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7G,CAAC;QACD,aAAa,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE;QAC1C,YAAY,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,cAAc,CAAC,EAAE,mBAAmB,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ;YAC3H,qBAAqB,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE;QAC/E,IAAI,mBAAmB;YACrB,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC;YACnC,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,qBAAqB,IAAI,EAAE,EAAE,CAAC;QACpG,CAAC;QACD,IAAI,0BAA0B,KAAK,OAAO,eAAe,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,mBAAmB,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/G,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC;KACzG,CAAC;IACF,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE;YAC7B,IAAI,OAAO;gBAAE,OAAO,OAAO,CAAC;YAC5B,MAAM,EAAE,CAAC;YACT,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE;gBACpB,IAAI,KAAgD,CAAC;gBACrD,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;wBAClG,IAAI,OAAO,CAAQ,OAAO,CAAC,EAAE,GAAG,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,SAAS,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC5I,IAAI,CAAC,SAAS;wBAAE,aAAa,GAAG,IAAI,CAAC;oBACrC,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,EAAoC,CAAC;gBACnG,CAAC;wBAAS,CAAC;oBAAC,YAAY,CAAC,KAAK,CAAC,CAAC;gBAAC,CAAC;YACpC,CAAC,CAAC,EAAE,CAAC;YACL,OAAO,OAAO,CAAC;QACjB,CAAC,EAAE,CAAC;AACN,CAAC"}
package/dist/run.d.ts CHANGED
@@ -800,6 +800,9 @@ export interface RunBundle {
800
800
  source: "synthetic" | "file";
801
801
  file: string;
802
802
  };
803
+ microphone?: {
804
+ source: "speech";
805
+ };
803
806
  permission: "prompt" | "granted";
804
807
  flags: string[];
805
808
  };
@@ -1231,12 +1234,12 @@ export declare function readReview(cwdInput: string, runInput: string): Promise<
1231
1234
  runId: string;
1232
1235
  })>;
1233
1236
  /**
1234
- * The @e2b/desktop release that stopped holding a background command's event stream open after
1235
- * the command was launched (its 2.3.1 changelog, 2026-06-08). On older releases the CLI process
1236
- * stayed alive minutes past a run's written result on one socket to the provider (#581, measured
1237
- * 2026-09-04 on 2.2.3: twelve minutes).
1237
+ * Version 2.3.1 stopped holding a launched background command's event stream open. Version 2.3.2
1238
+ * also requires an e2b release whose background command handle supports sendStdin and closeStdin,
1239
+ * which the optional speech transport needs. On older releases the CLI could stay alive minutes
1240
+ * past a written result (#581, measured 2026-09-04 on 2.2.3: twelve minutes).
1238
1241
  */
1239
- export declare const DESKTOP_SDK_FLOOR = "2.3.1";
1242
+ export declare const DESKTOP_SDK_FLOOR = "2.3.2";
1240
1243
  /** The advisory `doctor` attaches to an installed desktop SDK older than the floor, else undefined. */
1241
1244
  export declare function desktopSdkAdvisory(version: string | undefined): string | undefined;
1242
1245
  export declare function doctor(cwdInput: string, options?: {
package/dist/run.js CHANGED
@@ -3092,12 +3092,12 @@ export async function readReview(cwdInput, runInput) {
3092
3092
  };
3093
3093
  }
3094
3094
  /**
3095
- * The @e2b/desktop release that stopped holding a background command's event stream open after
3096
- * the command was launched (its 2.3.1 changelog, 2026-06-08). On older releases the CLI process
3097
- * stayed alive minutes past a run's written result on one socket to the provider (#581, measured
3098
- * 2026-09-04 on 2.2.3: twelve minutes).
3095
+ * Version 2.3.1 stopped holding a launched background command's event stream open. Version 2.3.2
3096
+ * also requires an e2b release whose background command handle supports sendStdin and closeStdin,
3097
+ * which the optional speech transport needs. On older releases the CLI could stay alive minutes
3098
+ * past a written result (#581, measured 2026-09-04 on 2.2.3: twelve minutes).
3099
3099
  */
3100
- export const DESKTOP_SDK_FLOOR = "2.3.1";
3100
+ export const DESKTOP_SDK_FLOOR = "2.3.2";
3101
3101
  /** The advisory `doctor` attaches to an installed desktop SDK older than the floor, else undefined. */
3102
3102
  export function desktopSdkAdvisory(version) {
3103
3103
  if (version === undefined)
@@ -3110,7 +3110,7 @@ export function desktopSdkAdvisory(version) {
3110
3110
  const older = have[0] < floor[0]
3111
3111
  || (have[0] === floor[0] && (have[1] < floor[1] || (have[1] === floor[1] && have[2] < floor[2])));
3112
3112
  return older
3113
- ? `@e2b/desktop ${version} is older than ${DESKTOP_SDK_FLOOR}, which stopped holding a background command stream open after a run (the CLI stayed alive minutes past its result, #581). Update with \`npm i -D @e2b/desktop@latest\`.`
3113
+ ? `@e2b/desktop ${version} is older than ${DESKTOP_SDK_FLOOR}, the supported floor for background command cleanup and stdin handles (older releases could keep the CLI alive minutes past its result, #581). Update with \`npm i -D @e2b/desktop@latest\`.`
3114
3114
  : undefined;
3115
3115
  }
3116
3116
  /** The version of the @e2b/desktop that `import("@e2b/desktop")` resolves to from here, if readable. */