@iwsdk/cli 1.1.0-rc.0 → 1.1.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -0
- package/dist/browser-lease.d.ts.map +1 -1
- package/dist/cli.js +669 -127
- package/dist/cli.js.map +1 -1
- package/dist/commands/runtime.d.ts.map +1 -1
- package/dist/contract.js +596 -118
- package/dist/contract.js.map +1 -1
- package/dist/help.d.ts.map +1 -1
- package/dist/index.js +669 -127
- package/dist/index.js.map +1 -1
- package/dist/runtime-contract.d.ts +22 -1
- package/dist/runtime-contract.d.ts.map +1 -1
- package/dist/runtime-state.d.ts +8 -0
- package/dist/runtime-state.d.ts.map +1 -1
- package/dist/runtime-transport.d.ts.map +1 -1
- package/dist/version.d.ts +1 -1
- package/guidance/AGENTS.md +25 -12
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -153,21 +153,48 @@ const DEFAULT_RUNTIME_COMMAND_TIMEOUT_MS = 30_000;
|
|
|
153
153
|
const UI_RENDER_PREVIEW_TIMEOUT_MS = 60_000;
|
|
154
154
|
const ASSET_RENDER_PREVIEW_TIMEOUT_MS = 120_000;
|
|
155
155
|
const BROWSER_RUNTIME_COMMAND_TIMEOUT_MS = 60_000;
|
|
156
|
+
/**
|
|
157
|
+
* IWER resolves a running hand sequence at most 28 s after dispatch, with a
|
|
158
|
+
* partial result when it runs out of time. The default budget, less the
|
|
159
|
+
* relay's margin, would give up first.
|
|
160
|
+
*/
|
|
161
|
+
const HAND_SEQUENCE_COMMAND_TIMEOUT_MS = 40_000;
|
|
162
|
+
/** The longest `duration` (seconds) that extends a default command timeout. */
|
|
163
|
+
const MAX_TIMED_COMMAND_DURATION_S = 3_600;
|
|
164
|
+
/** IWER replies to these only once their `duration` (seconds) has elapsed. */
|
|
165
|
+
const TIMED_RUNTIME_METHODS = new Set([
|
|
166
|
+
'animate_to',
|
|
167
|
+
'select',
|
|
168
|
+
'set_hand_pose',
|
|
169
|
+
'set_select_value',
|
|
170
|
+
]);
|
|
156
171
|
const BROWSER_RUNTIME_METHODS = new Set([
|
|
157
172
|
'browser_interact',
|
|
158
173
|
'browser_profile',
|
|
174
|
+
'browser_set_muted',
|
|
159
175
|
'browser_snapshot',
|
|
160
176
|
'get_console_logs',
|
|
161
177
|
'reload_page',
|
|
162
178
|
'screenshot',
|
|
163
179
|
]);
|
|
164
|
-
function getDefaultRuntimeCommandTimeoutMs(method) {
|
|
180
|
+
function getDefaultRuntimeCommandTimeoutMs(method, params) {
|
|
165
181
|
if (method === 'asset_render_preview') {
|
|
166
182
|
return ASSET_RENDER_PREVIEW_TIMEOUT_MS;
|
|
167
183
|
}
|
|
168
184
|
if (method === 'ui_render_preview') {
|
|
169
185
|
return UI_RENDER_PREVIEW_TIMEOUT_MS;
|
|
170
186
|
}
|
|
187
|
+
if (method === 'run_sequence') {
|
|
188
|
+
return HAND_SEQUENCE_COMMAND_TIMEOUT_MS;
|
|
189
|
+
}
|
|
190
|
+
if (TIMED_RUNTIME_METHODS.has(method)) {
|
|
191
|
+
// A long fade or ramp gets the default budget on top of its duration.
|
|
192
|
+
const duration = params?.duration;
|
|
193
|
+
return typeof duration === 'number' && duration > 0
|
|
194
|
+
? DEFAULT_RUNTIME_COMMAND_TIMEOUT_MS +
|
|
195
|
+
Math.ceil(Math.min(duration, MAX_TIMED_COMMAND_DURATION_S) * 1000)
|
|
196
|
+
: DEFAULT_RUNTIME_COMMAND_TIMEOUT_MS;
|
|
197
|
+
}
|
|
171
198
|
return BROWSER_RUNTIME_METHODS.has(method)
|
|
172
199
|
? BROWSER_RUNTIME_COMMAND_TIMEOUT_MS
|
|
173
200
|
: DEFAULT_RUNTIME_COMMAND_TIMEOUT_MS;
|
|
@@ -181,6 +208,41 @@ function describeValue(value) {
|
|
|
181
208
|
}
|
|
182
209
|
return typeof value;
|
|
183
210
|
}
|
|
211
|
+
/**
|
|
212
|
+
* The error of the union member a value was evidently meant for: the only one
|
|
213
|
+
* of its type, or the object shape sharing the most of its keys. A misspelt
|
|
214
|
+
* key in a hand direction then reports that key, not "a supported shape".
|
|
215
|
+
*/
|
|
216
|
+
function closestShapeError(value, candidates, errors) {
|
|
217
|
+
const kind = describeValue(value);
|
|
218
|
+
const sameKind = candidates
|
|
219
|
+
.map((candidate, index) => ({ candidate, error: errors[index] }))
|
|
220
|
+
.filter(({ candidate }) => candidate.type == null ||
|
|
221
|
+
candidate.type === kind ||
|
|
222
|
+
(candidate.type === 'integer' && kind === 'number'));
|
|
223
|
+
if (sameKind.length === 1) {
|
|
224
|
+
return sameKind[0].error;
|
|
225
|
+
}
|
|
226
|
+
if (!isRecord$6(value)) {
|
|
227
|
+
return null;
|
|
228
|
+
}
|
|
229
|
+
const keys = Object.keys(value);
|
|
230
|
+
let best = null;
|
|
231
|
+
let bestScore = 0;
|
|
232
|
+
let tied = false;
|
|
233
|
+
for (const entry of sameKind) {
|
|
234
|
+
const score = keys.filter((key) => entry.candidate.properties?.[key] != null).length;
|
|
235
|
+
if (score > bestScore) {
|
|
236
|
+
best = entry;
|
|
237
|
+
bestScore = score;
|
|
238
|
+
tied = false;
|
|
239
|
+
}
|
|
240
|
+
else if (score === bestScore) {
|
|
241
|
+
tied = true;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
return best && !tied ? best.error : null;
|
|
245
|
+
}
|
|
184
246
|
function assertSchemaValue(value, schema, path) {
|
|
185
247
|
if (schema.oneOf?.length) {
|
|
186
248
|
const matches = schema.oneOf.filter((candidate) => {
|
|
@@ -197,17 +259,20 @@ function assertSchemaValue(value, schema, path) {
|
|
|
197
259
|
}
|
|
198
260
|
}
|
|
199
261
|
if (schema.anyOf?.length) {
|
|
262
|
+
const errors = [];
|
|
200
263
|
const matches = schema.anyOf.some((candidate) => {
|
|
201
264
|
try {
|
|
202
265
|
assertSchemaValue(value, candidate, path);
|
|
203
266
|
return true;
|
|
204
267
|
}
|
|
205
|
-
catch {
|
|
268
|
+
catch (error) {
|
|
269
|
+
errors.push(error);
|
|
206
270
|
return false;
|
|
207
271
|
}
|
|
208
272
|
});
|
|
209
273
|
if (!matches) {
|
|
210
|
-
throw
|
|
274
|
+
throw (closestShapeError(value, schema.anyOf, errors) ??
|
|
275
|
+
new Error(`${path} must match a supported shape`));
|
|
211
276
|
}
|
|
212
277
|
}
|
|
213
278
|
if (schema.not) {
|
|
@@ -296,6 +361,7 @@ function assertSchemaValue(value, schema, path) {
|
|
|
296
361
|
const IWSDK_RUNTIME_LOGS_DIR = '.iwsdk/runtime/logs';
|
|
297
362
|
const IWSDK_RUNTIME_SESSION_PATH = '.iwsdk/runtime/session.json';
|
|
298
363
|
const IWSDK_RUNTIME_LAUNCH_PATH = '.iwsdk/runtime/launch.json';
|
|
364
|
+
const IWSDK_RUNTIME_COMMAND_CREDENTIAL_PATH = '.iwsdk/runtime/command-credential.json';
|
|
299
365
|
const IWSDK_RUNTIME_STATE_SCHEMA_VERSION = 2;
|
|
300
366
|
const IWSDK_RUNTIME_BROWSER_READY_SCHEMA_VERSION = 2;
|
|
301
367
|
const INTERNAL_BROWSER_PROBE_METHOD = '__iwsdk_browser_probe';
|
|
@@ -638,6 +704,364 @@ const SCENE_OBJECT_INSPECTION_ASSESSMENT_SCHEMA = {
|
|
|
638
704
|
},
|
|
639
705
|
required: ['id', 'status', 'evidenceRefs', 'observation'],
|
|
640
706
|
};
|
|
707
|
+
// =============================================================================
|
|
708
|
+
// Emulated device shapes
|
|
709
|
+
// =============================================================================
|
|
710
|
+
// The xr_* schemas mirror IWER's extension-bridge contract and its JSON Schema
|
|
711
|
+
// catalog (`iwer/lib/remote/catalog.js`): the same parameters, unions and
|
|
712
|
+
// descriptions. vite-plugin-dev's tests compare them with the catalog of the
|
|
713
|
+
// IWER it installs.
|
|
714
|
+
const XR_POSITION_SCHEMA = {
|
|
715
|
+
type: 'object',
|
|
716
|
+
description: 'A 3D position in metres, XR-origin relative.',
|
|
717
|
+
properties: {
|
|
718
|
+
x: { type: 'number' },
|
|
719
|
+
y: { type: 'number' },
|
|
720
|
+
z: { type: 'number' },
|
|
721
|
+
},
|
|
722
|
+
required: ['x', 'y', 'z'],
|
|
723
|
+
};
|
|
724
|
+
const XR_STRICT_POSITION_SCHEMA = {
|
|
725
|
+
...XR_POSITION_SCHEMA,
|
|
726
|
+
additionalProperties: false,
|
|
727
|
+
};
|
|
728
|
+
const XR_TUPLE3_SCHEMA = {
|
|
729
|
+
type: 'array',
|
|
730
|
+
items: { type: 'number' },
|
|
731
|
+
minItems: 3,
|
|
732
|
+
maxItems: 3,
|
|
733
|
+
};
|
|
734
|
+
const XR_QUATERNION_SCHEMA = {
|
|
735
|
+
type: 'object',
|
|
736
|
+
additionalProperties: false,
|
|
737
|
+
properties: {
|
|
738
|
+
x: { type: 'number', description: 'Quaternion X component' },
|
|
739
|
+
y: { type: 'number', description: 'Quaternion Y component' },
|
|
740
|
+
z: { type: 'number', description: 'Quaternion Z component' },
|
|
741
|
+
w: { type: 'number', description: 'Quaternion W component' },
|
|
742
|
+
},
|
|
743
|
+
required: ['x', 'y', 'z', 'w'],
|
|
744
|
+
};
|
|
745
|
+
const XR_EULER_SCHEMA = {
|
|
746
|
+
type: 'object',
|
|
747
|
+
additionalProperties: false,
|
|
748
|
+
properties: {
|
|
749
|
+
pitch: { type: 'number', description: 'Pitch in degrees (X rotation)' },
|
|
750
|
+
yaw: { type: 'number', description: 'Yaw in degrees (Y rotation)' },
|
|
751
|
+
roll: { type: 'number', description: 'Roll in degrees (Z rotation)' },
|
|
752
|
+
},
|
|
753
|
+
};
|
|
754
|
+
const XR_ORIENTATION_SCHEMA = {
|
|
755
|
+
anyOf: [XR_QUATERNION_SCHEMA, XR_EULER_SCHEMA],
|
|
756
|
+
description: 'Orientation as a quaternion {x,y,z,w} or euler degrees {pitch,yaw,roll}.',
|
|
757
|
+
};
|
|
758
|
+
const XR_DIRECTION_SCHEMA = {
|
|
759
|
+
anyOf: [
|
|
760
|
+
{
|
|
761
|
+
type: 'string',
|
|
762
|
+
enum: [
|
|
763
|
+
'up',
|
|
764
|
+
'down',
|
|
765
|
+
'forward',
|
|
766
|
+
'back',
|
|
767
|
+
'left',
|
|
768
|
+
'right',
|
|
769
|
+
'toward-head',
|
|
770
|
+
'away-from-head',
|
|
771
|
+
],
|
|
772
|
+
},
|
|
773
|
+
XR_STRICT_POSITION_SCHEMA,
|
|
774
|
+
XR_TUPLE3_SCHEMA,
|
|
775
|
+
],
|
|
776
|
+
description: 'A direction: a name (up, down, forward, back, left, right, toward-head, away-from-head; forward/left/right follow where the headset faces) or an XR-origin vector (approximate is fine).',
|
|
777
|
+
};
|
|
778
|
+
// Hand directions come before quaternion/euler and every orientation object
|
|
779
|
+
// is strict, so a misspelt hand direction is an error, not an empty euler.
|
|
780
|
+
const XR_HAND_DIRECTIONS_SCHEMA = {
|
|
781
|
+
type: 'object',
|
|
782
|
+
additionalProperties: false,
|
|
783
|
+
description: 'Hand orientation by direction: where the fingers point plus where the palm faces OR where the thumb side points. A thumbs up is {fingers:"forward", thumb:"up"}.',
|
|
784
|
+
properties: {
|
|
785
|
+
fingers: XR_DIRECTION_SCHEMA,
|
|
786
|
+
palm: XR_DIRECTION_SCHEMA,
|
|
787
|
+
thumb: XR_DIRECTION_SCHEMA,
|
|
788
|
+
},
|
|
789
|
+
required: ['fingers'],
|
|
790
|
+
};
|
|
791
|
+
const XR_PLACEMENT_ORIENTATION_SCHEMA = {
|
|
792
|
+
anyOf: [
|
|
793
|
+
{ type: 'string', enum: ['auto'] },
|
|
794
|
+
XR_HAND_DIRECTIONS_SCHEMA,
|
|
795
|
+
XR_QUATERNION_SCHEMA,
|
|
796
|
+
XR_EULER_SCHEMA,
|
|
797
|
+
],
|
|
798
|
+
description: 'For hands: {fingers, palm|thumb} directions, or "auto" (a natural reach from the shoulder toward the position). A quaternion {x,y,z,w} or euler degrees {pitch,yaw,roll} sets the device (for hands, the target ray) as before.',
|
|
799
|
+
};
|
|
800
|
+
const XR_HAND_ANCHOR_SCHEMA = {
|
|
801
|
+
type: 'string',
|
|
802
|
+
description: 'Hands: the point of the hand `position` places and keeps there while the hand changes shape: index-tip (press), grasp (where thumb and index tips meet in a full pinch), wrist (gestures), grip, thumb-tip, target-ray (default), or a WebXR joint name.',
|
|
803
|
+
};
|
|
804
|
+
const XR_EASING_SCHEMA = {
|
|
805
|
+
type: 'string',
|
|
806
|
+
enum: ['linear', 'minimum-jerk'],
|
|
807
|
+
description: 'Time curve; minimum-jerk starts and stops like a human reach.',
|
|
808
|
+
};
|
|
809
|
+
const XR_HAND_DEVICE_SCHEMA = {
|
|
810
|
+
type: 'string',
|
|
811
|
+
enum: ['hand-left', 'hand-right'],
|
|
812
|
+
description: 'A hand-tracking device.',
|
|
813
|
+
};
|
|
814
|
+
// One short gloss per built-in IWER hand pose, for xr_set_hand_pose. Keep in
|
|
815
|
+
// sync with IWER's BUILTIN_HAND_POSE_IDS (checked by vite-plugin-dev's tests).
|
|
816
|
+
const XR_HAND_POSE_GLOSS = {
|
|
817
|
+
default: 'relaxed',
|
|
818
|
+
pinch: 'thumb and index touching',
|
|
819
|
+
point: 'index extended',
|
|
820
|
+
open: 'high five',
|
|
821
|
+
flat: 'fingers together, blade',
|
|
822
|
+
fist: 'closed fist',
|
|
823
|
+
'thumbs-up': 'fist with thumb up',
|
|
824
|
+
'finger-gun': 'index and thumb out',
|
|
825
|
+
peace: 'V sign',
|
|
826
|
+
three: 'index, middle, ring up',
|
|
827
|
+
four: 'four fingers up, thumb folded',
|
|
828
|
+
ok: 'thumb-index circle',
|
|
829
|
+
rock: 'index and pinky out',
|
|
830
|
+
call: 'thumb and pinky out',
|
|
831
|
+
'pinch-middle': 'thumb to middle tip',
|
|
832
|
+
'pinch-ring': 'thumb to ring tip',
|
|
833
|
+
'pinch-pinky': 'thumb to pinky tip',
|
|
834
|
+
tripod: 'thumb, index, middle tips together',
|
|
835
|
+
'key-pinch': 'thumb against side of index',
|
|
836
|
+
microgesture: 'loose curl, thumb by the side of the index',
|
|
837
|
+
grip: 'wrapped around a handle',
|
|
838
|
+
claw: 'holding a ball',
|
|
839
|
+
hook: 'carrying a bag handle',
|
|
840
|
+
cup: 'scooping water',
|
|
841
|
+
};
|
|
842
|
+
const XR_HAND_POSE_GLOSSES = Object.entries(XR_HAND_POSE_GLOSS)
|
|
843
|
+
.map(([id, gloss]) => `${id} (${gloss})`)
|
|
844
|
+
.join(', ');
|
|
845
|
+
const XR_FINGER_POSE_SCHEMA = {
|
|
846
|
+
anyOf: [
|
|
847
|
+
{ type: 'string' },
|
|
848
|
+
{
|
|
849
|
+
type: 'object',
|
|
850
|
+
additionalProperties: false,
|
|
851
|
+
properties: {
|
|
852
|
+
poseId: { type: 'string' },
|
|
853
|
+
weight: { type: 'number', minimum: 0, maximum: 1 },
|
|
854
|
+
},
|
|
855
|
+
required: ['poseId'],
|
|
856
|
+
},
|
|
857
|
+
],
|
|
858
|
+
description: 'A pose id, or {poseId, weight} where weight 0-1 blends from the base pose toward it.',
|
|
859
|
+
};
|
|
860
|
+
const XR_FINGER_POSES_SCHEMA = {
|
|
861
|
+
type: 'object',
|
|
862
|
+
additionalProperties: false,
|
|
863
|
+
description: 'Fingers to take from other poses, on top of poseId.',
|
|
864
|
+
properties: {
|
|
865
|
+
thumb: XR_FINGER_POSE_SCHEMA,
|
|
866
|
+
index: XR_FINGER_POSE_SCHEMA,
|
|
867
|
+
middle: XR_FINGER_POSE_SCHEMA,
|
|
868
|
+
ring: XR_FINGER_POSE_SCHEMA,
|
|
869
|
+
pinky: XR_FINGER_POSE_SCHEMA,
|
|
870
|
+
},
|
|
871
|
+
};
|
|
872
|
+
const XR_INPUT_STATE_SCHEMA = {
|
|
873
|
+
type: 'object',
|
|
874
|
+
additionalProperties: false,
|
|
875
|
+
properties: {
|
|
876
|
+
connected: { type: 'boolean' },
|
|
877
|
+
position: XR_POSITION_SCHEMA,
|
|
878
|
+
orientation: XR_ORIENTATION_SCHEMA,
|
|
879
|
+
},
|
|
880
|
+
};
|
|
881
|
+
const XR_HAND_STATE_SCHEMA = {
|
|
882
|
+
...XR_INPUT_STATE_SCHEMA,
|
|
883
|
+
properties: {
|
|
884
|
+
...XR_INPUT_STATE_SCHEMA.properties,
|
|
885
|
+
poseId: { type: 'string' },
|
|
886
|
+
pinchValue: { type: 'number', minimum: 0, maximum: 1 },
|
|
887
|
+
},
|
|
888
|
+
};
|
|
889
|
+
const XR_MICROGESTURE_SCHEMA = {
|
|
890
|
+
type: 'string',
|
|
891
|
+
enum: ['swipe-left', 'swipe-right', 'swipe-forward', 'swipe-backward', 'tap'],
|
|
892
|
+
description: "A thumb microgesture on the side of the curled index finger: swipe left / right (the user's), forward (over the finger), backward (toward the palm), or tap.",
|
|
893
|
+
};
|
|
894
|
+
// --- run_sequence ------------------------------------------------------------
|
|
895
|
+
const XR_FRACTION_SCHEMA = {
|
|
896
|
+
type: 'number',
|
|
897
|
+
minimum: 0,
|
|
898
|
+
maximum: 1,
|
|
899
|
+
};
|
|
900
|
+
const XR_TIMED_WINDOW_PROPERTIES = {
|
|
901
|
+
start: XR_FRACTION_SCHEMA,
|
|
902
|
+
end: XR_FRACTION_SCHEMA,
|
|
903
|
+
easing: XR_EASING_SCHEMA,
|
|
904
|
+
};
|
|
905
|
+
const XR_SEQUENCE_TARGET_SCHEMA = {
|
|
906
|
+
anyOf: [
|
|
907
|
+
{ type: 'string' },
|
|
908
|
+
{
|
|
909
|
+
type: 'object',
|
|
910
|
+
additionalProperties: false,
|
|
911
|
+
properties: {
|
|
912
|
+
name: { type: 'string' },
|
|
913
|
+
face: {
|
|
914
|
+
type: 'string',
|
|
915
|
+
enum: ['front', 'back', 'top', 'bottom', 'left', 'right'],
|
|
916
|
+
},
|
|
917
|
+
},
|
|
918
|
+
required: ['name'],
|
|
919
|
+
},
|
|
920
|
+
],
|
|
921
|
+
description: 'A target name from xr_list_targets, or {name, face}; resolved again when the step starts.',
|
|
922
|
+
};
|
|
923
|
+
const XR_SEQUENCE_POSITION_SCHEMA = {
|
|
924
|
+
anyOf: [
|
|
925
|
+
XR_TUPLE3_SCHEMA,
|
|
926
|
+
{
|
|
927
|
+
type: 'object',
|
|
928
|
+
additionalProperties: false,
|
|
929
|
+
properties: {
|
|
930
|
+
x: { type: 'number' },
|
|
931
|
+
y: { type: 'number' },
|
|
932
|
+
z: { type: 'number' },
|
|
933
|
+
normal: XR_DIRECTION_SCHEMA,
|
|
934
|
+
},
|
|
935
|
+
required: ['x', 'y', 'z'],
|
|
936
|
+
},
|
|
937
|
+
{
|
|
938
|
+
type: 'object',
|
|
939
|
+
additionalProperties: false,
|
|
940
|
+
properties: {
|
|
941
|
+
relativeTo: { type: 'string', enum: ['head', 'body'] },
|
|
942
|
+
x: { type: 'number' },
|
|
943
|
+
y: { type: 'number' },
|
|
944
|
+
z: { type: 'number' },
|
|
945
|
+
},
|
|
946
|
+
required: ['relativeTo', 'x', 'y', 'z'],
|
|
947
|
+
},
|
|
948
|
+
],
|
|
949
|
+
description: 'XR-origin metres (optionally with the surface normal), or {relativeTo:"head"|"body", x right, y up, z back} turning with the body, never with head pitch; "body" measures y from the floor.',
|
|
950
|
+
};
|
|
951
|
+
const XR_STAND_OFF_SCHEMA = {
|
|
952
|
+
type: 'object',
|
|
953
|
+
additionalProperties: false,
|
|
954
|
+
properties: {
|
|
955
|
+
distance: { type: 'number', minimum: 0 },
|
|
956
|
+
normal: XR_DIRECTION_SCHEMA,
|
|
957
|
+
},
|
|
958
|
+
required: ['distance'],
|
|
959
|
+
};
|
|
960
|
+
const XR_SEQUENCE_STEP_SCHEMA = {
|
|
961
|
+
type: 'object',
|
|
962
|
+
additionalProperties: false,
|
|
963
|
+
properties: {
|
|
964
|
+
label: { type: 'string' },
|
|
965
|
+
duration: {
|
|
966
|
+
anyOf: [
|
|
967
|
+
{ type: 'number', minimum: 0 },
|
|
968
|
+
{ type: 'string', enum: ['auto'] },
|
|
969
|
+
],
|
|
970
|
+
description: 'Seconds for the reach, or "auto" (the default for moves): scaled to distance, turn and pose change.',
|
|
971
|
+
},
|
|
972
|
+
target: XR_SEQUENCE_TARGET_SCHEMA,
|
|
973
|
+
position: XR_SEQUENCE_POSITION_SCHEMA,
|
|
974
|
+
offset: {
|
|
975
|
+
anyOf: [
|
|
976
|
+
XR_STRICT_POSITION_SCHEMA,
|
|
977
|
+
{
|
|
978
|
+
type: 'object',
|
|
979
|
+
additionalProperties: false,
|
|
980
|
+
properties: { normal: { type: 'number' } },
|
|
981
|
+
required: ['normal'],
|
|
982
|
+
},
|
|
983
|
+
],
|
|
984
|
+
description: 'Added to the target or position; {normal:d} moves d metres along the surface normal. Setting a held object down: its clearance above the surface.',
|
|
985
|
+
},
|
|
986
|
+
approach: {
|
|
987
|
+
...XR_STAND_OFF_SCHEMA,
|
|
988
|
+
description: 'Come straight in along the surface normal (straight down while holding) for the last `distance` metres.',
|
|
989
|
+
},
|
|
990
|
+
depart: {
|
|
991
|
+
...XR_STAND_OFF_SCHEMA,
|
|
992
|
+
description: 'Lift straight off first, along the last surface normal (straight up while holding).',
|
|
993
|
+
},
|
|
994
|
+
press: {
|
|
995
|
+
type: 'object',
|
|
996
|
+
additionalProperties: false,
|
|
997
|
+
properties: {
|
|
998
|
+
depth: { type: 'number', minimum: 0, maximum: 0.015 },
|
|
999
|
+
hold: { type: 'number', minimum: 0, maximum: 0.2 },
|
|
1000
|
+
retract: { type: 'number', minimum: 0.05 },
|
|
1001
|
+
},
|
|
1002
|
+
description: 'Press the target with the index fingertip: reach, go `depth` m (default 0.008) past the surface, hold `hold` s, pull back `retract` m. Implies anchor index-tip and pose point. Use this for buttons, not a pinch.',
|
|
1003
|
+
},
|
|
1004
|
+
anchor: XR_HAND_ANCHOR_SCHEMA,
|
|
1005
|
+
orientation: XR_PLACEMENT_ORIENTATION_SCHEMA,
|
|
1006
|
+
poseId: {
|
|
1007
|
+
anyOf: [
|
|
1008
|
+
{ type: 'string' },
|
|
1009
|
+
{
|
|
1010
|
+
type: 'object',
|
|
1011
|
+
additionalProperties: false,
|
|
1012
|
+
properties: {
|
|
1013
|
+
value: { type: 'string' },
|
|
1014
|
+
...XR_TIMED_WINDOW_PROPERTIES,
|
|
1015
|
+
},
|
|
1016
|
+
required: ['value'],
|
|
1017
|
+
},
|
|
1018
|
+
],
|
|
1019
|
+
description: 'Pose to fade to during the step; {value, start, end} as fractions of the reach. Never "pinch": poses do not select.',
|
|
1020
|
+
},
|
|
1021
|
+
fingers: XR_FINGER_POSES_SCHEMA,
|
|
1022
|
+
pinch: {
|
|
1023
|
+
anyOf: [
|
|
1024
|
+
XR_FRACTION_SCHEMA,
|
|
1025
|
+
{
|
|
1026
|
+
type: 'object',
|
|
1027
|
+
additionalProperties: false,
|
|
1028
|
+
properties: {
|
|
1029
|
+
value: XR_FRACTION_SCHEMA,
|
|
1030
|
+
...XR_TIMED_WINDOW_PROPERTIES,
|
|
1031
|
+
},
|
|
1032
|
+
required: ['value'],
|
|
1033
|
+
},
|
|
1034
|
+
{
|
|
1035
|
+
type: 'object',
|
|
1036
|
+
additionalProperties: false,
|
|
1037
|
+
properties: {
|
|
1038
|
+
value: XR_FRACTION_SCHEMA,
|
|
1039
|
+
at: { type: 'string', enum: ['arrival'] },
|
|
1040
|
+
},
|
|
1041
|
+
required: ['value', 'at'],
|
|
1042
|
+
},
|
|
1043
|
+
],
|
|
1044
|
+
description: 'Pinch 0..1 (selects at 0.9, lets go at 0.7). {value:1, at:"arrival"} opens to a grip and closes as the hand arrives; on a target it implies anchor grasp.',
|
|
1045
|
+
},
|
|
1046
|
+
body: {
|
|
1047
|
+
type: 'object',
|
|
1048
|
+
additionalProperties: false,
|
|
1049
|
+
properties: {
|
|
1050
|
+
position: XR_SEQUENCE_POSITION_SCHEMA,
|
|
1051
|
+
stepTo: XR_SEQUENCE_TARGET_SCHEMA,
|
|
1052
|
+
facing: XR_DIRECTION_SCHEMA,
|
|
1053
|
+
},
|
|
1054
|
+
description: 'Move the body first: to position, or a step closer to stepTo; facing turns it. The hand goes along.',
|
|
1055
|
+
},
|
|
1056
|
+
microgesture: {
|
|
1057
|
+
...XR_MICROGESTURE_SCHEMA,
|
|
1058
|
+
description: 'Make a microgesture where the hand is, as a step of its own (only label alongside); events report it by name.',
|
|
1059
|
+
},
|
|
1060
|
+
},
|
|
1061
|
+
};
|
|
1062
|
+
const XR_RUN_SEQUENCE_DESCRIPTION = "Run a whole hand interaction in one call with natural motion: one smooth arc per move, the head turning to each target first, the body leaning or stepping closer when needed, the other hand resting by the hip. Steps run in order and each step's properties animate together. Positions are XR-origin metres; -Z is forward, +Y up. Resolves when done with the select events the app saw, step times, press contact times, warnings and the final hand state. " +
|
|
1063
|
+
'Rules: press buttons with `press`, not a pinch. To pick something up, target it with `pinch:{value:1,at:"arrival"}` (the grasp point, between thumb and index, goes on the target). While holding, targeting a surface sets the object down on it; let go with pinch 0. The anchor carries over to later steps. A thumbs up is poseId "thumbs-up" with orientation {fingers:"forward",thumb:"up"} at a head-relative position. A microgesture is a step of its own, {"microgesture":"swipe-left"}, with the hand raised in view and the pinch open (see xr_microgesture). Get names from xr_list_targets and check a plan with dryRun:true first. ' +
|
|
1064
|
+
'Example: {"device":"hand-right","steps":[{"label":"reach","target":"button","press":{}},{"label":"grab","target":"piece","approach":{"distance":0.06},"pinch":{"value":1,"at":"arrival"}},{"label":"carry","target":"tray","depart":{"distance":0.08},"approach":{"distance":0.06}},{"label":"release","pinch":0,"duration":0.15},{"label":"thumbs up","position":{"relativeTo":"head","x":0.12,"y":-0.35,"z":-0.35},"anchor":"wrist","orientation":{"fingers":"forward","thumb":"up"},"poseId":"thumbs-up"},{"label":"hold","duration":1}]}';
|
|
641
1065
|
const ALL_RUNTIME_MCP_TOOLS = [
|
|
642
1066
|
// =============================================================================
|
|
643
1067
|
// Session Management
|
|
@@ -696,9 +1120,10 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
696
1120
|
},
|
|
697
1121
|
{
|
|
698
1122
|
name: 'xr_set_transform',
|
|
699
|
-
description: 'Set position and/or orientation
|
|
1123
|
+
description: 'Set a device position and/or orientation immediately (XR-origin). Gaze accepts orientation only; move its origin with the headset. Hands: `anchor` puts that point of the hand at `position` and keeps it there while the hand changes shape (e.g. anchor "index-tip" on a button, "grasp" on an object before pinching); orientation may be hand directions or "auto". Requires an active session.',
|
|
700
1124
|
inputSchema: {
|
|
701
1125
|
type: 'object',
|
|
1126
|
+
additionalProperties: false,
|
|
702
1127
|
properties: {
|
|
703
1128
|
device: {
|
|
704
1129
|
type: 'string',
|
|
@@ -713,8 +1138,7 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
713
1138
|
description: 'The device to move',
|
|
714
1139
|
},
|
|
715
1140
|
position: {
|
|
716
|
-
|
|
717
|
-
description: 'World position in meters',
|
|
1141
|
+
...XR_POSITION_SCHEMA,
|
|
718
1142
|
properties: {
|
|
719
1143
|
x: { type: 'number', description: 'X position (left/right)' },
|
|
720
1144
|
y: {
|
|
@@ -727,25 +1151,8 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
727
1151
|
},
|
|
728
1152
|
},
|
|
729
1153
|
},
|
|
730
|
-
orientation:
|
|
731
|
-
|
|
732
|
-
description: 'Rotation as quaternion {x,y,z,w} or euler angles {pitch,yaw,roll} in degrees',
|
|
733
|
-
properties: {
|
|
734
|
-
x: { type: 'number', description: 'Quaternion X component' },
|
|
735
|
-
y: { type: 'number', description: 'Quaternion Y component' },
|
|
736
|
-
z: { type: 'number', description: 'Quaternion Z component' },
|
|
737
|
-
w: { type: 'number', description: 'Quaternion W component' },
|
|
738
|
-
pitch: {
|
|
739
|
-
type: 'number',
|
|
740
|
-
description: 'Pitch in degrees (X rotation)',
|
|
741
|
-
},
|
|
742
|
-
yaw: { type: 'number', description: 'Yaw in degrees (Y rotation)' },
|
|
743
|
-
roll: {
|
|
744
|
-
type: 'number',
|
|
745
|
-
description: 'Roll in degrees (Z rotation)',
|
|
746
|
-
},
|
|
747
|
-
},
|
|
748
|
-
},
|
|
1154
|
+
orientation: XR_PLACEMENT_ORIENTATION_SCHEMA,
|
|
1155
|
+
anchor: XR_HAND_ANCHOR_SCHEMA,
|
|
749
1156
|
},
|
|
750
1157
|
required: ['device'],
|
|
751
1158
|
},
|
|
@@ -788,9 +1195,10 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
788
1195
|
},
|
|
789
1196
|
{
|
|
790
1197
|
name: 'xr_animate_to',
|
|
791
|
-
description: '
|
|
1198
|
+
description: 'Animate a device to a target position and/or orientation (XR-origin) over `duration` seconds (default 0.5), from where it is when the animation starts. Gaze accepts orientation only. Hands take `anchor` and hand directions as in xr_set_transform. easing "minimum-jerk" moves like a person (default "linear"). For multi-step hand interactions (press, grab, carry, gestures) prefer xr_run_sequence. Requires an active, rendering session.',
|
|
792
1199
|
inputSchema: {
|
|
793
1200
|
type: 'object',
|
|
1201
|
+
additionalProperties: false,
|
|
794
1202
|
properties: {
|
|
795
1203
|
device: {
|
|
796
1204
|
type: 'string',
|
|
@@ -804,38 +1212,15 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
804
1212
|
],
|
|
805
1213
|
description: 'The device to animate',
|
|
806
1214
|
},
|
|
807
|
-
position:
|
|
808
|
-
|
|
809
|
-
description: 'Target world position in meters',
|
|
810
|
-
properties: {
|
|
811
|
-
x: { type: 'number' },
|
|
812
|
-
y: { type: 'number' },
|
|
813
|
-
z: { type: 'number' },
|
|
814
|
-
},
|
|
815
|
-
},
|
|
816
|
-
orientation: {
|
|
817
|
-
type: 'object',
|
|
818
|
-
description: 'Target rotation as quaternion {x,y,z,w} or euler angles {pitch,yaw,roll} in degrees',
|
|
819
|
-
properties: {
|
|
820
|
-
x: { type: 'number', description: 'Quaternion X component' },
|
|
821
|
-
y: { type: 'number', description: 'Quaternion Y component' },
|
|
822
|
-
z: { type: 'number', description: 'Quaternion Z component' },
|
|
823
|
-
w: { type: 'number', description: 'Quaternion W component' },
|
|
824
|
-
pitch: {
|
|
825
|
-
type: 'number',
|
|
826
|
-
description: 'Pitch in degrees (X rotation)',
|
|
827
|
-
},
|
|
828
|
-
yaw: { type: 'number', description: 'Yaw in degrees (Y rotation)' },
|
|
829
|
-
roll: {
|
|
830
|
-
type: 'number',
|
|
831
|
-
description: 'Roll in degrees (Z rotation)',
|
|
832
|
-
},
|
|
833
|
-
},
|
|
834
|
-
},
|
|
1215
|
+
position: XR_POSITION_SCHEMA,
|
|
1216
|
+
orientation: XR_PLACEMENT_ORIENTATION_SCHEMA,
|
|
835
1217
|
duration: {
|
|
836
1218
|
type: 'number',
|
|
1219
|
+
exclusiveMinimum: 0,
|
|
837
1220
|
description: 'Animation duration in seconds (default: 0.5)',
|
|
838
1221
|
},
|
|
1222
|
+
easing: XR_EASING_SCHEMA,
|
|
1223
|
+
anchor: XR_HAND_ANCHOR_SCHEMA,
|
|
839
1224
|
},
|
|
840
1225
|
required: ['device'],
|
|
841
1226
|
},
|
|
@@ -908,9 +1293,10 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
908
1293
|
},
|
|
909
1294
|
{
|
|
910
1295
|
name: 'xr_set_select_value',
|
|
911
|
-
description: 'Set the select (trigger/pinch) value
|
|
1296
|
+
description: 'Set the select value (trigger/pinch), 0..1, optionally ramping from the current value over `duration` seconds (easing default minimum-jerk). A hand pinch selects at 0.9 and lets go at 0.7; a trigger selects above 0. Use xr_select for a full press+release.',
|
|
912
1297
|
inputSchema: {
|
|
913
1298
|
type: 'object',
|
|
1299
|
+
additionalProperties: false,
|
|
914
1300
|
properties: {
|
|
915
1301
|
device: {
|
|
916
1302
|
type: 'string',
|
|
@@ -928,13 +1314,19 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
928
1314
|
maximum: 1,
|
|
929
1315
|
description: 'Select value (0=released, 1=fully pressed/pinched)',
|
|
930
1316
|
},
|
|
1317
|
+
duration: {
|
|
1318
|
+
type: 'number',
|
|
1319
|
+
minimum: 0,
|
|
1320
|
+
description: 'Seconds to ramp from the current value (default: 0)',
|
|
1321
|
+
},
|
|
1322
|
+
easing: XR_EASING_SCHEMA,
|
|
931
1323
|
},
|
|
932
1324
|
required: ['device', 'value'],
|
|
933
1325
|
},
|
|
934
1326
|
},
|
|
935
1327
|
{
|
|
936
1328
|
name: 'xr_select',
|
|
937
|
-
description: 'Perform a
|
|
1329
|
+
description: 'Perform a full select gesture: selectstart on press, then select and selectend on release, holding `duration` seconds (default 0.15). A hand closes its pinch over 0.1 s before the hold and opens it over 0.1 s after. Requires an active session.',
|
|
938
1330
|
inputSchema: {
|
|
939
1331
|
type: 'object',
|
|
940
1332
|
properties: {
|
|
@@ -950,6 +1342,7 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
950
1342
|
},
|
|
951
1343
|
duration: {
|
|
952
1344
|
type: 'number',
|
|
1345
|
+
exclusiveMinimum: 0,
|
|
953
1346
|
description: 'How long to hold in seconds (default: 0.15)',
|
|
954
1347
|
},
|
|
955
1348
|
},
|
|
@@ -1261,7 +1654,7 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
1261
1654
|
// =============================================================================
|
|
1262
1655
|
{
|
|
1263
1656
|
name: 'xr_get_device_state',
|
|
1264
|
-
description: 'Get
|
|
1657
|
+
description: 'Get a full snapshot: headset pose, input mode, controllers, hands (with poseId and pinchValue), gaze, stereo, and FOV. Positions are XR-origin relative.',
|
|
1265
1658
|
inputSchema: {
|
|
1266
1659
|
type: 'object',
|
|
1267
1660
|
properties: {},
|
|
@@ -1270,35 +1663,23 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
1270
1663
|
},
|
|
1271
1664
|
{
|
|
1272
1665
|
name: 'xr_set_device_state',
|
|
1273
|
-
description: '
|
|
1666
|
+
description: 'Apply a partial device state (positions XR-origin; hands also take poseId and pinchValue). Call with no `state` to reset everything to defaults.',
|
|
1274
1667
|
inputSchema: {
|
|
1275
1668
|
type: 'object',
|
|
1669
|
+
additionalProperties: false,
|
|
1276
1670
|
properties: {
|
|
1277
1671
|
state: {
|
|
1278
1672
|
type: 'object',
|
|
1673
|
+
additionalProperties: false,
|
|
1279
1674
|
description: 'Partial device state to apply. Omit to reset to defaults.',
|
|
1280
1675
|
properties: {
|
|
1281
1676
|
headset: {
|
|
1282
1677
|
type: 'object',
|
|
1678
|
+
additionalProperties: false,
|
|
1283
1679
|
description: 'Headset transform',
|
|
1284
1680
|
properties: {
|
|
1285
|
-
position:
|
|
1286
|
-
|
|
1287
|
-
properties: {
|
|
1288
|
-
x: { type: 'number' },
|
|
1289
|
-
y: { type: 'number' },
|
|
1290
|
-
z: { type: 'number' },
|
|
1291
|
-
},
|
|
1292
|
-
},
|
|
1293
|
-
orientation: {
|
|
1294
|
-
type: 'object',
|
|
1295
|
-
properties: {
|
|
1296
|
-
x: { type: 'number' },
|
|
1297
|
-
y: { type: 'number' },
|
|
1298
|
-
z: { type: 'number' },
|
|
1299
|
-
w: { type: 'number' },
|
|
1300
|
-
},
|
|
1301
|
-
},
|
|
1681
|
+
position: XR_POSITION_SCHEMA,
|
|
1682
|
+
orientation: XR_ORIENTATION_SCHEMA,
|
|
1302
1683
|
},
|
|
1303
1684
|
},
|
|
1304
1685
|
inputMode: {
|
|
@@ -1316,46 +1697,20 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
1316
1697
|
},
|
|
1317
1698
|
controllers: {
|
|
1318
1699
|
type: 'object',
|
|
1700
|
+
additionalProperties: false,
|
|
1319
1701
|
description: 'Controller states',
|
|
1320
1702
|
properties: {
|
|
1321
|
-
left:
|
|
1322
|
-
|
|
1323
|
-
properties: {
|
|
1324
|
-
position: { type: 'object' },
|
|
1325
|
-
orientation: { type: 'object' },
|
|
1326
|
-
connected: { type: 'boolean' },
|
|
1327
|
-
},
|
|
1328
|
-
},
|
|
1329
|
-
right: {
|
|
1330
|
-
type: 'object',
|
|
1331
|
-
properties: {
|
|
1332
|
-
position: { type: 'object' },
|
|
1333
|
-
orientation: { type: 'object' },
|
|
1334
|
-
connected: { type: 'boolean' },
|
|
1335
|
-
},
|
|
1336
|
-
},
|
|
1703
|
+
left: XR_INPUT_STATE_SCHEMA,
|
|
1704
|
+
right: XR_INPUT_STATE_SCHEMA,
|
|
1337
1705
|
},
|
|
1338
1706
|
},
|
|
1339
1707
|
hands: {
|
|
1340
1708
|
type: 'object',
|
|
1341
|
-
|
|
1709
|
+
additionalProperties: false,
|
|
1710
|
+
description: 'Hand states, including poseId and pinchValue',
|
|
1342
1711
|
properties: {
|
|
1343
|
-
left:
|
|
1344
|
-
|
|
1345
|
-
properties: {
|
|
1346
|
-
position: { type: 'object' },
|
|
1347
|
-
orientation: { type: 'object' },
|
|
1348
|
-
connected: { type: 'boolean' },
|
|
1349
|
-
},
|
|
1350
|
-
},
|
|
1351
|
-
right: {
|
|
1352
|
-
type: 'object',
|
|
1353
|
-
properties: {
|
|
1354
|
-
position: { type: 'object' },
|
|
1355
|
-
orientation: { type: 'object' },
|
|
1356
|
-
connected: { type: 'boolean' },
|
|
1357
|
-
},
|
|
1358
|
-
},
|
|
1712
|
+
left: XR_HAND_STATE_SCHEMA,
|
|
1713
|
+
right: XR_HAND_STATE_SCHEMA,
|
|
1359
1714
|
},
|
|
1360
1715
|
},
|
|
1361
1716
|
gaze: {
|
|
@@ -1363,16 +1718,8 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
1363
1718
|
additionalProperties: false,
|
|
1364
1719
|
description: 'Gaze state. Its origin is derived from the headset and cannot be set independently.',
|
|
1365
1720
|
properties: {
|
|
1366
|
-
orientation: {
|
|
1367
|
-
type: 'object',
|
|
1368
|
-
properties: {
|
|
1369
|
-
x: { type: 'number' },
|
|
1370
|
-
y: { type: 'number' },
|
|
1371
|
-
z: { type: 'number' },
|
|
1372
|
-
w: { type: 'number' },
|
|
1373
|
-
},
|
|
1374
|
-
},
|
|
1375
1721
|
connected: { type: 'boolean' },
|
|
1722
|
+
orientation: XR_ORIENTATION_SCHEMA,
|
|
1376
1723
|
},
|
|
1377
1724
|
},
|
|
1378
1725
|
},
|
|
@@ -1381,6 +1728,108 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
1381
1728
|
},
|
|
1382
1729
|
},
|
|
1383
1730
|
// =============================================================================
|
|
1731
|
+
// Hands
|
|
1732
|
+
// =============================================================================
|
|
1733
|
+
{
|
|
1734
|
+
name: 'xr_set_hand_pose',
|
|
1735
|
+
description: `Set a named hand pose on a hand device. Poses only shape the hand; to pinch (select), use the pinch value (xr_set_select_value). Poses: ${XR_HAND_POSE_GLOSSES}. ` +
|
|
1736
|
+
'Optionally take individual fingers from other poses, e.g. poseId "open" with fingers {"middle":"fist","ring":"fist","pinky":"fist"}, or blend partway with {"poseId":"fist","weight":0.5}. Setting a pose clears earlier finger overrides. `duration` fades from the hand as shown (easing default minimum-jerk); a hand placed by an anchor keeps the anchor in place.',
|
|
1737
|
+
inputSchema: {
|
|
1738
|
+
type: 'object',
|
|
1739
|
+
additionalProperties: false,
|
|
1740
|
+
properties: {
|
|
1741
|
+
device: XR_HAND_DEVICE_SCHEMA,
|
|
1742
|
+
poseId: { type: 'string' },
|
|
1743
|
+
fingers: XR_FINGER_POSES_SCHEMA,
|
|
1744
|
+
duration: { type: 'number', minimum: 0 },
|
|
1745
|
+
easing: XR_EASING_SCHEMA,
|
|
1746
|
+
},
|
|
1747
|
+
required: ['device', 'poseId'],
|
|
1748
|
+
},
|
|
1749
|
+
},
|
|
1750
|
+
{
|
|
1751
|
+
name: 'xr_microgesture',
|
|
1752
|
+
description: 'Swipe or tap the thumb on the side of the curled index finger, as on Quest. The app sees the hand gamepad (xr-standard) button pressed for 35 ms when the gesture completes: 5 swipe-left, 6 swipe-right, 7 swipe-forward, 8 swipe-backward, 9 tap; no select events. A hand not in the "microgesture" pose takes it first (0.25 s); a hand that is selecting must let go first. Raise the hand side-on in view first, e.g. xr_set_transform with anchor "wrist" and orientation {fingers:[0,-0.6,-0.8],palm:"left"} (right hand; "right" for the left). Requires an active session and hand input.',
|
|
1753
|
+
inputSchema: {
|
|
1754
|
+
type: 'object',
|
|
1755
|
+
additionalProperties: false,
|
|
1756
|
+
properties: {
|
|
1757
|
+
device: XR_HAND_DEVICE_SCHEMA,
|
|
1758
|
+
gesture: XR_MICROGESTURE_SCHEMA,
|
|
1759
|
+
},
|
|
1760
|
+
required: ['device', 'gesture'],
|
|
1761
|
+
},
|
|
1762
|
+
},
|
|
1763
|
+
{
|
|
1764
|
+
name: 'xr_get_hand_state',
|
|
1765
|
+
description: 'Read a hand: pose, finger blends, a pose fade in progress, pinch value, whether select is pressed, its anchor pin, and XR-origin positions of the wrist, grip, grasp point, index tip and thumb tip, plus its finger/palm/thumb directions (joints:"all" adds every joint).',
|
|
1766
|
+
inputSchema: {
|
|
1767
|
+
type: 'object',
|
|
1768
|
+
additionalProperties: false,
|
|
1769
|
+
properties: {
|
|
1770
|
+
device: XR_HAND_DEVICE_SCHEMA,
|
|
1771
|
+
joints: { type: 'string', enum: ['key', 'all'] },
|
|
1772
|
+
},
|
|
1773
|
+
required: ['device'],
|
|
1774
|
+
},
|
|
1775
|
+
},
|
|
1776
|
+
{
|
|
1777
|
+
name: 'xr_run_sequence',
|
|
1778
|
+
description: XR_RUN_SEQUENCE_DESCRIPTION,
|
|
1779
|
+
inputSchema: {
|
|
1780
|
+
type: 'object',
|
|
1781
|
+
additionalProperties: false,
|
|
1782
|
+
properties: {
|
|
1783
|
+
device: XR_HAND_DEVICE_SCHEMA,
|
|
1784
|
+
steps: {
|
|
1785
|
+
type: 'array',
|
|
1786
|
+
items: XR_SEQUENCE_STEP_SCHEMA,
|
|
1787
|
+
minItems: 1,
|
|
1788
|
+
},
|
|
1789
|
+
idleHand: {
|
|
1790
|
+
type: 'string',
|
|
1791
|
+
enum: ['keep', 'rest'],
|
|
1792
|
+
description: 'The other hand: rest it by the hip (the default unless it is pinching) or keep it where it is.',
|
|
1793
|
+
},
|
|
1794
|
+
body: {
|
|
1795
|
+
type: 'string',
|
|
1796
|
+
enum: ['auto', 'fixed'],
|
|
1797
|
+
description: 'auto (default): lean for low targets and step closer to ones out of reach. fixed: out of reach is an error.',
|
|
1798
|
+
},
|
|
1799
|
+
gaze: {
|
|
1800
|
+
type: 'string',
|
|
1801
|
+
enum: ['follow', 'fixed'],
|
|
1802
|
+
description: 'follow (default): the head turns to each target before the hand gets there.',
|
|
1803
|
+
},
|
|
1804
|
+
dryRun: {
|
|
1805
|
+
type: 'boolean',
|
|
1806
|
+
description: 'Validate and return the planned steps, walks, durations and warnings without moving anything.',
|
|
1807
|
+
},
|
|
1808
|
+
},
|
|
1809
|
+
required: ['device', 'steps'],
|
|
1810
|
+
},
|
|
1811
|
+
},
|
|
1812
|
+
{
|
|
1813
|
+
name: 'xr_cancel_sequence',
|
|
1814
|
+
description: 'Stop the running xr_run_sequence where it is; that call returns with cancelled: true and the steps done. The pinch is left as it is (no select events).',
|
|
1815
|
+
inputSchema: {
|
|
1816
|
+
type: 'object',
|
|
1817
|
+
additionalProperties: false,
|
|
1818
|
+
properties: {},
|
|
1819
|
+
required: [],
|
|
1820
|
+
},
|
|
1821
|
+
},
|
|
1822
|
+
{
|
|
1823
|
+
name: 'xr_list_targets',
|
|
1824
|
+
description: "List the named targets xr_run_sequence steps can use, from the page's target resolver: name, box centre and size (XR-origin). resolver:false means the page has none; use positions.",
|
|
1825
|
+
inputSchema: {
|
|
1826
|
+
type: 'object',
|
|
1827
|
+
additionalProperties: false,
|
|
1828
|
+
properties: {},
|
|
1829
|
+
required: [],
|
|
1830
|
+
},
|
|
1831
|
+
},
|
|
1832
|
+
// =============================================================================
|
|
1384
1833
|
// Console Logs (Plugin-specific, not in IWER)
|
|
1385
1834
|
// =============================================================================
|
|
1386
1835
|
{
|
|
@@ -1430,6 +1879,21 @@ const ALL_RUNTIME_MCP_TOOLS = [
|
|
|
1430
1879
|
properties: {},
|
|
1431
1880
|
},
|
|
1432
1881
|
},
|
|
1882
|
+
{
|
|
1883
|
+
name: 'browser_set_muted',
|
|
1884
|
+
description: 'Mute or unmute all audio in the managed browser tab, whose app-mode window hides the browser mute control. The state is shared with the workspace mute button and persists across reloads and browser relaunches for the dev session.',
|
|
1885
|
+
inputSchema: {
|
|
1886
|
+
type: 'object',
|
|
1887
|
+
additionalProperties: false,
|
|
1888
|
+
properties: {
|
|
1889
|
+
muted: {
|
|
1890
|
+
type: 'boolean',
|
|
1891
|
+
description: 'true mutes the tab; false unmutes it',
|
|
1892
|
+
},
|
|
1893
|
+
},
|
|
1894
|
+
required: ['muted'],
|
|
1895
|
+
},
|
|
1896
|
+
},
|
|
1433
1897
|
// =============================================================================
|
|
1434
1898
|
// Managed Workspace Tools
|
|
1435
1899
|
// =============================================================================
|
|
@@ -3002,6 +3466,12 @@ const RUNTIME_TOOL_TO_METHOD = {
|
|
|
3002
3466
|
xr_set_gamepad_state: 'set_gamepad_state',
|
|
3003
3467
|
xr_get_device_state: 'get_device_state',
|
|
3004
3468
|
xr_set_device_state: 'set_device_state',
|
|
3469
|
+
xr_set_hand_pose: 'set_hand_pose',
|
|
3470
|
+
xr_microgesture: 'microgesture',
|
|
3471
|
+
xr_get_hand_state: 'get_hand_state',
|
|
3472
|
+
xr_run_sequence: 'run_sequence',
|
|
3473
|
+
xr_cancel_sequence: 'cancel_sequence',
|
|
3474
|
+
xr_list_targets: 'list_targets',
|
|
3005
3475
|
browser_screenshot: 'screenshot',
|
|
3006
3476
|
browser_snapshot: 'browser_snapshot',
|
|
3007
3477
|
browser_interact: 'browser_interact',
|
|
@@ -3033,8 +3503,15 @@ const ALL_RUNTIME_CLI_PATHS = {
|
|
|
3033
3503
|
browser_profile: ['browser', 'profile'],
|
|
3034
3504
|
xr_get_device_state: ['xr', 'get-device-state'],
|
|
3035
3505
|
xr_set_device_state: ['xr', 'set-device-state'],
|
|
3506
|
+
xr_set_hand_pose: ['xr', 'set-hand-pose'],
|
|
3507
|
+
xr_microgesture: ['xr', 'microgesture'],
|
|
3508
|
+
xr_get_hand_state: ['xr', 'get-hand-state'],
|
|
3509
|
+
xr_run_sequence: ['xr', 'run-sequence'],
|
|
3510
|
+
xr_cancel_sequence: ['xr', 'cancel-sequence'],
|
|
3511
|
+
xr_list_targets: ['xr', 'list-targets'],
|
|
3036
3512
|
browser_get_console_logs: ['browser', 'logs'],
|
|
3037
3513
|
browser_reload_page: ['browser', 'reload'],
|
|
3514
|
+
browser_set_muted: ['browser', 'set-muted'],
|
|
3038
3515
|
workspace_get_state: ['workspace', 'state'],
|
|
3039
3516
|
workspace_set_view: ['workspace', 'set-view'],
|
|
3040
3517
|
workspace_open_scene: ['workspace', 'open-scene'],
|
|
@@ -3139,6 +3616,7 @@ const HOST_BROWSER_MCP_TOOL_NAME_SET = new Set([
|
|
|
3139
3616
|
'browser_profile',
|
|
3140
3617
|
'browser_get_console_logs',
|
|
3141
3618
|
'browser_reload_page',
|
|
3619
|
+
'browser_set_muted',
|
|
3142
3620
|
]);
|
|
3143
3621
|
const APP_TARGET_MCP_TOOL_NAME_SET = new Set([
|
|
3144
3622
|
...HOST_BROWSER_MCP_TOOL_NAME_SET,
|
|
@@ -3708,6 +4186,29 @@ function getRuntimeSessionFilePath(workspaceRoot) {
|
|
|
3708
4186
|
function getRuntimeLaunchFilePath(workspaceRoot) {
|
|
3709
4187
|
return getRuntimeFilePath(workspaceRoot, IWSDK_RUNTIME_LAUNCH_PATH);
|
|
3710
4188
|
}
|
|
4189
|
+
function getRuntimeCommandCredentialFilePath(workspaceRoot) {
|
|
4190
|
+
return getRuntimeFilePath(workspaceRoot, IWSDK_RUNTIME_COMMAND_CREDENTIAL_PATH);
|
|
4191
|
+
}
|
|
4192
|
+
/**
|
|
4193
|
+
* Reads the command credential the dev server wrote for this session. A
|
|
4194
|
+
* credential left by another session is never sent. Without one (an older dev
|
|
4195
|
+
* server writes none), commands go out without a credential. The read is
|
|
4196
|
+
* synchronous so a command starts connecting without an extra turn.
|
|
4197
|
+
*/
|
|
4198
|
+
function getRuntimeCommandToken(session) {
|
|
4199
|
+
let credential;
|
|
4200
|
+
try {
|
|
4201
|
+
credential = JSON.parse(readFileSync(getRuntimeCommandCredentialFilePath(session.workspaceRoot), 'utf8'));
|
|
4202
|
+
}
|
|
4203
|
+
catch {
|
|
4204
|
+
return undefined;
|
|
4205
|
+
}
|
|
4206
|
+
return credential?.sessionId === session.sessionId &&
|
|
4207
|
+
typeof credential.commandToken === 'string' &&
|
|
4208
|
+
credential.commandToken.length > 0
|
|
4209
|
+
? credential.commandToken
|
|
4210
|
+
: undefined;
|
|
4211
|
+
}
|
|
3711
4212
|
function getRuntimeLogsDir(workspaceRoot) {
|
|
3712
4213
|
return path.join(normalizeWorkspaceRoot(workspaceRoot), IWSDK_RUNTIME_LOGS_DIR);
|
|
3713
4214
|
}
|
|
@@ -9075,6 +9576,7 @@ const { CLOSE_TIMEOUT, GUID, kWebSocket } = constants;
|
|
|
9075
9576
|
/** Keep a command connection open for the complete secondary-CDP operation. */
|
|
9076
9577
|
async function acquireBrowserLease(session, timeoutMs) {
|
|
9077
9578
|
const id = randomUUID();
|
|
9579
|
+
const commandToken = getRuntimeCommandToken(session);
|
|
9078
9580
|
const protocol = session.localUrl.startsWith('https:') ? 'wss' : 'ws';
|
|
9079
9581
|
const ws = new WebSocket$1(`${protocol}://127.0.0.1:${session.port}/__iwer_mcp`, { rejectUnauthorized: false });
|
|
9080
9582
|
let granted = false;
|
|
@@ -9093,7 +9595,11 @@ async function acquireBrowserLease(session, timeoutMs) {
|
|
|
9093
9595
|
closed = true;
|
|
9094
9596
|
released = true;
|
|
9095
9597
|
if (granted && ws.readyState === WebSocket$1.OPEN) {
|
|
9096
|
-
ws.send(JSON.stringify({
|
|
9598
|
+
ws.send(JSON.stringify({
|
|
9599
|
+
id,
|
|
9600
|
+
method: '__iwsdk_browser_lease_release',
|
|
9601
|
+
commandToken,
|
|
9602
|
+
}));
|
|
9097
9603
|
}
|
|
9098
9604
|
ws.close();
|
|
9099
9605
|
};
|
|
@@ -9134,6 +9640,7 @@ async function acquireBrowserLease(session, timeoutMs) {
|
|
|
9134
9640
|
targetId: session.browserAutomation?.targetId,
|
|
9135
9641
|
timeoutMs,
|
|
9136
9642
|
},
|
|
9643
|
+
commandToken,
|
|
9137
9644
|
})));
|
|
9138
9645
|
ws.on('message', (raw) => {
|
|
9139
9646
|
try {
|
|
@@ -9278,7 +9785,7 @@ function inferRuntimeIssueCause(message, browser, explicitCause) {
|
|
|
9278
9785
|
}
|
|
9279
9786
|
return undefined;
|
|
9280
9787
|
}
|
|
9281
|
-
async function trySendRuntimeCommand(protocol, port, method, params, target, deadline, connectDeadline, browser, expectedSessionId) {
|
|
9788
|
+
async function trySendRuntimeCommand(protocol, port, method, params, target, deadline, connectDeadline, browser, expectedSessionId, commandToken) {
|
|
9282
9789
|
const inferCause = (message, explicitCause) => target?.deviceClass === 'physical'
|
|
9283
9790
|
? inferPhysicalIssueCause(message, explicitCause)
|
|
9284
9791
|
: inferRuntimeIssueCause(message, browser, explicitCause);
|
|
@@ -9358,6 +9865,7 @@ async function trySendRuntimeCommand(protocol, port, method, params, target, dea
|
|
|
9358
9865
|
method,
|
|
9359
9866
|
params: params ?? {},
|
|
9360
9867
|
...(target ? { target } : {}),
|
|
9868
|
+
commandToken,
|
|
9361
9869
|
});
|
|
9362
9870
|
traceTransport('open', {
|
|
9363
9871
|
method,
|
|
@@ -9499,7 +10007,7 @@ async function trySendRuntimeCommand(protocol, port, method, params, target, dea
|
|
|
9499
10007
|
});
|
|
9500
10008
|
}
|
|
9501
10009
|
async function sendRuntimeCommandBeforeDeadline(options, deadline) {
|
|
9502
|
-
const { method, params, port, runtimeSession, target } = options;
|
|
10010
|
+
const { commandToken, method, params, port, runtimeSession, target } = options;
|
|
9503
10011
|
// Managed browser state describes the host browser only, never a headset.
|
|
9504
10012
|
const browser = target?.deviceClass === 'physical' ? undefined : runtimeSession?.browser;
|
|
9505
10013
|
const protocolOrder = getProtocolOrder(runtimeSession);
|
|
@@ -9511,7 +10019,7 @@ async function sendRuntimeCommandBeforeDeadline(options, deadline) {
|
|
|
9511
10019
|
? Date.now() + FAST_WSS_FALLBACK_TIMEOUT_MS
|
|
9512
10020
|
: deadline;
|
|
9513
10021
|
try {
|
|
9514
|
-
return await trySendRuntimeCommand(firstProtocol, port, method, params, target, deadline, connectDeadline, browser, runtimeSession?.sessionId);
|
|
10022
|
+
return await trySendRuntimeCommand(firstProtocol, port, method, params, target, deadline, connectDeadline, browser, runtimeSession?.sessionId, commandToken);
|
|
9515
10023
|
}
|
|
9516
10024
|
catch (error) {
|
|
9517
10025
|
if (!(error instanceof RuntimeCommandTransportError)) {
|
|
@@ -9538,7 +10046,7 @@ async function sendRuntimeCommandBeforeDeadline(options, deadline) {
|
|
|
9538
10046
|
})
|
|
9539
10047
|
: false,
|
|
9540
10048
|
});
|
|
9541
|
-
return trySendRuntimeCommand(fallbackProtocol, port, method, params, target, deadline, deadline, browser, runtimeSession?.sessionId);
|
|
10049
|
+
return trySendRuntimeCommand(fallbackProtocol, port, method, params, target, deadline, deadline, browser, runtimeSession?.sessionId, commandToken);
|
|
9542
10050
|
}
|
|
9543
10051
|
}
|
|
9544
10052
|
function sceneOpenReadinessRequest(method, params, response, previousSceneSessionId) {
|
|
@@ -9574,6 +10082,7 @@ async function captureSceneOpenSessionBaseline(options, deadline) {
|
|
|
9574
10082
|
while (Date.now() < deadline) {
|
|
9575
10083
|
try {
|
|
9576
10084
|
const stateResponse = await sendRuntimeCommandBeforeDeadline({
|
|
10085
|
+
commandToken: options.commandToken,
|
|
9577
10086
|
method: 'scene_get_state',
|
|
9578
10087
|
params: {},
|
|
9579
10088
|
port: options.port,
|
|
@@ -9631,6 +10140,7 @@ async function confirmOpenedSceneReady(options, response, deadline, previousScen
|
|
|
9631
10140
|
while (Date.now() < deadline) {
|
|
9632
10141
|
try {
|
|
9633
10142
|
const stateResponse = await sendRuntimeCommandBeforeDeadline({
|
|
10143
|
+
commandToken: options.commandToken,
|
|
9634
10144
|
method: 'scene_get_state',
|
|
9635
10145
|
params: {},
|
|
9636
10146
|
port: options.port,
|
|
@@ -9687,9 +10197,17 @@ async function confirmOpenedSceneReady(options, response, deadline, previousScen
|
|
|
9687
10197
|
: 'browser_not_ready',
|
|
9688
10198
|
});
|
|
9689
10199
|
}
|
|
9690
|
-
async function sendRuntimeCommand({ port, method, params, target, timeoutMs = getDefaultRuntimeCommandTimeoutMs(method), runtimeSession, }) {
|
|
10200
|
+
async function sendRuntimeCommand({ port, method, params, target, timeoutMs = getDefaultRuntimeCommandTimeoutMs(method, params), runtimeSession, }) {
|
|
9691
10201
|
const deadline = Date.now() + Math.max(timeoutMs, 1);
|
|
9692
|
-
const
|
|
10202
|
+
const commandToken = runtimeSession == null ? undefined : getRuntimeCommandToken(runtimeSession);
|
|
10203
|
+
const options = {
|
|
10204
|
+
commandToken,
|
|
10205
|
+
method,
|
|
10206
|
+
params,
|
|
10207
|
+
port,
|
|
10208
|
+
runtimeSession,
|
|
10209
|
+
target,
|
|
10210
|
+
};
|
|
9693
10211
|
const previousSceneSessionId = await captureSceneOpenSessionBaseline(options, deadline);
|
|
9694
10212
|
const response = await sendRuntimeCommandBeforeDeadline(options, deadline);
|
|
9695
10213
|
return confirmOpenedSceneReady(options, response, deadline, previousSceneSessionId);
|
|
@@ -28174,7 +28692,7 @@ function reportToolCall(toolName, success, durationMs, error, sessionId, clientV
|
|
|
28174
28692
|
metaVrTelemetry(args, clientVersion);
|
|
28175
28693
|
}
|
|
28176
28694
|
|
|
28177
|
-
const CLI_VERSION = "1.1.0-rc.
|
|
28695
|
+
const CLI_VERSION = "1.1.0-rc.2";
|
|
28178
28696
|
|
|
28179
28697
|
/**
|
|
28180
28698
|
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
@@ -28796,6 +29314,9 @@ const DIRECT_OPTION_ALIASES = {
|
|
|
28796
29314
|
pattern: { parameter: 'pattern', type: 'string' },
|
|
28797
29315
|
since: { parameter: 'since', type: 'number' },
|
|
28798
29316
|
},
|
|
29317
|
+
browser_set_muted: {
|
|
29318
|
+
muted: { parameter: 'muted', type: 'boolean' },
|
|
29319
|
+
},
|
|
28799
29320
|
ecs_step: {
|
|
28800
29321
|
count: { parameter: 'count', type: 'number' },
|
|
28801
29322
|
frames: { parameter: 'count', type: 'number' },
|
|
@@ -28803,6 +29324,15 @@ const DIRECT_OPTION_ALIASES = {
|
|
|
28803
29324
|
},
|
|
28804
29325
|
};
|
|
28805
29326
|
function parseDirectValue(value, option, type) {
|
|
29327
|
+
if (type === 'boolean') {
|
|
29328
|
+
if (value === true || value === 'true') {
|
|
29329
|
+
return true;
|
|
29330
|
+
}
|
|
29331
|
+
if (value === 'false') {
|
|
29332
|
+
return false;
|
|
29333
|
+
}
|
|
29334
|
+
throw new Error(`--${option} must be true or false`);
|
|
29335
|
+
}
|
|
28806
29336
|
if (typeof value !== 'string') {
|
|
28807
29337
|
throw new Error(`--${option} requires a value`);
|
|
28808
29338
|
}
|
|
@@ -28912,7 +29442,7 @@ async function runRuntimeOperation(operation, options, io, telemetry) {
|
|
|
28912
29442
|
try {
|
|
28913
29443
|
const parsedParams = resolveRuntimeParams(operation.mcpName, options);
|
|
28914
29444
|
command = resolveRuntimeOperationRequest(operation, parsedParams);
|
|
28915
|
-
timeoutMs = parseIntegerOption(options.timeout, '--timeout', getDefaultRuntimeCommandTimeoutMs(operation.wsMethod));
|
|
29445
|
+
timeoutMs = parseIntegerOption(options.timeout, '--timeout', getDefaultRuntimeCommandTimeoutMs(operation.wsMethod, command.params));
|
|
28916
29446
|
}
|
|
28917
29447
|
catch (error) {
|
|
28918
29448
|
telemetry.fail('invalid_input');
|
|
@@ -29034,6 +29564,14 @@ function formatSchemaPropertyLines(propertyPath, schema, required, indent = 2) {
|
|
|
29034
29564
|
}
|
|
29035
29565
|
return lines;
|
|
29036
29566
|
}
|
|
29567
|
+
// Where the schema alone would make an empty or placeholder example.
|
|
29568
|
+
const RUNTIME_HELP_EXAMPLES = {
|
|
29569
|
+
xr_run_sequence: {
|
|
29570
|
+
device: 'hand-right',
|
|
29571
|
+
steps: [{ label: 'press', target: 'counter-button', press: {} }],
|
|
29572
|
+
},
|
|
29573
|
+
xr_set_hand_pose: { device: 'hand-right', poseId: 'point' },
|
|
29574
|
+
};
|
|
29037
29575
|
function schemaExample(schema) {
|
|
29038
29576
|
if (schema.enum?.length) {
|
|
29039
29577
|
return schema.enum[0];
|
|
@@ -29091,13 +29629,17 @@ function buildRuntimeCommandHelp(domain, action) {
|
|
|
29091
29629
|
lines.push(...formatSchemaPropertyLines(name, schema, required.has(name)));
|
|
29092
29630
|
}
|
|
29093
29631
|
}
|
|
29094
|
-
lines.push('', 'Example:', ` iwsdk ${domain} ${action} --input-json '${JSON.stringify(
|
|
29632
|
+
lines.push('', 'Example:', ` iwsdk ${domain} ${action} --input-json '${JSON.stringify(RUNTIME_HELP_EXAMPLES[operation.mcpName] ??
|
|
29633
|
+
schemaExample(operation.inputSchema))}'`, '', 'Options:', ' --input-json <json>', ' --timeout <ms>', ' --raw');
|
|
29095
29634
|
if (writesScreenshot) {
|
|
29096
29635
|
lines.push(' --output-file <path> Write the PNG to this path and return screenshotPath; takes precedence over --raw');
|
|
29097
29636
|
}
|
|
29098
29637
|
if (operation.mcpName === 'ecs_step') {
|
|
29099
29638
|
lines.push(' --frames <count> Alias for input count', ' --count <count> Alias for input count', ' --delta <seconds> Alias for input delta');
|
|
29100
29639
|
}
|
|
29640
|
+
if (operation.mcpName === 'browser_set_muted') {
|
|
29641
|
+
lines.push(' --muted <true|false> Alias for input muted');
|
|
29642
|
+
}
|
|
29101
29643
|
if (operation.mcpName === 'browser_get_console_logs') {
|
|
29102
29644
|
lines.push(' --level <levels> Comma-separated log-level alias', ' --count <count> Maximum log-count alias', ' --pattern <regex> Message-pattern alias', ' --since <timestamp> Timestamp alias');
|
|
29103
29645
|
}
|