@ateam-ai-mobile/plugin-runtime 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-PKDQLNBL.mjs → chunk-RXR6M623.mjs} +269 -57
- package/dist/chunk-RXR6M623.mjs.map +1 -0
- package/dist/index.d.mts +356 -36
- package/dist/index.d.ts +356 -36
- package/dist/index.js +288 -161
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +13 -101
- package/dist/index.mjs.map +1 -1
- package/dist/{loader-5LUVXUY2.mjs → loader-UUOPZQSI.mjs} +2 -2
- package/package.json +2 -2
- package/src/bundles/bundleManager.ts +161 -57
- package/src/bundles/storage.ts +19 -0
- package/src/deviceStateHost.ts +25 -0
- package/src/hooks/usePlugin.ts +13 -3
- package/src/host.ts +2 -0
- package/src/index.ts +11 -6
- package/src/pluginRuntime.ts +22 -11
- package/src/sdk/device-state.ts +100 -10
- package/src/sdk/index.ts +7 -0
- package/src/sdk/registration.ts +2 -1
- package/src/sdk/types.ts +149 -5
- package/src/sdk/voice.ts +38 -0
- package/src/tools/loader.ts +24 -6
- package/src/tools/registry.ts +12 -5
- package/src/voiceHost.ts +2 -0
- package/dist/chunk-PKDQLNBL.mjs.map +0 -1
- /package/dist/{loader-5LUVXUY2.mjs.map → loader-UUOPZQSI.mjs.map} +0 -0
package/src/sdk/device-state.ts
CHANGED
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
* VERB names: takePhoto, getCurrent, scanBarcode,
|
|
14
14
|
* readOnce, authenticate, pickImage.
|
|
15
15
|
* Returns a BARE value. THROWS when it cannot.
|
|
16
|
-
* Declared in the manifest as `
|
|
16
|
+
* Declared in the manifest as `capabilities: { camera: true }`
|
|
17
|
+
* (the legacy `native: {}` field is still honoured).
|
|
17
18
|
* Lives in '@adas/plugin-sdk'.
|
|
18
19
|
*
|
|
19
20
|
* props.deviceState.* SEMANTIC. "What is true right now, and how well do
|
|
@@ -163,17 +164,19 @@ export type RestrictionState =
|
|
|
163
164
|
* never raw sensor streams." If you find yourself wanting an event per frame
|
|
164
165
|
* or per GPS fix, you are in the wrong layer.
|
|
165
166
|
*/
|
|
166
|
-
export interface StateEvent<TName extends string = string> {
|
|
167
|
+
export interface StateEvent<TName extends string = string, TPayload = unknown> {
|
|
167
168
|
domain: DeviceBridgeDomain;
|
|
168
169
|
name: TName;
|
|
169
170
|
/** Epoch ms. */
|
|
170
171
|
timestamp: number;
|
|
171
172
|
/**
|
|
172
173
|
* OPEN: per-event payload schemas are not specified anywhere in the three
|
|
173
|
-
* design docs.
|
|
174
|
-
* narrow it rather than read a field that may not exist.
|
|
174
|
+
* design docs. Defaults to `unknown` rather than `any` so a consumer is forced
|
|
175
|
+
* to narrow it rather than read a field that may not exist. A domain that
|
|
176
|
+
* decides its schemas (spatial — see `SpatialStateEvent`) passes `TPayload`,
|
|
177
|
+
* which is the freeze's own second parameter, dropped here until one did.
|
|
175
178
|
*/
|
|
176
|
-
payload?:
|
|
179
|
+
payload?: TPayload;
|
|
177
180
|
}
|
|
178
181
|
|
|
179
182
|
export interface Subscription {
|
|
@@ -192,7 +195,9 @@ export type StateListener<TName extends string = string> = (
|
|
|
192
195
|
* connectivity, capability, interaction and environment only. Rather than mint
|
|
193
196
|
* names those docs deliberately did not mint, their events are typed against
|
|
194
197
|
* the naming RULE. A wrong-shaped name is a compile error; the specific
|
|
195
|
-
* vocabulary stays open, as the docs left it
|
|
198
|
+
* vocabulary stays open, as the docs left it — except where a DECISION closes
|
|
199
|
+
* it (spatial, see `SpatialEvent`), and that list is checked against the rule
|
|
200
|
+
* through `AssertConforming`.
|
|
196
201
|
*/
|
|
197
202
|
export type ConformingEventName =
|
|
198
203
|
| `${string}Changed`
|
|
@@ -202,6 +207,13 @@ export type ConformingEventName =
|
|
|
202
207
|
| `${string}Unavailable`
|
|
203
208
|
| `${string}StateStale`;
|
|
204
209
|
|
|
210
|
+
/**
|
|
211
|
+
* A closed list of names, checked against the naming rule at compile time.
|
|
212
|
+
* Wrap a decided vocabulary in this and a name like `turnCompleted` stops
|
|
213
|
+
* compiling instead of reaching a plugin.
|
|
214
|
+
*/
|
|
215
|
+
export type AssertConforming<T extends ConformingEventName> = T;
|
|
216
|
+
|
|
205
217
|
// ── Errors ────────────────────────────────────────────────────────────────
|
|
206
218
|
|
|
207
219
|
/** FROZEN (API_FREEZE §12). */
|
|
@@ -373,12 +385,88 @@ export interface SpatialSnapshot {
|
|
|
373
385
|
quality: StateQuality;
|
|
374
386
|
}
|
|
375
387
|
|
|
388
|
+
/**
|
|
389
|
+
* DECISION (not frozen) — the spatial event vocabulary. Six names, all ending
|
|
390
|
+
* in a suffix the naming rule allows, which `AssertConforming` enforces.
|
|
391
|
+
*
|
|
392
|
+
* Drawn from the architecture doc's spatial candidates (meaningful movement,
|
|
393
|
+
* meaningful turn, location quality degraded/recovered, state became stale)
|
|
394
|
+
* and named after the SNAPSHOT fields they report, so an event and a read use
|
|
395
|
+
* one vocabulary: `lastMeaningfulTurnChanged` is `changes.lastMeaningfulTurnDeg`
|
|
396
|
+
* changing. The names an earlier note floated (`turnCompleted`, `arrived`,
|
|
397
|
+
* `headingLost`) are rejected — the first two fail the rule, and "arrived" is a
|
|
398
|
+
* destination, which the vocabulary's Governance puts above Device Bridge.
|
|
399
|
+
*/
|
|
400
|
+
export type SpatialEvent = AssertConforming<
|
|
401
|
+
| 'movementStarted'
|
|
402
|
+
| 'movementStopped'
|
|
403
|
+
| 'lastMeaningfulTurnChanged'
|
|
404
|
+
| 'positionConfidenceChanged'
|
|
405
|
+
| 'positionChanged'
|
|
406
|
+
| 'spatialStateStale'
|
|
407
|
+
>;
|
|
408
|
+
|
|
409
|
+
/** A spatial event: the frozen envelope, with its payload always present. */
|
|
410
|
+
type SpatialEventOf<TName extends SpatialEvent, TPayload> =
|
|
411
|
+
StateEvent<TName, TPayload> & { domain: 'spatial'; payload: TPayload };
|
|
412
|
+
|
|
413
|
+
/**
|
|
414
|
+
* DECISION — what each spatial event carries. Only the facts of the
|
|
415
|
+
* transition, in the snapshot's own field names; everything else is one
|
|
416
|
+
* `getSnapshot()` away, which keeps two shapes of the same fact from drifting.
|
|
417
|
+
*
|
|
418
|
+
* `timestamp` on the envelope is when the transition was RECOGNISED. Where a
|
|
419
|
+
* hold delays that on purpose, `since` carries the true onset.
|
|
420
|
+
*/
|
|
421
|
+
export type SpatialStateEvent =
|
|
422
|
+
/** `movement.state` became `moving` and held (navigation: 3 s). */
|
|
423
|
+
| SpatialEventOf<'movementStarted', {
|
|
424
|
+
previous: 'stationary' | 'unknown';
|
|
425
|
+
speedMps: number | null;
|
|
426
|
+
bearingDeg: number | null;
|
|
427
|
+
since: number;
|
|
428
|
+
}>
|
|
429
|
+
/** `moving` → `stationary`, held (navigation: 5 s). Never fired on moving →
|
|
430
|
+
* `unknown`: losing the speed is not evidence of stopping. */
|
|
431
|
+
| SpatialEventOf<'movementStopped', { previous: 'moving'; since: number }>
|
|
432
|
+
/** A new completed turn. Signed: positive = clockwise = right. */
|
|
433
|
+
| SpatialEventOf<'lastMeaningfulTurnChanged', { lastMeaningfulTurnDeg: number; timestamp: number }>
|
|
434
|
+
/** The GPS fix band (high ≤10 m, medium ≤50 m, else low) changed and held.
|
|
435
|
+
* The fix alone — a compass calibration change is not a position event. */
|
|
436
|
+
| SpatialEventOf<'positionConfidenceChanged', {
|
|
437
|
+
level: ConfidenceLevel;
|
|
438
|
+
previous: ConfidenceLevel;
|
|
439
|
+
accuracyM: number | null;
|
|
440
|
+
direction: 'dropped' | 'recovered';
|
|
441
|
+
}>
|
|
442
|
+
/** Moved at least max(profile floor, 2 × accuracyM) from the last reported
|
|
443
|
+
* position, so GPS jitter never fires it. The session's first fix fires
|
|
444
|
+
* with `previous: null`. */
|
|
445
|
+
| SpatialEventOf<'positionChanged', {
|
|
446
|
+
position: SpatialPoint;
|
|
447
|
+
previous: SpatialPoint | null;
|
|
448
|
+
distanceM: number | null;
|
|
449
|
+
}>
|
|
450
|
+
/** `quality.freshness.state` became `stale`. Once; re-armed by a fresh read. */
|
|
451
|
+
| SpatialEventOf<'spatialStateStale', { observedAt: number | null; ageMs: number | null }>;
|
|
452
|
+
|
|
376
453
|
export interface SpatialDomain {
|
|
377
454
|
getSnapshot(): Promise<SpatialSnapshot>;
|
|
378
455
|
start(options: StartOptions<SpatialIntent>): Promise<StateSession>;
|
|
379
456
|
stop(sessionId?: string): Promise<void>;
|
|
380
|
-
/**
|
|
381
|
-
|
|
457
|
+
/**
|
|
458
|
+
* DECISION: the six `SpatialEvent`s, typed per name — narrow on `e.name` and
|
|
459
|
+
* `e.payload` is that event's shape.
|
|
460
|
+
*
|
|
461
|
+
* Events flow only while a spatial session is open; subscribing opens no
|
|
462
|
+
* sensor. They are derived from the same snapshot `getSnapshot()` returns: a
|
|
463
|
+
* listener that re-reads inside the callback sees the snapshot its event came
|
|
464
|
+
* from. A state change must hold, on at least two fixes, before it fires.
|
|
465
|
+
*
|
|
466
|
+
* From a device tool's `ctx.deviceState` this refuses (RESTRICTED): a call
|
|
467
|
+
* has no lifetime to own the listener.
|
|
468
|
+
*/
|
|
469
|
+
subscribe(listener: (event: SpatialStateEvent) => void): Subscription;
|
|
382
470
|
}
|
|
383
471
|
|
|
384
472
|
// ═════════════════════════════════════════════════════════════════════════
|
|
@@ -1213,8 +1301,10 @@ export interface DeviceToolContextV2Extension {
|
|
|
1213
1301
|
* - Multi-consumer session ownership / reference counting.
|
|
1214
1302
|
* - MediaHandle → bytes resolution, and expiry behaviour.
|
|
1215
1303
|
* - Whether `visual.capture()` may use a different intent from the session.
|
|
1216
|
-
* - Event vocabularies for visual, activity,
|
|
1217
|
-
*
|
|
1304
|
+
* - Event vocabularies for visual, activity, deviceContext. (Spatial's is a
|
|
1305
|
+
* DECISION, recorded at `SpatialEvent`, not a reading of the docs.)
|
|
1306
|
+
* - Per-event payload schemas for every domain but spatial
|
|
1307
|
+
* (`SpatialStateEvent`, likewise a DECISION).
|
|
1218
1308
|
* - Background/foreground semantics for activity and connectivity (specified
|
|
1219
1309
|
* only for visual).
|
|
1220
1310
|
* - Numeric thresholds for fresh/aging/stale, and for "materially changed".
|
package/src/sdk/index.ts
CHANGED
|
@@ -35,6 +35,12 @@ export type {
|
|
|
35
35
|
VisionLabel,
|
|
36
36
|
VisionObject,
|
|
37
37
|
VisionFrameQuality,
|
|
38
|
+
MapsAPI,
|
|
39
|
+
MapsCoordinate,
|
|
40
|
+
MapsAddress,
|
|
41
|
+
MapsPlace,
|
|
42
|
+
MapsRoute,
|
|
43
|
+
MapsRouteStep,
|
|
38
44
|
BiometricsAPI,
|
|
39
45
|
BiometricsAPIv2,
|
|
40
46
|
NativeCapabilities,
|
|
@@ -57,6 +63,7 @@ export type {
|
|
|
57
63
|
VoiceSnapshot,
|
|
58
64
|
VoiceTranscriptEntry,
|
|
59
65
|
VoiceStartOptions,
|
|
66
|
+
VoicePersona,
|
|
60
67
|
VoiceEvent,
|
|
61
68
|
VoiceSubscription,
|
|
62
69
|
VoiceCaller,
|
package/src/sdk/registration.ts
CHANGED
|
@@ -54,7 +54,8 @@ export function registerTools(
|
|
|
54
54
|
id: string,
|
|
55
55
|
config: {
|
|
56
56
|
capabilities?: NativeCapabilities;
|
|
57
|
-
/** L3 DeviceState domains the tools
|
|
57
|
+
/** L3 DeviceState domains the tools read via ctx.deviceState (e.g. ['visual']).
|
|
58
|
+
* For a remote bundle the MANIFEST grants them — see RegisteredToolsPlugin. */
|
|
58
59
|
stateDomains?: DeviceBridgeDomain[];
|
|
59
60
|
tools: DeviceToolDefinition[];
|
|
60
61
|
},
|
package/src/sdk/types.ts
CHANGED
|
@@ -352,6 +352,139 @@ export interface VisionAPI {
|
|
|
352
352
|
assessQuality(frameId: string): Promise<VisionFrameQuality>;
|
|
353
353
|
}
|
|
354
354
|
|
|
355
|
+
// ── Maps — where is X, what is here, how do I walk there ──
|
|
356
|
+
//
|
|
357
|
+
// One-shot verbs that THROW, like vision — not a state domain. Following a
|
|
358
|
+
// route over time (progress, off-route, "turn left in 20 m") is the reasoning
|
|
359
|
+
// loop's job over deviceState.spatial; this layer answers each question once.
|
|
360
|
+
//
|
|
361
|
+
// NOT ON-DEVICE — the opposite of vision, and the thing not to get wrong when
|
|
362
|
+
// describing a feature built on it. Every call sends the query and the
|
|
363
|
+
// coordinates you pass to a map service over the network: Apple's (MapKit /
|
|
364
|
+
// CLGeocoder) on iOS, the platform geocoder on Android. So the `maps` flag
|
|
365
|
+
// means exactly one thing: "this plugin may send place queries to Apple Maps".
|
|
366
|
+
//
|
|
367
|
+
// IT NEVER READS THE USER'S LOCATION. Every coordinate is the caller's — take a
|
|
368
|
+
// fix from `location.getCurrent` or `deviceState.spatial` and pass it in. That
|
|
369
|
+
// keeps `maps` from becoming a second, differently-gated door to where the
|
|
370
|
+
// user is.
|
|
371
|
+
//
|
|
372
|
+
// FOUND NOTHING IS AN ANSWER; COULD NOT LOOK THROWS. `[]` / `null` mean the
|
|
373
|
+
// service looked and there is nothing. Anything else THROWS, with a code:
|
|
374
|
+
// UNAVAILABLE this app build has no AdaMaps module (needs a native rebuild),
|
|
375
|
+
// or the platform has none — search/route are iOS only (MapKit)
|
|
376
|
+
// and refuse by name on Android. Also the service's own failures.
|
|
377
|
+
// OFFLINE the map service could not be reached.
|
|
378
|
+
// THROTTLED Apple is rate-limiting this app. A burst of queries is what
|
|
379
|
+
// trips it — search once per intent, not once per frame.
|
|
380
|
+
// And one refusal that is not a failure to look:
|
|
381
|
+
// INVALID_ARGUMENT the call itself is malformed — an empty query, a
|
|
382
|
+
// coordinate that is not { latitude, longitude } finite numbers in
|
|
383
|
+
// range (`{ lat, lng }` refuses), a limit or radius out of range.
|
|
384
|
+
// Nothing was sent; fix the call rather than retrying it.
|
|
385
|
+
|
|
386
|
+
/** The same `{ latitude, longitude }` `location.getCurrent` returns, so a fix
|
|
387
|
+
* passes straight in. */
|
|
388
|
+
export interface MapsCoordinate {
|
|
389
|
+
latitude: number;
|
|
390
|
+
longitude: number;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/** A postal address, field by field. `null` = the map data has no value — never
|
|
394
|
+
* an empty string, which would read as a street with no name. */
|
|
395
|
+
export interface MapsAddress {
|
|
396
|
+
/** The placemark's own name — a building, or often number + street. */
|
|
397
|
+
name: string | null;
|
|
398
|
+
street: string | null;
|
|
399
|
+
streetNumber: string | null;
|
|
400
|
+
/** Neighbourhood / sub-locality. */
|
|
401
|
+
district: string | null;
|
|
402
|
+
city: string | null;
|
|
403
|
+
/** State / province / administrative area. */
|
|
404
|
+
region: string | null;
|
|
405
|
+
postalCode: string | null;
|
|
406
|
+
country: string | null;
|
|
407
|
+
/** ISO 3166-1 alpha-2, e.g. "GB". */
|
|
408
|
+
isoCountryCode: string | null;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
export interface MapsPlace {
|
|
412
|
+
/** What the map calls it ("Boots", "10 Downing St"). Null when unnamed. */
|
|
413
|
+
name: string | null;
|
|
414
|
+
coordinate: MapsCoordinate;
|
|
415
|
+
address: MapsAddress;
|
|
416
|
+
/** Apple's point-of-interest category as a vocabulary term ("Pharmacy",
|
|
417
|
+
* "Cafe", "PublicTransport") — NOT free text. Null for a plain address, or
|
|
418
|
+
* when the service gives none. */
|
|
419
|
+
category: string | null;
|
|
420
|
+
/** Straight-line metres from `opts.near`, when you passed one — NOT walking
|
|
421
|
+
* distance (ask `route` for that). Null without `near`. */
|
|
422
|
+
distanceM: number | null;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
export interface MapsRouteStep {
|
|
426
|
+
/** Apple's instruction ("Turn right onto Baker St"). The FIRST step is usually
|
|
427
|
+
* an origin step with an empty instruction — kept, not dropped, so step
|
|
428
|
+
* indexes line up with Apple's. */
|
|
429
|
+
instructions: string;
|
|
430
|
+
distanceM: number;
|
|
431
|
+
/** An advisory Apple attaches to this step. Null when there is none. */
|
|
432
|
+
notice: string | null;
|
|
433
|
+
polyline: MapsCoordinate[];
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
export interface MapsRoute {
|
|
437
|
+
mode: 'walking';
|
|
438
|
+
distanceM: number;
|
|
439
|
+
/** Apple's estimate for the whole route. */
|
|
440
|
+
expectedTravelTimeMs: number;
|
|
441
|
+
steps: MapsRouteStep[];
|
|
442
|
+
/** The whole route, start to end. */
|
|
443
|
+
polyline: MapsCoordinate[];
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
export interface MapsAPI {
|
|
447
|
+
/** Can this build search and route? True when the AdaMaps module is in the
|
|
448
|
+
* binary (iOS). False on Android and on an older build — `reverseGeocode`
|
|
449
|
+
* does not need the module and may still work there. */
|
|
450
|
+
isSupported(): Promise<boolean>;
|
|
451
|
+
/**
|
|
452
|
+
* Find places by free text — "pharmacy", "10 Downing Street".
|
|
453
|
+
*
|
|
454
|
+
* `near` centres the search on a coordinate YOU supply, within `radiusM`
|
|
455
|
+
* (default 3000, above 0 and at most 100 000). On iOS 18+ results are
|
|
456
|
+
* confined to that area; before 18 it is a strong hint and a farther result
|
|
457
|
+
* can come back — each result's `distanceM` says how far. At most `limit`
|
|
458
|
+
* results (default 8, from 1; more than 25 is capped at 25).
|
|
459
|
+
*
|
|
460
|
+
* Network, Apple's servers. `[]` = Apple looked and found nothing. iOS only:
|
|
461
|
+
* throws UNAVAILABLE on Android or a build without AdaMaps.
|
|
462
|
+
*/
|
|
463
|
+
search(query: string, opts?: { near?: MapsCoordinate; radiusM?: number; limit?: number }): Promise<MapsPlace[]>;
|
|
464
|
+
/**
|
|
465
|
+
* What is at a coordinate — its address and name.
|
|
466
|
+
*
|
|
467
|
+
* iOS AND Android (the platform geocoder); needs no AdaMaps module. The
|
|
468
|
+
* result's `coordinate` is the point you asked about, and its
|
|
469
|
+
* `category` / `distanceM` are null. `null` = nothing there.
|
|
470
|
+
*
|
|
471
|
+
* Network, not on-device. iOS reports a rate limit and a lost connection as
|
|
472
|
+
* the SAME error, so OFFLINE here can also mean "too many requests". Android's
|
|
473
|
+
* geocoder refuses without location permission (UNAVAILABLE) — that is the
|
|
474
|
+
* platform's rule, not a location read.
|
|
475
|
+
*/
|
|
476
|
+
reverseGeocode(at: MapsCoordinate): Promise<MapsPlace | null>;
|
|
477
|
+
/**
|
|
478
|
+
* A walking route between two coordinates YOU supply.
|
|
479
|
+
*
|
|
480
|
+
* `null` = Apple found no route — never a fake 0 m route, which would read as
|
|
481
|
+
* "you have arrived". Walking is the only mode; any other refuses by name.
|
|
482
|
+
* Network, Apple's servers. iOS only: throws UNAVAILABLE on Android or a build
|
|
483
|
+
* without AdaMaps.
|
|
484
|
+
*/
|
|
485
|
+
route(from: MapsCoordinate, to: MapsCoordinate, opts?: { mode?: 'walking' }): Promise<MapsRoute | null>;
|
|
486
|
+
}
|
|
487
|
+
|
|
355
488
|
export interface BluetoothAPI {
|
|
356
489
|
/** Is BLE hardware present and permitted? */
|
|
357
490
|
isSupported(): Promise<boolean>;
|
|
@@ -420,6 +553,9 @@ export interface PluginNative {
|
|
|
420
553
|
bluetooth: BluetoothAPI;
|
|
421
554
|
/** SHAPE ONLY in this build — see VisionAPI. On-device image understanding. */
|
|
422
555
|
vision: VisionAPI;
|
|
556
|
+
/** Place search, reverse geocoding, walking routes — see MapsAPI. Sends
|
|
557
|
+
* queries to Apple's map servers; NOT on-device. */
|
|
558
|
+
maps: MapsAPI;
|
|
423
559
|
}
|
|
424
560
|
|
|
425
561
|
// ── Native capability flags (manifest declaration) ──────────
|
|
@@ -438,6 +574,9 @@ export interface NativeCapabilities {
|
|
|
438
574
|
/** On-device image understanding (OCR, labels, frame quality). Refuses by
|
|
439
575
|
* name until the native module lands — see VisionAPI. */
|
|
440
576
|
vision?: boolean;
|
|
577
|
+
/** Place search, reverse geocoding, walking routes. Means "may send place
|
|
578
|
+
* queries to Apple Maps" — network, not on-device. See MapsAPI. */
|
|
579
|
+
maps?: boolean;
|
|
441
580
|
/** The live voice SESSION (props.voice). A privacy surface — a declaring
|
|
442
581
|
* plugin can read the conversation transcript — so it is gated like any
|
|
443
582
|
* other capability, and coupled to the platform's unsuppressable mic
|
|
@@ -570,8 +709,9 @@ export interface PluginManifest {
|
|
|
570
709
|
channels?: string[] | { events: string[]; actions: string[] };
|
|
571
710
|
capabilities?: Record<string, unknown>;
|
|
572
711
|
/** L3 DeviceState domains this plugin reads (props.deviceState), as Core passes
|
|
573
|
-
* them through from the
|
|
574
|
-
* bundle's own `stateDomains
|
|
712
|
+
* them through from the solution's declaration. Absent → NO domains (fail
|
|
713
|
+
* closed): the loaded bundle's own `stateDomains` is never consulted when a
|
|
714
|
+
* manifest exists. Read via resolvePluginRuntimeProps. */
|
|
575
715
|
stateDomains?: DeviceBridgeDomain[];
|
|
576
716
|
_source?: string;
|
|
577
717
|
_connectorId?: string;
|
|
@@ -616,6 +756,9 @@ export interface DeviceToolContext {
|
|
|
616
756
|
* the same gating a UI plugin's `props.deviceState` gets. This is what lets an
|
|
617
757
|
* EXTERNAL tool (e.g. Ada Guide's `guide.observe`) read `ctx.deviceState.visual`
|
|
618
758
|
* over the public surface, with no SDK-internal access.
|
|
759
|
+
*
|
|
760
|
+
* `subscribe` refuses (RESTRICTED) on every domain: a call has no lifetime to
|
|
761
|
+
* own a subscription, and the engines outlive it. Read instead.
|
|
619
762
|
*/
|
|
620
763
|
deviceState: DeviceState;
|
|
621
764
|
/** Cooperative cancellation: caller cancelled, or the session went away. */
|
|
@@ -639,9 +782,10 @@ export interface RegisteredToolsPlugin {
|
|
|
639
782
|
type: 'tools';
|
|
640
783
|
capabilities?: NativeCapabilities;
|
|
641
784
|
/**
|
|
642
|
-
* L3 DeviceState domains this plugin's tools
|
|
643
|
-
*
|
|
644
|
-
*
|
|
785
|
+
* L3 DeviceState domains this plugin's tools read via `ctx.deviceState` (e.g.
|
|
786
|
+
* `['visual']`). For a remote bundle this is the author's statement, not a
|
|
787
|
+
* grant: the tools get the manifest's `stateDomains` (resolvePluginGrants),
|
|
788
|
+
* and a domain declared here but not there is denied with a warning.
|
|
645
789
|
*/
|
|
646
790
|
stateDomains?: DeviceBridgeDomain[];
|
|
647
791
|
tools: DeviceToolDefinition[];
|
package/src/sdk/voice.ts
CHANGED
|
@@ -52,12 +52,50 @@ export interface VoiceSnapshot {
|
|
|
52
52
|
/** The last error, if `state === 'error'`; also carries a refusal reason when
|
|
53
53
|
* this surface was handed to a plugin that did not declare `voice`. */
|
|
54
54
|
lastError: string | null;
|
|
55
|
+
/** The most recent LOCAL device-tool call the model made through the session
|
|
56
|
+
* (name + epoch ms), or null if none yet. Diagnostic: proves whether the
|
|
57
|
+
* model's call_device_tool actually reached the phone vs was never made. */
|
|
58
|
+
lastDeviceTool: { name: string; at: number } | null;
|
|
59
|
+
/** The sessionId the Core allowset is bound to. Diagnostic: authorization
|
|
60
|
+
* requires it to equal `sessionId`; a mismatch is why tools refuse. */
|
|
61
|
+
authorizedSessionId: string | null;
|
|
55
62
|
}
|
|
56
63
|
|
|
57
64
|
export interface VoiceStartOptions {
|
|
58
65
|
/** Hand the mic to a specific skill on connect (voice /session → takeMic),
|
|
59
66
|
* exactly as `sys.focusUiPlugin("platform:voice", { skillSlug })` does. */
|
|
60
67
|
skillSlug?: string;
|
|
68
|
+
/**
|
|
69
|
+
* A PER-CALL persona override, same shape as the authored
|
|
70
|
+
* `solution.voice.persona` ({ name, style }). Declared on the voice
|
|
71
|
+
* session-create call (POST /session `persona`); the server sanitizes it
|
|
72
|
+
* (name/style only, length-capped) and layers it OVER the solution's authored
|
|
73
|
+
* persona for THIS call only — prompt-layer, so it can never touch caller
|
|
74
|
+
* verification or tool allow-lists. Set once at session start and constant
|
|
75
|
+
* for the call, so it adds no per-turn prompt pushes.
|
|
76
|
+
*
|
|
77
|
+
* Use it when a plugin needs its own voice persona without owning a skill
|
|
78
|
+
* (e.g. a guide HUD). A solution-wide persona belongs in solution.voice.
|
|
79
|
+
*/
|
|
80
|
+
persona?: VoicePersona;
|
|
81
|
+
/**
|
|
82
|
+
* `'none'` = a CLEAN call: the voice gets none of the workspace's skills — no
|
|
83
|
+
* skill manifest, no skill tools, no platform welcome line, no mid-call
|
|
84
|
+
* skill-state prompt refresh — only the voice basics, this call's persona and
|
|
85
|
+
* the CALLING plugin's own device tools (other solutions' tools registered on
|
|
86
|
+
* the same phone are neither offered nor run). For a plugin that owns its conversation (e.g.
|
|
87
|
+
* a guide HUD). Reduce-only: it can remove what the voice gets, never add.
|
|
88
|
+
* Absent = the workspace's skills, as today. Not combinable with `skillSlug`.
|
|
89
|
+
*/
|
|
90
|
+
skills?: 'none';
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** The authored persona shape (solution.voice.persona). */
|
|
94
|
+
export interface VoicePersona {
|
|
95
|
+
/** The name the voice uses for itself ("You are <name> …"). */
|
|
96
|
+
name?: string;
|
|
97
|
+
/** Free-text style/behaviour guidance rendered under the identity line. */
|
|
98
|
+
style?: string;
|
|
61
99
|
}
|
|
62
100
|
|
|
63
101
|
/**
|
package/src/tools/loader.ts
CHANGED
|
@@ -14,6 +14,7 @@ import { listPlugins, getPlugin, getNativeBundleUrl, getNativeBundleHash } from
|
|
|
14
14
|
import { bundleManager } from '../bundles/bundleManager';
|
|
15
15
|
import { listDeviceTools } from './registry';
|
|
16
16
|
import { getBundleStorage, selectOwnedCacheEntries } from '../bundles/storage';
|
|
17
|
+
import { resolvePluginGrants } from '../pluginRuntime';
|
|
17
18
|
import { getTenant } from '@ateam-ai-mobile/core';
|
|
18
19
|
|
|
19
20
|
let inflight: Promise<string[]> | null = null;
|
|
@@ -54,9 +55,9 @@ export async function ensureDeviceToolsLoaded({ force = false } = {}): Promise<s
|
|
|
54
55
|
// a fact about a server process, not about this phone — the bundle we
|
|
55
56
|
// already downloaded still has the tools in it.
|
|
56
57
|
//
|
|
57
|
-
// So fall back to what is on disk
|
|
58
|
-
//
|
|
59
|
-
//
|
|
58
|
+
// So fall back to what is on disk: re-evaluate this tenant's cached
|
|
59
|
+
// bundles under the trusted declaration recorded when a manifest last
|
|
60
|
+
// loaded them (see loadFromDiskCache).
|
|
60
61
|
const recovered = await loadFromDiskCache();
|
|
61
62
|
if (recovered.length > 0) {
|
|
62
63
|
console.log(`[deviceTools] catalog listed none — recovered from cache: ${recovered.join(', ')}`);
|
|
@@ -75,11 +76,14 @@ export async function ensureDeviceToolsLoaded({ force = false } = {}): Promise<s
|
|
|
75
76
|
return;
|
|
76
77
|
}
|
|
77
78
|
try {
|
|
79
|
+
// The manifest is the authority on what the tools may touch — the
|
|
80
|
+
// same grants its rendered surface gets, never the bundle's claim.
|
|
78
81
|
await bundleManager.loadTools(
|
|
79
82
|
m.id,
|
|
80
83
|
bundleUrl,
|
|
81
84
|
m.version || '1.0.0',
|
|
82
85
|
getNativeBundleHash(m) || undefined,
|
|
86
|
+
resolvePluginGrants({ pluginId: m.id, manifest: m }),
|
|
83
87
|
);
|
|
84
88
|
} catch (err: any) {
|
|
85
89
|
console.warn(`[deviceTools] ${m.id} failed to load:`, err?.message || String(err));
|
|
@@ -127,6 +131,14 @@ export async function ensureDeviceToolsLoaded({ force = false } = {}): Promise<s
|
|
|
127
131
|
* An entry with no recorded tenant predates the stamp. Its owner is unknown,
|
|
128
132
|
* which means untrusted — skip it and let the normal catalog path re-download
|
|
129
133
|
* it under a real hash gate.
|
|
134
|
+
*
|
|
135
|
+
* GRANTS COME FROM THE RECORDED MANIFEST, NEVER THE BUNDLE. With no manifest
|
|
136
|
+
* here, the tools register under the `trusted` declaration (pluginId +
|
|
137
|
+
* capabilities + stateDomains) the bundle manager recorded on the entry when a
|
|
138
|
+
* manifest last loaded these bytes. An entry without one — written before the
|
|
139
|
+
* record existed, or never loaded under a manifest — is skipped (fail closed):
|
|
140
|
+
* the only other source is the bundle's own claim, and the catalog path
|
|
141
|
+
* registers it properly once its connector answers.
|
|
130
142
|
*/
|
|
131
143
|
async function loadFromDiskCache(): Promise<string[]> {
|
|
132
144
|
try {
|
|
@@ -145,11 +157,17 @@ async function loadFromDiskCache(): Promise<string[]> {
|
|
|
145
157
|
if (skipped > 0) {
|
|
146
158
|
console.log(`[deviceTools] disk-cache recovery: ${mine.length} for "${tenant}", skipped ${skipped} not owned by it`);
|
|
147
159
|
}
|
|
148
|
-
|
|
160
|
+
const declared = mine.filter((e) => !!e.trusted?.pluginId);
|
|
161
|
+
if (declared.length < mine.length) {
|
|
162
|
+
console.log(
|
|
163
|
+
`[deviceTools] disk-cache recovery: skipped ${mine.length - declared.length} with no recorded manifest grant (fail closed)`,
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
if (declared.length === 0) return [];
|
|
149
167
|
|
|
150
|
-
await Promise.all(
|
|
168
|
+
await Promise.all(declared.map(async (e) => {
|
|
151
169
|
try {
|
|
152
|
-
await bundleManager.loadTools(
|
|
170
|
+
await bundleManager.loadTools(e.trusted!.pluginId, e.bundleUrl, e.version, e.bundleHash, e.trusted!);
|
|
153
171
|
} catch {
|
|
154
172
|
// A cached bundle that no longer loads is not worth failing a call over.
|
|
155
173
|
}
|
package/src/tools/registry.ts
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { createNativeBridge } from '../bridge/native';
|
|
16
16
|
import { getPluginNative, gatePluginNative } from '../host';
|
|
17
|
-
import { gateDeviceState } from '../deviceStateHost';
|
|
17
|
+
import { gateDeviceState, withoutSubscribe } from '../deviceStateHost';
|
|
18
18
|
import type { DeviceToolContext, DeviceToolDefinition, NativeCapabilities } from '../sdk/types';
|
|
19
19
|
import type { DeviceBridgeDomain } from '../sdk/device-state';
|
|
20
20
|
|
|
@@ -62,7 +62,8 @@ export function setDeviceToolContextProvider(
|
|
|
62
62
|
* Default context: the plugin's own bridge + a capability-gated Device Bridge +
|
|
63
63
|
* the L3 DeviceState gated to the domains the plugin declared. All three are the
|
|
64
64
|
* SAME gated objects the plugin gets when it renders — a tool reaches nothing a
|
|
65
|
-
* UI plugin couldn't.
|
|
65
|
+
* UI plugin couldn't. (One thing less: `callDeviceTool` takes `subscribe` away,
|
|
66
|
+
* because a call has no lifetime to own one.)
|
|
66
67
|
*/
|
|
67
68
|
function defaultContext(
|
|
68
69
|
pluginId: string,
|
|
@@ -120,8 +121,11 @@ export function unregisterDeviceTools(pluginId: string): void {
|
|
|
120
121
|
}
|
|
121
122
|
}
|
|
122
123
|
|
|
123
|
-
|
|
124
|
-
|
|
124
|
+
/** Is `name` registered — and, given `pluginId`, registered BY that plugin?
|
|
125
|
+
* (A clean voice call only lets its owner's own tools run.) */
|
|
126
|
+
export function hasDeviceTool(name: string, pluginId?: string): boolean {
|
|
127
|
+
const entry = registry.get(name);
|
|
128
|
+
return !!entry && (pluginId === undefined || entry.pluginId === pluginId);
|
|
125
129
|
}
|
|
126
130
|
|
|
127
131
|
export function listDeviceTools(): DeviceToolSummary[] {
|
|
@@ -152,8 +156,11 @@ export async function callDeviceTool(
|
|
|
152
156
|
|
|
153
157
|
try {
|
|
154
158
|
const build = contextProvider || defaultContext;
|
|
159
|
+
const built = build(entry.pluginId, entry.capabilities, entry.stateDomains);
|
|
155
160
|
const ctx: DeviceToolContext = {
|
|
156
|
-
...
|
|
161
|
+
...built,
|
|
162
|
+
// Whoever built the context, a call cannot own a subscription.
|
|
163
|
+
deviceState: withoutSubscribe(built.deviceState),
|
|
157
164
|
signal: opts.signal,
|
|
158
165
|
};
|
|
159
166
|
|