@oxvo/ai-live-assist 7.5.1 → 7.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cjs/AiLiveAssist.d.ts +2 -1
- package/cjs/AiLiveAssist.js +56 -8
- package/cjs/context.d.ts +2 -0
- package/cjs/context.js +70 -32
- package/cjs/control.d.ts +9 -1
- package/cjs/control.js +81 -3
- package/cjs/liveControlChannel.d.ts +1 -1
- package/cjs/media.d.ts +1 -0
- package/cjs/media.js +3 -0
- package/cjs/ui.js +2 -0
- package/cjs/version.d.ts +1 -1
- package/cjs/version.js +1 -1
- package/lib/AiLiveAssist.d.ts +2 -1
- package/lib/AiLiveAssist.js +56 -8
- package/lib/context.d.ts +2 -0
- package/lib/context.js +70 -32
- package/lib/control.d.ts +9 -1
- package/lib/control.js +81 -3
- package/lib/liveControlChannel.d.ts +1 -1
- package/lib/media.d.ts +1 -0
- package/lib/media.js +3 -0
- package/lib/ui.js +2 -0
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/package.json +1 -1
package/cjs/AiLiveAssist.d.ts
CHANGED
|
@@ -34,7 +34,7 @@ export declare const normalizeAiGuideOpenInput: (input?: AiGuideOpenInput) => Re
|
|
|
34
34
|
export declare const resolveAiGuideProtocolVersion: (bootstrap: Pick<BootstrapConfig, "experienceMode" | "minimumProtocolVersion" | "protocol">) => 1 | 2;
|
|
35
35
|
export default class AiLiveAssist {
|
|
36
36
|
private readonly options;
|
|
37
|
-
readonly version = "7.5.
|
|
37
|
+
readonly version = "7.5.8";
|
|
38
38
|
private readonly host;
|
|
39
39
|
private readonly guideAdmission;
|
|
40
40
|
private readonly standaloneAdmissionPending;
|
|
@@ -627,6 +627,7 @@ export default class AiLiveAssist {
|
|
|
627
627
|
private submitFeedback;
|
|
628
628
|
private retry;
|
|
629
629
|
private scheduleReconnect;
|
|
630
|
+
private canRecoverControlTransport;
|
|
630
631
|
private reconnectPendingHandoffControlInMemory;
|
|
631
632
|
private reconnectMessengerTerminalControl;
|
|
632
633
|
private clearReconnectTimer;
|
package/cjs/AiLiveAssist.js
CHANGED
|
@@ -7945,7 +7945,28 @@ class AiLiveAssist {
|
|
|
7945
7945
|
? this.navigationConsentResetReady
|
|
7946
7946
|
: null;
|
|
7947
7947
|
const resumeRecordAtConnect = this.readResume();
|
|
7948
|
-
|
|
7948
|
+
let recoveryPath = null;
|
|
7949
|
+
const channel = new control_js_1.ControlChannel(result, this.tabId, version_js_1.VERSION, () => this.scheduleReconnect(true), this.readResume()?.clientSequence ?? 0, (sequence) => this.updateResumeSequence(sequence), () => this.handleControlSequenceGap(), {
|
|
7950
|
+
canRecover: () => {
|
|
7951
|
+
const currentPath = path();
|
|
7952
|
+
if (!this.canRecoverControlTransport(channel, result, media) ||
|
|
7953
|
+
(recoveryPath !== null && recoveryPath !== currentPath))
|
|
7954
|
+
return false;
|
|
7955
|
+
recoveryPath = currentPath;
|
|
7956
|
+
return true;
|
|
7957
|
+
},
|
|
7958
|
+
onRecovered: async () => {
|
|
7959
|
+
assertCurrent();
|
|
7960
|
+
const fencingToken = channel.leaseToken;
|
|
7961
|
+
channel.startHeartbeat(this.heartbeatSeconds);
|
|
7962
|
+
await this.replayPreFenceAuthorityEnvelopes(channel, fencingToken, assertCurrent);
|
|
7963
|
+
assertCurrent();
|
|
7964
|
+
await this.context?.captureAndPublish();
|
|
7965
|
+
assertCurrent();
|
|
7966
|
+
this.flushPlaybackState();
|
|
7967
|
+
this.reapplyConservativePreFencePresence(channel, fencingToken);
|
|
7968
|
+
},
|
|
7969
|
+
});
|
|
7949
7970
|
const pauseHelloStateSignal = deferredSignal();
|
|
7950
7971
|
(this.pauseHelloStateSignals ?? (this.pauseHelloStateSignals = new WeakMap())).set(channel, pauseHelloStateSignal);
|
|
7951
7972
|
this.handoffContextPromptReceiptChannel = null;
|
|
@@ -8315,7 +8336,7 @@ class AiLiveAssist {
|
|
|
8315
8336
|
this.context === context;
|
|
8316
8337
|
context = new context_js_1.PageContextCollector({
|
|
8317
8338
|
onSnapshot: (snapshot) => {
|
|
8318
|
-
if (isCurrent() && this.channel?.
|
|
8339
|
+
if (isCurrent() && this.channel?.hasFencedAuthority) {
|
|
8319
8340
|
this.send("context.snapshot", snapshot);
|
|
8320
8341
|
}
|
|
8321
8342
|
},
|
|
@@ -9674,7 +9695,15 @@ class AiLiveAssist {
|
|
|
9674
9695
|
this.activePageAssistanceAuthorityEpoch === null &&
|
|
9675
9696
|
this.activeActionAuthorityEpoch === null;
|
|
9676
9697
|
if (!exactApplied || !localAuthorityInactive) {
|
|
9677
|
-
|
|
9698
|
+
// A resumed document still owes the original action's terminal result.
|
|
9699
|
+
// Preserve only its grantless outbox, never browser-control authority.
|
|
9700
|
+
// Delivery remains fenced by the server's exact already_started receipt.
|
|
9701
|
+
const preserveNavigationResult = source === "navigation_resume" &&
|
|
9702
|
+
this.awaitingNavigationConsentReset &&
|
|
9703
|
+
structurallyValid &&
|
|
9704
|
+
localAuthorityInactive &&
|
|
9705
|
+
this.pendingNavigationActionResult?.grant === null;
|
|
9706
|
+
this.invalidatePageAssistanceAuthority(false, preserveNavigationResult);
|
|
9678
9707
|
}
|
|
9679
9708
|
if (!structurallyValid ||
|
|
9680
9709
|
identity.authorityEpoch < this.minimumPageAssistanceAuthorityEpoch) {
|
|
@@ -11997,15 +12026,19 @@ class AiLiveAssist {
|
|
|
11997
12026
|
// intentionally result-less until navigation settles, preserving the
|
|
11998
12027
|
// cross-document recovery contract if this click unloads the page.
|
|
11999
12028
|
pendingExecution.result = deliveryResult;
|
|
12029
|
+
let navigationReleased = false;
|
|
12000
12030
|
try {
|
|
12001
|
-
|
|
12002
|
-
return deliveryResult;
|
|
12003
|
-
}
|
|
12004
|
-
deliveryResult = this.markDeferredNavigationUnconfirmed(pendingExecution, context.currentRevision);
|
|
12031
|
+
navigationReleased = await this.navigateTo(deferredNavigation);
|
|
12005
12032
|
}
|
|
12006
12033
|
catch {
|
|
12007
|
-
|
|
12034
|
+
// A rejected router and an outgoing document both settle here.
|
|
12008
12035
|
}
|
|
12036
|
+
// Pagehide stops the old collector and settles its pending router
|
|
12037
|
+
// callback. It must not replace the destination document's grantless
|
|
12038
|
+
// recovery record with an unconfirmed result from the outgoing page.
|
|
12039
|
+
if (navigationReleased || !authorityIsCurrent())
|
|
12040
|
+
return deliveryResult;
|
|
12041
|
+
deliveryResult = this.markDeferredNavigationUnconfirmed(pendingExecution, context.currentRevision);
|
|
12009
12042
|
}
|
|
12010
12043
|
else if (deferredNavigation) {
|
|
12011
12044
|
deliveryResult = this.markDeferredNavigationUnconfirmed(pendingExecution, context.currentRevision);
|
|
@@ -14669,6 +14702,21 @@ class AiLiveAssist {
|
|
|
14669
14702
|
}, delay + Math.floor(Math.random() * 250));
|
|
14670
14703
|
this.reconnectTimer = timer;
|
|
14671
14704
|
}
|
|
14705
|
+
canRecoverControlTransport(channel, result, media) {
|
|
14706
|
+
return (this.channel === channel &&
|
|
14707
|
+
this.result === result &&
|
|
14708
|
+
this.media === media && media.isConnected && navigator.onLine &&
|
|
14709
|
+
!this.ended && !this.destroyed && !this.terminalRequestPending &&
|
|
14710
|
+
!this.reconnecting && !this.launchInFlight && !this.isVoicePresence() &&
|
|
14711
|
+
(this.lifecyclePhase === "active_text" || this.lifecyclePhase === "active_voice") &&
|
|
14712
|
+
!this.pendingAction && !this.pendingConfirmation && !this.pendingPageAssistance &&
|
|
14713
|
+
!this.pendingNavigationActionResult && !this.legacyActionExecution &&
|
|
14714
|
+
this.actionGrantInFlight.size === 0 &&
|
|
14715
|
+
!this.pendingModeUpgrade && !this.pendingTextModeTransition &&
|
|
14716
|
+
!this.messengerTransferPending && !this.messengerTransferred &&
|
|
14717
|
+
!this.handoffRequestLifecycleOwner && !this.readResume()?.pauseCommand &&
|
|
14718
|
+
this.modeUpgradePathIsCurrent(path()));
|
|
14719
|
+
}
|
|
14672
14720
|
async reconnectPendingHandoffControlInMemory(result, record, clientSequence, fencingToken, reconnectGeneration) {
|
|
14673
14721
|
if (reconnectGeneration !== this.reconnectGeneration ||
|
|
14674
14722
|
this.reconnecting ||
|
package/cjs/context.d.ts
CHANGED
|
@@ -23,6 +23,7 @@ export declare class PageContextCollector {
|
|
|
23
23
|
private mutationObserver;
|
|
24
24
|
private captureTimer;
|
|
25
25
|
private mutationCaptureTimer;
|
|
26
|
+
private mutationCaptureStartedAt;
|
|
26
27
|
private targetIds;
|
|
27
28
|
private pageScrollIds;
|
|
28
29
|
private navigationTargetBindings;
|
|
@@ -74,6 +75,7 @@ export declare class PageContextCollector {
|
|
|
74
75
|
private waitForScrollSettlement;
|
|
75
76
|
private actionApplied;
|
|
76
77
|
private clippedGeometry;
|
|
78
|
+
private clearMutationCapture;
|
|
77
79
|
private scheduleCapture;
|
|
78
80
|
private scheduleMutationCapture;
|
|
79
81
|
private scheduleViewportCapture;
|
package/cjs/context.js
CHANGED
|
@@ -10,6 +10,7 @@ const MAX_SELECTED_TEXT = 2000;
|
|
|
10
10
|
const SELECTION_RETENTION_MS = 120000;
|
|
11
11
|
const CAPTURE_SETTLE_MS = 250;
|
|
12
12
|
const MUTATION_CAPTURE_DEBOUNCE_MS = 350;
|
|
13
|
+
const MUTATION_CAPTURE_MAX_WAIT_MS = 1000;
|
|
13
14
|
const SCROLL_SETTLEMENT_TIMEOUT_MS = 2000;
|
|
14
15
|
const SCROLL_SETTLEMENT_FRAME_FALLBACK_MS = 50;
|
|
15
16
|
const SCROLL_SETTLEMENT_POSITION_TOLERANCE_PX = 1;
|
|
@@ -198,10 +199,10 @@ const visibleRect = (element) => {
|
|
|
198
199
|
return right > left && bottom > top ? { left, top, right, bottom } : null;
|
|
199
200
|
};
|
|
200
201
|
const isVisible = (element) => visibleRect(element) !== null;
|
|
201
|
-
const
|
|
202
|
+
const interactionPoint = (element, trustedGuideRoot = null) => {
|
|
202
203
|
const rect = visibleRect(element);
|
|
203
204
|
if (!rect)
|
|
204
|
-
return
|
|
205
|
+
return null;
|
|
205
206
|
// Five bounded hit tests permit an exposed part of a link without treating
|
|
206
207
|
// an ancestor, empty hit, or a covering sibling as an interactable target.
|
|
207
208
|
const insetX = Math.min(1, (rect.right - rect.left) / 4);
|
|
@@ -211,13 +212,16 @@ const isCovered = (element, trustedGuideRoot = null) => {
|
|
|
211
212
|
[rect.left + insetX, rect.top + insetY], [rect.right - insetX, rect.top + insetY],
|
|
212
213
|
[rect.left + insetX, rect.bottom - insetY], [rect.right - insetX, rect.bottom - insetY],
|
|
213
214
|
];
|
|
214
|
-
|
|
215
|
+
for (const [x, y] of points) {
|
|
215
216
|
const top = trustedGuideRoot?.isConnected && typeof document.elementsFromPoint === 'function'
|
|
216
217
|
? document.elementsFromPoint(x, y).find(candidate => candidate !== trustedGuideRoot) ?? null
|
|
217
218
|
: document.elementFromPoint(x, y);
|
|
218
|
-
|
|
219
|
-
|
|
219
|
+
if (top !== null && (top === element || element.contains(top)))
|
|
220
|
+
return { x: x, y: y, hit: top };
|
|
221
|
+
}
|
|
222
|
+
return null;
|
|
220
223
|
};
|
|
224
|
+
const isCovered = (element, trustedGuideRoot = null) => interactionPoint(element, trustedGuideRoot) === null;
|
|
221
225
|
const isNavigationMenu = (element, role) => {
|
|
222
226
|
if (role !== 'button' || literalBooleanAttribute(element, 'aria-expanded') === undefined ||
|
|
223
227
|
element.hasAttribute('aria-pressed') || element.hasAttribute('aria-checked') ||
|
|
@@ -492,15 +496,17 @@ const checkedStateFor = (element) => {
|
|
|
492
496
|
}
|
|
493
497
|
return framerToggleDescriptor(element)?.checked;
|
|
494
498
|
};
|
|
495
|
-
const dispatchFramerPrimaryPointerSequence = (element) => {
|
|
499
|
+
const dispatchFramerPrimaryPointerSequence = (element, trustedGuideRoot = null) => {
|
|
496
500
|
if (typeof window.PointerEvent !== 'function') {
|
|
497
501
|
throw new Error('Framer pointer interaction is unavailable.');
|
|
498
502
|
}
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
const
|
|
502
|
-
|
|
503
|
-
|
|
503
|
+
// Capture and dispatch must agree on the exposed point and exact descendant,
|
|
504
|
+
// including when our own mobile surface covers a link's router-owned logo.
|
|
505
|
+
const point = interactionPoint(element, trustedGuideRoot);
|
|
506
|
+
if (!point)
|
|
507
|
+
throw new Error('The approved pointer target is covered.');
|
|
508
|
+
const { x: clientX, y: clientY } = point;
|
|
509
|
+
const eventTarget = element instanceof HTMLAnchorElement ? point.hit : element;
|
|
504
510
|
const nestedLink = eventTarget.closest('a[href]');
|
|
505
511
|
if (element instanceof HTMLAnchorElement && nestedLink instanceof HTMLAnchorElement && nestedLink !== element &&
|
|
506
512
|
(!sameNavigationDestination(nestedLink.href, element.href) || nestedLink.hasAttribute('download') ||
|
|
@@ -572,11 +578,11 @@ const dispatchFramerPrimaryPointerSequence = (element) => {
|
|
|
572
578
|
detail: 1,
|
|
573
579
|
}));
|
|
574
580
|
};
|
|
575
|
-
const dispatchFramerTogglePrimaryPointer = (element) => {
|
|
581
|
+
const dispatchFramerTogglePrimaryPointer = (element, trustedGuideRoot = null) => {
|
|
576
582
|
if (!framerToggleDescriptor(element)) {
|
|
577
583
|
throw new Error('Framer toggle pointer interaction is unavailable.');
|
|
578
584
|
}
|
|
579
|
-
dispatchFramerPrimaryPointerSequence(element);
|
|
585
|
+
dispatchFramerPrimaryPointerSequence(element, trustedGuideRoot);
|
|
580
586
|
};
|
|
581
587
|
const semanticSourceSensitivity = (source) => {
|
|
582
588
|
if (source.closest(PRIVATE_SELECTOR) || source.querySelector(PRIVATE_SELECTOR)) {
|
|
@@ -634,6 +640,18 @@ const safeOptionLabel = (option) => {
|
|
|
634
640
|
return sensitivityFor(option, label) === 'none' ? label : '';
|
|
635
641
|
};
|
|
636
642
|
const isActionTarget = (element) => element.matches('a[href],button,input,select,textarea,summary,form,[contenteditable="true"],[role="button"],[role="link"],[role="checkbox"],[role="radio"],[role="combobox"],[role="textbox"],[tabindex]');
|
|
643
|
+
const belongsToInteractiveControl = (element) => {
|
|
644
|
+
for (let current = element; current; current = current.parentElement) {
|
|
645
|
+
// Form copy can carry useful public instructions. Only controls and links
|
|
646
|
+
// already represented in targets are omitted from visible page text.
|
|
647
|
+
if (current.matches('form'))
|
|
648
|
+
continue;
|
|
649
|
+
// A bare tabindex is often a focusable Framer content card, not an action.
|
|
650
|
+
if (current.matches('a[href],button,input,select,textarea,summary,[contenteditable="true"],[role="button"],[role="link"],[role="checkbox"],[role="radio"],[role="combobox"],[role="textbox"]'))
|
|
651
|
+
return true;
|
|
652
|
+
}
|
|
653
|
+
return false;
|
|
654
|
+
};
|
|
637
655
|
const READONLY_INPUT_TYPES = new Set([
|
|
638
656
|
'date',
|
|
639
657
|
'datetime-local',
|
|
@@ -746,6 +764,7 @@ class PageContextCollector {
|
|
|
746
764
|
this.mutationObserver = null;
|
|
747
765
|
this.captureTimer = null;
|
|
748
766
|
this.mutationCaptureTimer = null;
|
|
767
|
+
this.mutationCaptureStartedAt = null;
|
|
749
768
|
this.targetIds = new WeakMap();
|
|
750
769
|
this.pageScrollIds = new WeakMap();
|
|
751
770
|
this.navigationTargetBindings = new WeakMap();
|
|
@@ -773,9 +792,7 @@ class PageContextCollector {
|
|
|
773
792
|
this.pendingCapture = true;
|
|
774
793
|
return;
|
|
775
794
|
}
|
|
776
|
-
|
|
777
|
-
clearTimeout(this.mutationCaptureTimer);
|
|
778
|
-
this.mutationCaptureTimer = null;
|
|
795
|
+
this.clearMutationCapture();
|
|
779
796
|
if (this.captureTimer)
|
|
780
797
|
return;
|
|
781
798
|
this.captureTimer = setTimeout(() => {
|
|
@@ -792,12 +809,19 @@ class PageContextCollector {
|
|
|
792
809
|
}
|
|
793
810
|
if (this.captureTimer)
|
|
794
811
|
return;
|
|
812
|
+
// Decorative animation must not indefinitely hide newly available links.
|
|
813
|
+
// Retain trailing-edge batching, with at most one forced scan per second
|
|
814
|
+
// during continuous churn. Unchanged semantic snapshots are still deduped.
|
|
815
|
+
const now = performance.now();
|
|
816
|
+
this.mutationCaptureStartedAt ?? (this.mutationCaptureStartedAt = now);
|
|
817
|
+
const delay = Math.max(0, Math.min(MUTATION_CAPTURE_DEBOUNCE_MS, MUTATION_CAPTURE_MAX_WAIT_MS - (now - this.mutationCaptureStartedAt)));
|
|
795
818
|
if (this.mutationCaptureTimer)
|
|
796
819
|
clearTimeout(this.mutationCaptureTimer);
|
|
797
820
|
this.mutationCaptureTimer = setTimeout(() => {
|
|
798
821
|
this.mutationCaptureTimer = null;
|
|
822
|
+
this.mutationCaptureStartedAt = null;
|
|
799
823
|
void this.captureAndPublishIfChanged();
|
|
800
|
-
},
|
|
824
|
+
}, delay);
|
|
801
825
|
};
|
|
802
826
|
this.scheduleViewportCapture = () => {
|
|
803
827
|
this.scheduleMutationCapture();
|
|
@@ -870,9 +894,7 @@ class PageContextCollector {
|
|
|
870
894
|
if (this.captureTimer)
|
|
871
895
|
clearTimeout(this.captureTimer);
|
|
872
896
|
this.captureTimer = null;
|
|
873
|
-
|
|
874
|
-
clearTimeout(this.mutationCaptureTimer);
|
|
875
|
-
this.mutationCaptureTimer = null;
|
|
897
|
+
this.clearMutationCapture();
|
|
876
898
|
if (this.releasedNavigationSettlement &&
|
|
877
899
|
window.location.href !== this.releasedNavigationSettlement.fromUrl) {
|
|
878
900
|
this.finishReleasedNavigationSettlement(sameNavigationDestination(window.location.href, this.releasedNavigationSettlement.toUrl)
|
|
@@ -1003,9 +1025,7 @@ class PageContextCollector {
|
|
|
1003
1025
|
if (this.captureTimer)
|
|
1004
1026
|
clearTimeout(this.captureTimer);
|
|
1005
1027
|
this.captureTimer = null;
|
|
1006
|
-
|
|
1007
|
-
clearTimeout(this.mutationCaptureTimer);
|
|
1008
|
-
this.mutationCaptureTimer = null;
|
|
1028
|
+
this.clearMutationCapture();
|
|
1009
1029
|
window.removeEventListener('resize', this.scheduleViewportCapture);
|
|
1010
1030
|
document.removeEventListener('scroll', this.scheduleViewportCapture, true);
|
|
1011
1031
|
window.visualViewport?.removeEventListener('resize', this.scheduleViewportCapture);
|
|
@@ -1177,23 +1197,33 @@ class PageContextCollector {
|
|
|
1177
1197
|
});
|
|
1178
1198
|
}
|
|
1179
1199
|
const visibleText = [];
|
|
1200
|
+
const capturedSemanticText = new Set();
|
|
1180
1201
|
let totalText = 0;
|
|
1181
1202
|
const walker = document.createTreeWalker(document.body ?? document.documentElement, NodeFilter.SHOW_TEXT);
|
|
1182
1203
|
let node = walker.nextNode();
|
|
1183
1204
|
while (node && visibleText.length < MAX_TEXT_ITEMS && totalText < MAX_TOTAL_TEXT) {
|
|
1184
1205
|
const parent = node.parentElement;
|
|
1185
|
-
const
|
|
1206
|
+
const semanticTextElement = parent?.closest('[role="img"][aria-label]') ?? null;
|
|
1207
|
+
const textElement = semanticTextElement && isRendered(semanticTextElement)
|
|
1208
|
+
? semanticTextElement
|
|
1209
|
+
: parent;
|
|
1210
|
+
const text = normalize(semanticTextElement?.getAttribute('aria-label') ?? node.textContent ?? '', 500);
|
|
1186
1211
|
if (parent &&
|
|
1212
|
+
textElement &&
|
|
1187
1213
|
text &&
|
|
1188
1214
|
!parent.closest(PRIVATE_SELECTOR) &&
|
|
1189
1215
|
!parent.closest('script,style,noscript,template,svg') &&
|
|
1190
1216
|
!parent.closest('input,select,textarea,option') &&
|
|
1191
|
-
|
|
1192
|
-
|
|
1217
|
+
(!semanticTextElement || !capturedSemanticText.has(semanticTextElement)) &&
|
|
1218
|
+
!belongsToInteractiveControl(textElement) &&
|
|
1219
|
+
isRendered(textElement) &&
|
|
1220
|
+
sensitivityFor(textElement, nameFor(textElement)) === 'none') {
|
|
1193
1221
|
const textId = this.textIds.get(node) ?? id('text');
|
|
1194
1222
|
this.textIds.set(node, textId);
|
|
1195
1223
|
visibleText.push({ id: textId, text, provenance: 'page' });
|
|
1196
1224
|
totalText += text.length;
|
|
1225
|
+
if (semanticTextElement)
|
|
1226
|
+
capturedSemanticText.add(semanticTextElement);
|
|
1197
1227
|
}
|
|
1198
1228
|
node = walker.nextNode();
|
|
1199
1229
|
}
|
|
@@ -1842,7 +1872,7 @@ class PageContextCollector {
|
|
|
1842
1872
|
};
|
|
1843
1873
|
window.addEventListener('click', preventNativeNavigation);
|
|
1844
1874
|
try {
|
|
1845
|
-
dispatchFramerPrimaryPointerSequence(navigation.source);
|
|
1875
|
+
dispatchFramerPrimaryPointerSequence(navigation.source, this.trustedGuideRoot);
|
|
1846
1876
|
}
|
|
1847
1877
|
finally {
|
|
1848
1878
|
window.removeEventListener('click', preventNativeNavigation);
|
|
@@ -1924,7 +1954,7 @@ class PageContextCollector {
|
|
|
1924
1954
|
if (desiredStates.length > 0 && desiredStates.every(Boolean))
|
|
1925
1955
|
return;
|
|
1926
1956
|
if (desiredChecked !== undefined && framerToggleDescriptor(element)) {
|
|
1927
|
-
dispatchFramerTogglePrimaryPointer(element);
|
|
1957
|
+
dispatchFramerTogglePrimaryPointer(element, this.trustedGuideRoot);
|
|
1928
1958
|
return;
|
|
1929
1959
|
}
|
|
1930
1960
|
if (element instanceof HTMLAnchorElement) {
|
|
@@ -2232,9 +2262,19 @@ class PageContextCollector {
|
|
|
2232
2262
|
height,
|
|
2233
2263
|
};
|
|
2234
2264
|
}
|
|
2265
|
+
clearMutationCapture() {
|
|
2266
|
+
if (this.mutationCaptureTimer)
|
|
2267
|
+
clearTimeout(this.mutationCaptureTimer);
|
|
2268
|
+
this.mutationCaptureTimer = null;
|
|
2269
|
+
this.mutationCaptureStartedAt = null;
|
|
2270
|
+
}
|
|
2235
2271
|
async captureAndPublishIfChanged() {
|
|
2236
|
-
if (this.navigationSettlement)
|
|
2272
|
+
if (!this.running || this.navigationSettlement)
|
|
2273
|
+
return;
|
|
2274
|
+
if (this.executingActionId) {
|
|
2275
|
+
this.pendingCapture = true;
|
|
2237
2276
|
return;
|
|
2277
|
+
}
|
|
2238
2278
|
const snapshot = this.captureSnapshot(false);
|
|
2239
2279
|
if (snapshot && this.running)
|
|
2240
2280
|
await this.callbacks.onSnapshot(snapshot);
|
|
@@ -2319,9 +2359,7 @@ class PageContextCollector {
|
|
|
2319
2359
|
if (this.captureTimer)
|
|
2320
2360
|
clearTimeout(this.captureTimer);
|
|
2321
2361
|
this.captureTimer = null;
|
|
2322
|
-
|
|
2323
|
-
clearTimeout(this.mutationCaptureTimer);
|
|
2324
|
-
this.mutationCaptureTimer = null;
|
|
2362
|
+
this.clearMutationCapture();
|
|
2325
2363
|
return settlement;
|
|
2326
2364
|
}
|
|
2327
2365
|
observeCausalNavigationAfterMutation(actionId, destinationPath, beforeActionUrl) {
|
package/cjs/control.d.ts
CHANGED
|
@@ -24,6 +24,10 @@ export type ControlOutboundOperation = {
|
|
|
24
24
|
type: string;
|
|
25
25
|
};
|
|
26
26
|
export type ReservedControlOperation = Readonly<ControlOutboundOperation>;
|
|
27
|
+
type TransportRecovery = {
|
|
28
|
+
canRecover: () => boolean;
|
|
29
|
+
onRecovered: () => Promise<void>;
|
|
30
|
+
};
|
|
27
31
|
export declare class ControlChannel {
|
|
28
32
|
private readonly result;
|
|
29
33
|
private readonly tabId;
|
|
@@ -31,6 +35,7 @@ export declare class ControlChannel {
|
|
|
31
35
|
private readonly onDisconnect;
|
|
32
36
|
private readonly onClientSequence;
|
|
33
37
|
private readonly onUnsafeDisconnect;
|
|
38
|
+
private readonly transportRecovery?;
|
|
34
39
|
private socket;
|
|
35
40
|
private clientSequence;
|
|
36
41
|
private serverSequence;
|
|
@@ -54,7 +59,9 @@ export declare class ControlChannel {
|
|
|
54
59
|
private pendingConnection;
|
|
55
60
|
private helloOperation;
|
|
56
61
|
private fencedAuthorityConfirmed;
|
|
57
|
-
|
|
62
|
+
private transportRecoveryAttempted;
|
|
63
|
+
private recoveringTransport;
|
|
64
|
+
constructor(result: RuntimeAuthorization, tabId: string, clientVersion: string, onDisconnect: () => void, initialClientSequence?: number, onClientSequence?: (sequence: number) => void, onUnsafeDisconnect?: () => void, transportRecovery?: TransportRecovery | undefined);
|
|
58
65
|
get leaseToken(): number;
|
|
59
66
|
get lastClientSequence(): number;
|
|
60
67
|
get isOpen(): boolean;
|
|
@@ -78,6 +85,7 @@ export declare class ControlChannel {
|
|
|
78
85
|
private handleMessage;
|
|
79
86
|
private recoverFromSequenceGap;
|
|
80
87
|
private notifyUnsafeDisconnect;
|
|
88
|
+
private recoverTransport;
|
|
81
89
|
private enqueueHandlers;
|
|
82
90
|
private dispatchHandlers;
|
|
83
91
|
private handleHandlerFailure;
|
package/cjs/control.js
CHANGED
|
@@ -6,6 +6,7 @@ const PROTOCOL_VIOLATION_CLOSE_CODE = 4008;
|
|
|
6
6
|
const SERVER_SEQUENCE_GAP_CLOSE_CODE = 4009;
|
|
7
7
|
const HANDLER_FAILURE_CLOSE_CODE = 4010;
|
|
8
8
|
const HANDLER_TIMEOUT_MS = 15000;
|
|
9
|
+
const TRANSPORT_RECOVERY_TIMEOUT_MS = 5000;
|
|
9
10
|
const MAX_PENDING_HANDLER_MESSAGES = 64;
|
|
10
11
|
const MAX_PENDING_STATEFUL_OPERATIONS = 64;
|
|
11
12
|
const MAX_SETTLED_OPERATION_HISTORY = 100;
|
|
@@ -38,13 +39,14 @@ const STATEFUL_CONTROL_TYPES = new Set([
|
|
|
38
39
|
]);
|
|
39
40
|
const outcomeKey = (messageId, sequence) => `${messageId}\u0000${sequence}`;
|
|
40
41
|
class ControlChannel {
|
|
41
|
-
constructor(result, tabId, clientVersion, onDisconnect, initialClientSequence = 0, onClientSequence = () => undefined, onUnsafeDisconnect = () => undefined) {
|
|
42
|
+
constructor(result, tabId, clientVersion, onDisconnect, initialClientSequence = 0, onClientSequence = () => undefined, onUnsafeDisconnect = () => undefined, transportRecovery) {
|
|
42
43
|
this.result = result;
|
|
43
44
|
this.tabId = tabId;
|
|
44
45
|
this.clientVersion = clientVersion;
|
|
45
46
|
this.onDisconnect = onDisconnect;
|
|
46
47
|
this.onClientSequence = onClientSequence;
|
|
47
48
|
this.onUnsafeDisconnect = onUnsafeDisconnect;
|
|
49
|
+
this.transportRecovery = transportRecovery;
|
|
48
50
|
this.socket = null;
|
|
49
51
|
this.serverSequence = 0;
|
|
50
52
|
this.serverBaselineEstablished = false;
|
|
@@ -67,6 +69,8 @@ class ControlChannel {
|
|
|
67
69
|
this.pendingConnection = null;
|
|
68
70
|
this.helloOperation = null;
|
|
69
71
|
this.fencedAuthorityConfirmed = false;
|
|
72
|
+
this.transportRecoveryAttempted = false;
|
|
73
|
+
this.recoveringTransport = false;
|
|
70
74
|
this.clientSequence =
|
|
71
75
|
Number.isSafeInteger(initialClientSequence) && initialClientSequence >= 0
|
|
72
76
|
? initialClientSequence
|
|
@@ -79,7 +83,7 @@ class ControlChannel {
|
|
|
79
83
|
return this.clientSequence;
|
|
80
84
|
}
|
|
81
85
|
get isOpen() {
|
|
82
|
-
return this.socket?.readyState === WebSocket.OPEN;
|
|
86
|
+
return (!this.recoveringTransport && this.socket?.readyState === WebSocket.OPEN);
|
|
83
87
|
}
|
|
84
88
|
get connectionHelloOperation() {
|
|
85
89
|
return this.helloOperation ? { ...this.helloOperation } : null;
|
|
@@ -180,10 +184,19 @@ class ControlChannel {
|
|
|
180
184
|
this.rejectPendingConnection(socket, generation, new Error("The AI Guide control channel could not connect."));
|
|
181
185
|
}
|
|
182
186
|
}, { once: true });
|
|
183
|
-
socket.addEventListener("close", () => {
|
|
187
|
+
socket.addEventListener("close", (event) => {
|
|
184
188
|
if (!this.isCurrentSocket(socket, generation))
|
|
185
189
|
return;
|
|
186
190
|
const wasIntentionallyClosed = this.intentionallyClosed;
|
|
191
|
+
// A settled transport loss is not a new session. Keep its provider
|
|
192
|
+
// connection alive for one bounded retry, but never carry ambiguous
|
|
193
|
+
// browser writes, an active handler, or a protocol failure across it.
|
|
194
|
+
const canRecover = event.code === 1006 &&
|
|
195
|
+
this.fencedAuthorityConfirmed &&
|
|
196
|
+
!this.transportRecoveryAttempted &&
|
|
197
|
+
!this.sequenceGapDetected &&
|
|
198
|
+
this.pendingHandlerMessages === 0 &&
|
|
199
|
+
this.unsettledStatefulOperations.size === 0;
|
|
187
200
|
this.socket = null;
|
|
188
201
|
this.socketGeneration += 1;
|
|
189
202
|
this.stopHeartbeat();
|
|
@@ -197,12 +210,24 @@ class ControlChannel {
|
|
|
197
210
|
this.pendingHandlerMessages = 0;
|
|
198
211
|
this.rejectPendingConnection(socket, generation, new Error("The AI Guide control channel closed before connecting."));
|
|
199
212
|
if (!wasIntentionallyClosed && !signal.aborted) {
|
|
213
|
+
try {
|
|
214
|
+
if (canRecover && this.transportRecovery?.canRecover()) {
|
|
215
|
+
void this.recoverTransport(signal);
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
/* A failed eligibility check uses full recovery. */
|
|
221
|
+
}
|
|
200
222
|
this.notifyUnsafeDisconnect();
|
|
201
223
|
}
|
|
202
224
|
});
|
|
203
225
|
});
|
|
204
226
|
}
|
|
205
227
|
send(type, payload, stableMessageId) {
|
|
228
|
+
if (this.recoveringTransport && type !== "hello" && type !== "heartbeat") {
|
|
229
|
+
throw new Error("The AI Guide control channel is reconnecting.");
|
|
230
|
+
}
|
|
206
231
|
if (!this.socket || this.socket.readyState !== WebSocket.OPEN) {
|
|
207
232
|
throw new Error("The AI Guide control channel is not connected.");
|
|
208
233
|
}
|
|
@@ -513,6 +538,59 @@ class ControlChannel {
|
|
|
513
538
|
this.onDisconnect();
|
|
514
539
|
}
|
|
515
540
|
}
|
|
541
|
+
async recoverTransport(signal) {
|
|
542
|
+
this.transportRecoveryAttempted = true;
|
|
543
|
+
this.recoveringTransport = true;
|
|
544
|
+
const controller = new AbortController();
|
|
545
|
+
const abort = () => controller.abort();
|
|
546
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
547
|
+
let timeout;
|
|
548
|
+
const deadline = new Promise((_resolve, reject) => {
|
|
549
|
+
timeout = setTimeout(() => {
|
|
550
|
+
controller.abort();
|
|
551
|
+
reject(new Error("The control reconnect timed out."));
|
|
552
|
+
}, TRANSPORT_RECOVERY_TIMEOUT_MS);
|
|
553
|
+
});
|
|
554
|
+
const fencingToken = this.fencingToken;
|
|
555
|
+
let recovered = false;
|
|
556
|
+
try {
|
|
557
|
+
await Promise.race([
|
|
558
|
+
deadline,
|
|
559
|
+
(async () => {
|
|
560
|
+
if (signal.aborted)
|
|
561
|
+
throw new Error("The control connection was retired.");
|
|
562
|
+
await this.connect(controller.signal, true);
|
|
563
|
+
const hello = await this.waitForHelloOutcome(2000);
|
|
564
|
+
if (hello.status !== "ack" || controller.signal.aborted) {
|
|
565
|
+
throw new Error("The control reconnect was not acknowledged.");
|
|
566
|
+
}
|
|
567
|
+
const authority = await this.proveFencedAuthority(2000);
|
|
568
|
+
if (authority.status !== "ack" ||
|
|
569
|
+
controller.signal.aborted ||
|
|
570
|
+
this.fencingToken !== fencingToken ||
|
|
571
|
+
!this.transportRecovery?.canRecover()) {
|
|
572
|
+
throw new Error("The control reconnect no longer owns this session.");
|
|
573
|
+
}
|
|
574
|
+
this.recoveringTransport = false;
|
|
575
|
+
await this.transportRecovery.onRecovered();
|
|
576
|
+
recovered = !controller.signal.aborted && this.hasFencedAuthority;
|
|
577
|
+
})(),
|
|
578
|
+
]);
|
|
579
|
+
}
|
|
580
|
+
catch {
|
|
581
|
+
// The existing full-resume path revokes local executable authority and
|
|
582
|
+
// handles expired credentials, changed ownership, and longer outages.
|
|
583
|
+
}
|
|
584
|
+
finally {
|
|
585
|
+
clearTimeout(timeout);
|
|
586
|
+
signal.removeEventListener("abort", abort);
|
|
587
|
+
this.recoveringTransport = false;
|
|
588
|
+
if (!recovered && !signal.aborted && !this.intentionallyClosed) {
|
|
589
|
+
this.retireSocket("control reconnect failed");
|
|
590
|
+
this.notifyUnsafeDisconnect();
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
}
|
|
516
594
|
enqueueHandlers(message, socket, generation) {
|
|
517
595
|
const isCurrent = () => this.isMessageSourceCurrent(socket, generation);
|
|
518
596
|
if (!isCurrent())
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type LiveDuplexMediaOptions, type LiveDuplexMediaDependencies } from './liveDuplexMedia.js';
|
|
2
|
-
export declare const LIVE_CONTROL_CLIENT_VERSION = "7.5.
|
|
2
|
+
export declare const LIVE_CONTROL_CLIENT_VERSION = "7.5.8";
|
|
3
3
|
export declare const LIVE_CONTROL_CLIENT_CAPABILITIES: readonly ["live_duplex_v1", "live_local_wake_v1", "live_fragment_captions_v1"];
|
|
4
4
|
declare const phases: readonly ["voice_off", "local_listening", "wake_buffering", "starting", "active", "tentative_idle", "closing", "uncertain", "muted", "suspended", "budget_blocked", "ended"];
|
|
5
5
|
export type LiveControlState = {
|
package/cjs/media.d.ts
CHANGED
|
@@ -36,6 +36,7 @@ export declare class RealtimeMedia {
|
|
|
36
36
|
connect(result: LaunchResult, mode: AssistMode, signal: AbortSignal, offerIdempotencyKey?: string, expectedSessionState?: "active_text" | "active_voice"): Promise<number>;
|
|
37
37
|
setMuted(muted: boolean): void;
|
|
38
38
|
get isMuted(): boolean;
|
|
39
|
+
get isConnected(): boolean;
|
|
39
40
|
setAssistantPlaybackAudible(audible: boolean): void;
|
|
40
41
|
setManualTurn(manual: boolean): void;
|
|
41
42
|
beginPushToTalk(): void;
|
package/cjs/media.js
CHANGED
|
@@ -137,6 +137,9 @@ class RealtimeMedia {
|
|
|
137
137
|
get isMuted() {
|
|
138
138
|
return this.muted;
|
|
139
139
|
}
|
|
140
|
+
get isConnected() {
|
|
141
|
+
return !this.closed && this.connection?.connectionState === "connected";
|
|
142
|
+
}
|
|
140
143
|
setAssistantPlaybackAudible(audible) {
|
|
141
144
|
if (this.closed)
|
|
142
145
|
return;
|
package/cjs/ui.js
CHANGED
|
@@ -6793,7 +6793,9 @@ class WidgetView {
|
|
|
6793
6793
|
this.setActionStateRunning(status === "running");
|
|
6794
6794
|
if (this.actionStateRunning) {
|
|
6795
6795
|
this.clearAssistantReadyVisualTimer();
|
|
6796
|
+
const previousState = this.assistantState;
|
|
6796
6797
|
this.assistantState = "thinking";
|
|
6798
|
+
this.updateRemoteAudioEvidenceExpectation(previousState, "thinking");
|
|
6797
6799
|
const assistantCaption = this.captions.querySelector('[data-caption="assistant-partial"]');
|
|
6798
6800
|
if (!assistantCaption || assistantCaption.classList.contains("pending")) {
|
|
6799
6801
|
this.ensurePendingCaption("assistant", this.messages.thinking);
|
package/cjs/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const VERSION = "7.5.
|
|
1
|
+
export declare const VERSION = "7.5.8";
|
package/cjs/version.js
CHANGED
package/lib/AiLiveAssist.d.ts
CHANGED
|
@@ -34,7 +34,7 @@ export declare const normalizeAiGuideOpenInput: (input?: AiGuideOpenInput) => Re
|
|
|
34
34
|
export declare const resolveAiGuideProtocolVersion: (bootstrap: Pick<BootstrapConfig, "experienceMode" | "minimumProtocolVersion" | "protocol">) => 1 | 2;
|
|
35
35
|
export default class AiLiveAssist {
|
|
36
36
|
private readonly options;
|
|
37
|
-
readonly version = "7.5.
|
|
37
|
+
readonly version = "7.5.8";
|
|
38
38
|
private readonly host;
|
|
39
39
|
private readonly guideAdmission;
|
|
40
40
|
private readonly standaloneAdmissionPending;
|
|
@@ -627,6 +627,7 @@ export default class AiLiveAssist {
|
|
|
627
627
|
private submitFeedback;
|
|
628
628
|
private retry;
|
|
629
629
|
private scheduleReconnect;
|
|
630
|
+
private canRecoverControlTransport;
|
|
630
631
|
private reconnectPendingHandoffControlInMemory;
|
|
631
632
|
private reconnectMessengerTerminalControl;
|
|
632
633
|
private clearReconnectTimer;
|
package/lib/AiLiveAssist.js
CHANGED
|
@@ -7937,7 +7937,28 @@ export default class AiLiveAssist {
|
|
|
7937
7937
|
? this.navigationConsentResetReady
|
|
7938
7938
|
: null;
|
|
7939
7939
|
const resumeRecordAtConnect = this.readResume();
|
|
7940
|
-
|
|
7940
|
+
let recoveryPath = null;
|
|
7941
|
+
const channel = new ControlChannel(result, this.tabId, VERSION, () => this.scheduleReconnect(true), this.readResume()?.clientSequence ?? 0, (sequence) => this.updateResumeSequence(sequence), () => this.handleControlSequenceGap(), {
|
|
7942
|
+
canRecover: () => {
|
|
7943
|
+
const currentPath = path();
|
|
7944
|
+
if (!this.canRecoverControlTransport(channel, result, media) ||
|
|
7945
|
+
(recoveryPath !== null && recoveryPath !== currentPath))
|
|
7946
|
+
return false;
|
|
7947
|
+
recoveryPath = currentPath;
|
|
7948
|
+
return true;
|
|
7949
|
+
},
|
|
7950
|
+
onRecovered: async () => {
|
|
7951
|
+
assertCurrent();
|
|
7952
|
+
const fencingToken = channel.leaseToken;
|
|
7953
|
+
channel.startHeartbeat(this.heartbeatSeconds);
|
|
7954
|
+
await this.replayPreFenceAuthorityEnvelopes(channel, fencingToken, assertCurrent);
|
|
7955
|
+
assertCurrent();
|
|
7956
|
+
await this.context?.captureAndPublish();
|
|
7957
|
+
assertCurrent();
|
|
7958
|
+
this.flushPlaybackState();
|
|
7959
|
+
this.reapplyConservativePreFencePresence(channel, fencingToken);
|
|
7960
|
+
},
|
|
7961
|
+
});
|
|
7941
7962
|
const pauseHelloStateSignal = deferredSignal();
|
|
7942
7963
|
(this.pauseHelloStateSignals ?? (this.pauseHelloStateSignals = new WeakMap())).set(channel, pauseHelloStateSignal);
|
|
7943
7964
|
this.handoffContextPromptReceiptChannel = null;
|
|
@@ -8307,7 +8328,7 @@ export default class AiLiveAssist {
|
|
|
8307
8328
|
this.context === context;
|
|
8308
8329
|
context = new PageContextCollector({
|
|
8309
8330
|
onSnapshot: (snapshot) => {
|
|
8310
|
-
if (isCurrent() && this.channel?.
|
|
8331
|
+
if (isCurrent() && this.channel?.hasFencedAuthority) {
|
|
8311
8332
|
this.send("context.snapshot", snapshot);
|
|
8312
8333
|
}
|
|
8313
8334
|
},
|
|
@@ -9666,7 +9687,15 @@ export default class AiLiveAssist {
|
|
|
9666
9687
|
this.activePageAssistanceAuthorityEpoch === null &&
|
|
9667
9688
|
this.activeActionAuthorityEpoch === null;
|
|
9668
9689
|
if (!exactApplied || !localAuthorityInactive) {
|
|
9669
|
-
|
|
9690
|
+
// A resumed document still owes the original action's terminal result.
|
|
9691
|
+
// Preserve only its grantless outbox, never browser-control authority.
|
|
9692
|
+
// Delivery remains fenced by the server's exact already_started receipt.
|
|
9693
|
+
const preserveNavigationResult = source === "navigation_resume" &&
|
|
9694
|
+
this.awaitingNavigationConsentReset &&
|
|
9695
|
+
structurallyValid &&
|
|
9696
|
+
localAuthorityInactive &&
|
|
9697
|
+
this.pendingNavigationActionResult?.grant === null;
|
|
9698
|
+
this.invalidatePageAssistanceAuthority(false, preserveNavigationResult);
|
|
9670
9699
|
}
|
|
9671
9700
|
if (!structurallyValid ||
|
|
9672
9701
|
identity.authorityEpoch < this.minimumPageAssistanceAuthorityEpoch) {
|
|
@@ -11989,15 +12018,19 @@ export default class AiLiveAssist {
|
|
|
11989
12018
|
// intentionally result-less until navigation settles, preserving the
|
|
11990
12019
|
// cross-document recovery contract if this click unloads the page.
|
|
11991
12020
|
pendingExecution.result = deliveryResult;
|
|
12021
|
+
let navigationReleased = false;
|
|
11992
12022
|
try {
|
|
11993
|
-
|
|
11994
|
-
return deliveryResult;
|
|
11995
|
-
}
|
|
11996
|
-
deliveryResult = this.markDeferredNavigationUnconfirmed(pendingExecution, context.currentRevision);
|
|
12023
|
+
navigationReleased = await this.navigateTo(deferredNavigation);
|
|
11997
12024
|
}
|
|
11998
12025
|
catch {
|
|
11999
|
-
|
|
12026
|
+
// A rejected router and an outgoing document both settle here.
|
|
12000
12027
|
}
|
|
12028
|
+
// Pagehide stops the old collector and settles its pending router
|
|
12029
|
+
// callback. It must not replace the destination document's grantless
|
|
12030
|
+
// recovery record with an unconfirmed result from the outgoing page.
|
|
12031
|
+
if (navigationReleased || !authorityIsCurrent())
|
|
12032
|
+
return deliveryResult;
|
|
12033
|
+
deliveryResult = this.markDeferredNavigationUnconfirmed(pendingExecution, context.currentRevision);
|
|
12001
12034
|
}
|
|
12002
12035
|
else if (deferredNavigation) {
|
|
12003
12036
|
deliveryResult = this.markDeferredNavigationUnconfirmed(pendingExecution, context.currentRevision);
|
|
@@ -14661,6 +14694,21 @@ export default class AiLiveAssist {
|
|
|
14661
14694
|
}, delay + Math.floor(Math.random() * 250));
|
|
14662
14695
|
this.reconnectTimer = timer;
|
|
14663
14696
|
}
|
|
14697
|
+
canRecoverControlTransport(channel, result, media) {
|
|
14698
|
+
return (this.channel === channel &&
|
|
14699
|
+
this.result === result &&
|
|
14700
|
+
this.media === media && media.isConnected && navigator.onLine &&
|
|
14701
|
+
!this.ended && !this.destroyed && !this.terminalRequestPending &&
|
|
14702
|
+
!this.reconnecting && !this.launchInFlight && !this.isVoicePresence() &&
|
|
14703
|
+
(this.lifecyclePhase === "active_text" || this.lifecyclePhase === "active_voice") &&
|
|
14704
|
+
!this.pendingAction && !this.pendingConfirmation && !this.pendingPageAssistance &&
|
|
14705
|
+
!this.pendingNavigationActionResult && !this.legacyActionExecution &&
|
|
14706
|
+
this.actionGrantInFlight.size === 0 &&
|
|
14707
|
+
!this.pendingModeUpgrade && !this.pendingTextModeTransition &&
|
|
14708
|
+
!this.messengerTransferPending && !this.messengerTransferred &&
|
|
14709
|
+
!this.handoffRequestLifecycleOwner && !this.readResume()?.pauseCommand &&
|
|
14710
|
+
this.modeUpgradePathIsCurrent(path()));
|
|
14711
|
+
}
|
|
14664
14712
|
async reconnectPendingHandoffControlInMemory(result, record, clientSequence, fencingToken, reconnectGeneration) {
|
|
14665
14713
|
if (reconnectGeneration !== this.reconnectGeneration ||
|
|
14666
14714
|
this.reconnecting ||
|
package/lib/context.d.ts
CHANGED
|
@@ -23,6 +23,7 @@ export declare class PageContextCollector {
|
|
|
23
23
|
private mutationObserver;
|
|
24
24
|
private captureTimer;
|
|
25
25
|
private mutationCaptureTimer;
|
|
26
|
+
private mutationCaptureStartedAt;
|
|
26
27
|
private targetIds;
|
|
27
28
|
private pageScrollIds;
|
|
28
29
|
private navigationTargetBindings;
|
|
@@ -74,6 +75,7 @@ export declare class PageContextCollector {
|
|
|
74
75
|
private waitForScrollSettlement;
|
|
75
76
|
private actionApplied;
|
|
76
77
|
private clippedGeometry;
|
|
78
|
+
private clearMutationCapture;
|
|
77
79
|
private scheduleCapture;
|
|
78
80
|
private scheduleMutationCapture;
|
|
79
81
|
private scheduleViewportCapture;
|
package/lib/context.js
CHANGED
|
@@ -7,6 +7,7 @@ const MAX_SELECTED_TEXT = 2000;
|
|
|
7
7
|
const SELECTION_RETENTION_MS = 120000;
|
|
8
8
|
const CAPTURE_SETTLE_MS = 250;
|
|
9
9
|
const MUTATION_CAPTURE_DEBOUNCE_MS = 350;
|
|
10
|
+
const MUTATION_CAPTURE_MAX_WAIT_MS = 1000;
|
|
10
11
|
const SCROLL_SETTLEMENT_TIMEOUT_MS = 2000;
|
|
11
12
|
const SCROLL_SETTLEMENT_FRAME_FALLBACK_MS = 50;
|
|
12
13
|
const SCROLL_SETTLEMENT_POSITION_TOLERANCE_PX = 1;
|
|
@@ -194,10 +195,10 @@ const visibleRect = (element) => {
|
|
|
194
195
|
return right > left && bottom > top ? { left, top, right, bottom } : null;
|
|
195
196
|
};
|
|
196
197
|
const isVisible = (element) => visibleRect(element) !== null;
|
|
197
|
-
const
|
|
198
|
+
const interactionPoint = (element, trustedGuideRoot = null) => {
|
|
198
199
|
const rect = visibleRect(element);
|
|
199
200
|
if (!rect)
|
|
200
|
-
return
|
|
201
|
+
return null;
|
|
201
202
|
// Five bounded hit tests permit an exposed part of a link without treating
|
|
202
203
|
// an ancestor, empty hit, or a covering sibling as an interactable target.
|
|
203
204
|
const insetX = Math.min(1, (rect.right - rect.left) / 4);
|
|
@@ -207,13 +208,16 @@ const isCovered = (element, trustedGuideRoot = null) => {
|
|
|
207
208
|
[rect.left + insetX, rect.top + insetY], [rect.right - insetX, rect.top + insetY],
|
|
208
209
|
[rect.left + insetX, rect.bottom - insetY], [rect.right - insetX, rect.bottom - insetY],
|
|
209
210
|
];
|
|
210
|
-
|
|
211
|
+
for (const [x, y] of points) {
|
|
211
212
|
const top = trustedGuideRoot?.isConnected && typeof document.elementsFromPoint === 'function'
|
|
212
213
|
? document.elementsFromPoint(x, y).find(candidate => candidate !== trustedGuideRoot) ?? null
|
|
213
214
|
: document.elementFromPoint(x, y);
|
|
214
|
-
|
|
215
|
-
|
|
215
|
+
if (top !== null && (top === element || element.contains(top)))
|
|
216
|
+
return { x: x, y: y, hit: top };
|
|
217
|
+
}
|
|
218
|
+
return null;
|
|
216
219
|
};
|
|
220
|
+
const isCovered = (element, trustedGuideRoot = null) => interactionPoint(element, trustedGuideRoot) === null;
|
|
217
221
|
const isNavigationMenu = (element, role) => {
|
|
218
222
|
if (role !== 'button' || literalBooleanAttribute(element, 'aria-expanded') === undefined ||
|
|
219
223
|
element.hasAttribute('aria-pressed') || element.hasAttribute('aria-checked') ||
|
|
@@ -488,15 +492,17 @@ const checkedStateFor = (element) => {
|
|
|
488
492
|
}
|
|
489
493
|
return framerToggleDescriptor(element)?.checked;
|
|
490
494
|
};
|
|
491
|
-
const dispatchFramerPrimaryPointerSequence = (element) => {
|
|
495
|
+
const dispatchFramerPrimaryPointerSequence = (element, trustedGuideRoot = null) => {
|
|
492
496
|
if (typeof window.PointerEvent !== 'function') {
|
|
493
497
|
throw new Error('Framer pointer interaction is unavailable.');
|
|
494
498
|
}
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
const
|
|
498
|
-
|
|
499
|
-
|
|
499
|
+
// Capture and dispatch must agree on the exposed point and exact descendant,
|
|
500
|
+
// including when our own mobile surface covers a link's router-owned logo.
|
|
501
|
+
const point = interactionPoint(element, trustedGuideRoot);
|
|
502
|
+
if (!point)
|
|
503
|
+
throw new Error('The approved pointer target is covered.');
|
|
504
|
+
const { x: clientX, y: clientY } = point;
|
|
505
|
+
const eventTarget = element instanceof HTMLAnchorElement ? point.hit : element;
|
|
500
506
|
const nestedLink = eventTarget.closest('a[href]');
|
|
501
507
|
if (element instanceof HTMLAnchorElement && nestedLink instanceof HTMLAnchorElement && nestedLink !== element &&
|
|
502
508
|
(!sameNavigationDestination(nestedLink.href, element.href) || nestedLink.hasAttribute('download') ||
|
|
@@ -568,11 +574,11 @@ const dispatchFramerPrimaryPointerSequence = (element) => {
|
|
|
568
574
|
detail: 1,
|
|
569
575
|
}));
|
|
570
576
|
};
|
|
571
|
-
const dispatchFramerTogglePrimaryPointer = (element) => {
|
|
577
|
+
const dispatchFramerTogglePrimaryPointer = (element, trustedGuideRoot = null) => {
|
|
572
578
|
if (!framerToggleDescriptor(element)) {
|
|
573
579
|
throw new Error('Framer toggle pointer interaction is unavailable.');
|
|
574
580
|
}
|
|
575
|
-
dispatchFramerPrimaryPointerSequence(element);
|
|
581
|
+
dispatchFramerPrimaryPointerSequence(element, trustedGuideRoot);
|
|
576
582
|
};
|
|
577
583
|
const semanticSourceSensitivity = (source) => {
|
|
578
584
|
if (source.closest(PRIVATE_SELECTOR) || source.querySelector(PRIVATE_SELECTOR)) {
|
|
@@ -630,6 +636,18 @@ const safeOptionLabel = (option) => {
|
|
|
630
636
|
return sensitivityFor(option, label) === 'none' ? label : '';
|
|
631
637
|
};
|
|
632
638
|
const isActionTarget = (element) => element.matches('a[href],button,input,select,textarea,summary,form,[contenteditable="true"],[role="button"],[role="link"],[role="checkbox"],[role="radio"],[role="combobox"],[role="textbox"],[tabindex]');
|
|
639
|
+
const belongsToInteractiveControl = (element) => {
|
|
640
|
+
for (let current = element; current; current = current.parentElement) {
|
|
641
|
+
// Form copy can carry useful public instructions. Only controls and links
|
|
642
|
+
// already represented in targets are omitted from visible page text.
|
|
643
|
+
if (current.matches('form'))
|
|
644
|
+
continue;
|
|
645
|
+
// A bare tabindex is often a focusable Framer content card, not an action.
|
|
646
|
+
if (current.matches('a[href],button,input,select,textarea,summary,[contenteditable="true"],[role="button"],[role="link"],[role="checkbox"],[role="radio"],[role="combobox"],[role="textbox"]'))
|
|
647
|
+
return true;
|
|
648
|
+
}
|
|
649
|
+
return false;
|
|
650
|
+
};
|
|
633
651
|
const READONLY_INPUT_TYPES = new Set([
|
|
634
652
|
'date',
|
|
635
653
|
'datetime-local',
|
|
@@ -742,6 +760,7 @@ export class PageContextCollector {
|
|
|
742
760
|
this.mutationObserver = null;
|
|
743
761
|
this.captureTimer = null;
|
|
744
762
|
this.mutationCaptureTimer = null;
|
|
763
|
+
this.mutationCaptureStartedAt = null;
|
|
745
764
|
this.targetIds = new WeakMap();
|
|
746
765
|
this.pageScrollIds = new WeakMap();
|
|
747
766
|
this.navigationTargetBindings = new WeakMap();
|
|
@@ -769,9 +788,7 @@ export class PageContextCollector {
|
|
|
769
788
|
this.pendingCapture = true;
|
|
770
789
|
return;
|
|
771
790
|
}
|
|
772
|
-
|
|
773
|
-
clearTimeout(this.mutationCaptureTimer);
|
|
774
|
-
this.mutationCaptureTimer = null;
|
|
791
|
+
this.clearMutationCapture();
|
|
775
792
|
if (this.captureTimer)
|
|
776
793
|
return;
|
|
777
794
|
this.captureTimer = setTimeout(() => {
|
|
@@ -788,12 +805,19 @@ export class PageContextCollector {
|
|
|
788
805
|
}
|
|
789
806
|
if (this.captureTimer)
|
|
790
807
|
return;
|
|
808
|
+
// Decorative animation must not indefinitely hide newly available links.
|
|
809
|
+
// Retain trailing-edge batching, with at most one forced scan per second
|
|
810
|
+
// during continuous churn. Unchanged semantic snapshots are still deduped.
|
|
811
|
+
const now = performance.now();
|
|
812
|
+
this.mutationCaptureStartedAt ?? (this.mutationCaptureStartedAt = now);
|
|
813
|
+
const delay = Math.max(0, Math.min(MUTATION_CAPTURE_DEBOUNCE_MS, MUTATION_CAPTURE_MAX_WAIT_MS - (now - this.mutationCaptureStartedAt)));
|
|
791
814
|
if (this.mutationCaptureTimer)
|
|
792
815
|
clearTimeout(this.mutationCaptureTimer);
|
|
793
816
|
this.mutationCaptureTimer = setTimeout(() => {
|
|
794
817
|
this.mutationCaptureTimer = null;
|
|
818
|
+
this.mutationCaptureStartedAt = null;
|
|
795
819
|
void this.captureAndPublishIfChanged();
|
|
796
|
-
},
|
|
820
|
+
}, delay);
|
|
797
821
|
};
|
|
798
822
|
this.scheduleViewportCapture = () => {
|
|
799
823
|
this.scheduleMutationCapture();
|
|
@@ -866,9 +890,7 @@ export class PageContextCollector {
|
|
|
866
890
|
if (this.captureTimer)
|
|
867
891
|
clearTimeout(this.captureTimer);
|
|
868
892
|
this.captureTimer = null;
|
|
869
|
-
|
|
870
|
-
clearTimeout(this.mutationCaptureTimer);
|
|
871
|
-
this.mutationCaptureTimer = null;
|
|
893
|
+
this.clearMutationCapture();
|
|
872
894
|
if (this.releasedNavigationSettlement &&
|
|
873
895
|
window.location.href !== this.releasedNavigationSettlement.fromUrl) {
|
|
874
896
|
this.finishReleasedNavigationSettlement(sameNavigationDestination(window.location.href, this.releasedNavigationSettlement.toUrl)
|
|
@@ -999,9 +1021,7 @@ export class PageContextCollector {
|
|
|
999
1021
|
if (this.captureTimer)
|
|
1000
1022
|
clearTimeout(this.captureTimer);
|
|
1001
1023
|
this.captureTimer = null;
|
|
1002
|
-
|
|
1003
|
-
clearTimeout(this.mutationCaptureTimer);
|
|
1004
|
-
this.mutationCaptureTimer = null;
|
|
1024
|
+
this.clearMutationCapture();
|
|
1005
1025
|
window.removeEventListener('resize', this.scheduleViewportCapture);
|
|
1006
1026
|
document.removeEventListener('scroll', this.scheduleViewportCapture, true);
|
|
1007
1027
|
window.visualViewport?.removeEventListener('resize', this.scheduleViewportCapture);
|
|
@@ -1173,23 +1193,33 @@ export class PageContextCollector {
|
|
|
1173
1193
|
});
|
|
1174
1194
|
}
|
|
1175
1195
|
const visibleText = [];
|
|
1196
|
+
const capturedSemanticText = new Set();
|
|
1176
1197
|
let totalText = 0;
|
|
1177
1198
|
const walker = document.createTreeWalker(document.body ?? document.documentElement, NodeFilter.SHOW_TEXT);
|
|
1178
1199
|
let node = walker.nextNode();
|
|
1179
1200
|
while (node && visibleText.length < MAX_TEXT_ITEMS && totalText < MAX_TOTAL_TEXT) {
|
|
1180
1201
|
const parent = node.parentElement;
|
|
1181
|
-
const
|
|
1202
|
+
const semanticTextElement = parent?.closest('[role="img"][aria-label]') ?? null;
|
|
1203
|
+
const textElement = semanticTextElement && isRendered(semanticTextElement)
|
|
1204
|
+
? semanticTextElement
|
|
1205
|
+
: parent;
|
|
1206
|
+
const text = normalize(semanticTextElement?.getAttribute('aria-label') ?? node.textContent ?? '', 500);
|
|
1182
1207
|
if (parent &&
|
|
1208
|
+
textElement &&
|
|
1183
1209
|
text &&
|
|
1184
1210
|
!parent.closest(PRIVATE_SELECTOR) &&
|
|
1185
1211
|
!parent.closest('script,style,noscript,template,svg') &&
|
|
1186
1212
|
!parent.closest('input,select,textarea,option') &&
|
|
1187
|
-
|
|
1188
|
-
|
|
1213
|
+
(!semanticTextElement || !capturedSemanticText.has(semanticTextElement)) &&
|
|
1214
|
+
!belongsToInteractiveControl(textElement) &&
|
|
1215
|
+
isRendered(textElement) &&
|
|
1216
|
+
sensitivityFor(textElement, nameFor(textElement)) === 'none') {
|
|
1189
1217
|
const textId = this.textIds.get(node) ?? id('text');
|
|
1190
1218
|
this.textIds.set(node, textId);
|
|
1191
1219
|
visibleText.push({ id: textId, text, provenance: 'page' });
|
|
1192
1220
|
totalText += text.length;
|
|
1221
|
+
if (semanticTextElement)
|
|
1222
|
+
capturedSemanticText.add(semanticTextElement);
|
|
1193
1223
|
}
|
|
1194
1224
|
node = walker.nextNode();
|
|
1195
1225
|
}
|
|
@@ -1838,7 +1868,7 @@ export class PageContextCollector {
|
|
|
1838
1868
|
};
|
|
1839
1869
|
window.addEventListener('click', preventNativeNavigation);
|
|
1840
1870
|
try {
|
|
1841
|
-
dispatchFramerPrimaryPointerSequence(navigation.source);
|
|
1871
|
+
dispatchFramerPrimaryPointerSequence(navigation.source, this.trustedGuideRoot);
|
|
1842
1872
|
}
|
|
1843
1873
|
finally {
|
|
1844
1874
|
window.removeEventListener('click', preventNativeNavigation);
|
|
@@ -1920,7 +1950,7 @@ export class PageContextCollector {
|
|
|
1920
1950
|
if (desiredStates.length > 0 && desiredStates.every(Boolean))
|
|
1921
1951
|
return;
|
|
1922
1952
|
if (desiredChecked !== undefined && framerToggleDescriptor(element)) {
|
|
1923
|
-
dispatchFramerTogglePrimaryPointer(element);
|
|
1953
|
+
dispatchFramerTogglePrimaryPointer(element, this.trustedGuideRoot);
|
|
1924
1954
|
return;
|
|
1925
1955
|
}
|
|
1926
1956
|
if (element instanceof HTMLAnchorElement) {
|
|
@@ -2228,9 +2258,19 @@ export class PageContextCollector {
|
|
|
2228
2258
|
height,
|
|
2229
2259
|
};
|
|
2230
2260
|
}
|
|
2261
|
+
clearMutationCapture() {
|
|
2262
|
+
if (this.mutationCaptureTimer)
|
|
2263
|
+
clearTimeout(this.mutationCaptureTimer);
|
|
2264
|
+
this.mutationCaptureTimer = null;
|
|
2265
|
+
this.mutationCaptureStartedAt = null;
|
|
2266
|
+
}
|
|
2231
2267
|
async captureAndPublishIfChanged() {
|
|
2232
|
-
if (this.navigationSettlement)
|
|
2268
|
+
if (!this.running || this.navigationSettlement)
|
|
2269
|
+
return;
|
|
2270
|
+
if (this.executingActionId) {
|
|
2271
|
+
this.pendingCapture = true;
|
|
2233
2272
|
return;
|
|
2273
|
+
}
|
|
2234
2274
|
const snapshot = this.captureSnapshot(false);
|
|
2235
2275
|
if (snapshot && this.running)
|
|
2236
2276
|
await this.callbacks.onSnapshot(snapshot);
|
|
@@ -2315,9 +2355,7 @@ export class PageContextCollector {
|
|
|
2315
2355
|
if (this.captureTimer)
|
|
2316
2356
|
clearTimeout(this.captureTimer);
|
|
2317
2357
|
this.captureTimer = null;
|
|
2318
|
-
|
|
2319
|
-
clearTimeout(this.mutationCaptureTimer);
|
|
2320
|
-
this.mutationCaptureTimer = null;
|
|
2358
|
+
this.clearMutationCapture();
|
|
2321
2359
|
return settlement;
|
|
2322
2360
|
}
|
|
2323
2361
|
observeCausalNavigationAfterMutation(actionId, destinationPath, beforeActionUrl) {
|
package/lib/control.d.ts
CHANGED
|
@@ -24,6 +24,10 @@ export type ControlOutboundOperation = {
|
|
|
24
24
|
type: string;
|
|
25
25
|
};
|
|
26
26
|
export type ReservedControlOperation = Readonly<ControlOutboundOperation>;
|
|
27
|
+
type TransportRecovery = {
|
|
28
|
+
canRecover: () => boolean;
|
|
29
|
+
onRecovered: () => Promise<void>;
|
|
30
|
+
};
|
|
27
31
|
export declare class ControlChannel {
|
|
28
32
|
private readonly result;
|
|
29
33
|
private readonly tabId;
|
|
@@ -31,6 +35,7 @@ export declare class ControlChannel {
|
|
|
31
35
|
private readonly onDisconnect;
|
|
32
36
|
private readonly onClientSequence;
|
|
33
37
|
private readonly onUnsafeDisconnect;
|
|
38
|
+
private readonly transportRecovery?;
|
|
34
39
|
private socket;
|
|
35
40
|
private clientSequence;
|
|
36
41
|
private serverSequence;
|
|
@@ -54,7 +59,9 @@ export declare class ControlChannel {
|
|
|
54
59
|
private pendingConnection;
|
|
55
60
|
private helloOperation;
|
|
56
61
|
private fencedAuthorityConfirmed;
|
|
57
|
-
|
|
62
|
+
private transportRecoveryAttempted;
|
|
63
|
+
private recoveringTransport;
|
|
64
|
+
constructor(result: RuntimeAuthorization, tabId: string, clientVersion: string, onDisconnect: () => void, initialClientSequence?: number, onClientSequence?: (sequence: number) => void, onUnsafeDisconnect?: () => void, transportRecovery?: TransportRecovery | undefined);
|
|
58
65
|
get leaseToken(): number;
|
|
59
66
|
get lastClientSequence(): number;
|
|
60
67
|
get isOpen(): boolean;
|
|
@@ -78,6 +85,7 @@ export declare class ControlChannel {
|
|
|
78
85
|
private handleMessage;
|
|
79
86
|
private recoverFromSequenceGap;
|
|
80
87
|
private notifyUnsafeDisconnect;
|
|
88
|
+
private recoverTransport;
|
|
81
89
|
private enqueueHandlers;
|
|
82
90
|
private dispatchHandlers;
|
|
83
91
|
private handleHandlerFailure;
|
package/lib/control.js
CHANGED
|
@@ -3,6 +3,7 @@ const PROTOCOL_VIOLATION_CLOSE_CODE = 4008;
|
|
|
3
3
|
const SERVER_SEQUENCE_GAP_CLOSE_CODE = 4009;
|
|
4
4
|
const HANDLER_FAILURE_CLOSE_CODE = 4010;
|
|
5
5
|
const HANDLER_TIMEOUT_MS = 15000;
|
|
6
|
+
const TRANSPORT_RECOVERY_TIMEOUT_MS = 5000;
|
|
6
7
|
const MAX_PENDING_HANDLER_MESSAGES = 64;
|
|
7
8
|
const MAX_PENDING_STATEFUL_OPERATIONS = 64;
|
|
8
9
|
const MAX_SETTLED_OPERATION_HISTORY = 100;
|
|
@@ -35,13 +36,14 @@ const STATEFUL_CONTROL_TYPES = new Set([
|
|
|
35
36
|
]);
|
|
36
37
|
const outcomeKey = (messageId, sequence) => `${messageId}\u0000${sequence}`;
|
|
37
38
|
export class ControlChannel {
|
|
38
|
-
constructor(result, tabId, clientVersion, onDisconnect, initialClientSequence = 0, onClientSequence = () => undefined, onUnsafeDisconnect = () => undefined) {
|
|
39
|
+
constructor(result, tabId, clientVersion, onDisconnect, initialClientSequence = 0, onClientSequence = () => undefined, onUnsafeDisconnect = () => undefined, transportRecovery) {
|
|
39
40
|
this.result = result;
|
|
40
41
|
this.tabId = tabId;
|
|
41
42
|
this.clientVersion = clientVersion;
|
|
42
43
|
this.onDisconnect = onDisconnect;
|
|
43
44
|
this.onClientSequence = onClientSequence;
|
|
44
45
|
this.onUnsafeDisconnect = onUnsafeDisconnect;
|
|
46
|
+
this.transportRecovery = transportRecovery;
|
|
45
47
|
this.socket = null;
|
|
46
48
|
this.serverSequence = 0;
|
|
47
49
|
this.serverBaselineEstablished = false;
|
|
@@ -64,6 +66,8 @@ export class ControlChannel {
|
|
|
64
66
|
this.pendingConnection = null;
|
|
65
67
|
this.helloOperation = null;
|
|
66
68
|
this.fencedAuthorityConfirmed = false;
|
|
69
|
+
this.transportRecoveryAttempted = false;
|
|
70
|
+
this.recoveringTransport = false;
|
|
67
71
|
this.clientSequence =
|
|
68
72
|
Number.isSafeInteger(initialClientSequence) && initialClientSequence >= 0
|
|
69
73
|
? initialClientSequence
|
|
@@ -76,7 +80,7 @@ export class ControlChannel {
|
|
|
76
80
|
return this.clientSequence;
|
|
77
81
|
}
|
|
78
82
|
get isOpen() {
|
|
79
|
-
return this.socket?.readyState === WebSocket.OPEN;
|
|
83
|
+
return (!this.recoveringTransport && this.socket?.readyState === WebSocket.OPEN);
|
|
80
84
|
}
|
|
81
85
|
get connectionHelloOperation() {
|
|
82
86
|
return this.helloOperation ? { ...this.helloOperation } : null;
|
|
@@ -177,10 +181,19 @@ export class ControlChannel {
|
|
|
177
181
|
this.rejectPendingConnection(socket, generation, new Error("The AI Guide control channel could not connect."));
|
|
178
182
|
}
|
|
179
183
|
}, { once: true });
|
|
180
|
-
socket.addEventListener("close", () => {
|
|
184
|
+
socket.addEventListener("close", (event) => {
|
|
181
185
|
if (!this.isCurrentSocket(socket, generation))
|
|
182
186
|
return;
|
|
183
187
|
const wasIntentionallyClosed = this.intentionallyClosed;
|
|
188
|
+
// A settled transport loss is not a new session. Keep its provider
|
|
189
|
+
// connection alive for one bounded retry, but never carry ambiguous
|
|
190
|
+
// browser writes, an active handler, or a protocol failure across it.
|
|
191
|
+
const canRecover = event.code === 1006 &&
|
|
192
|
+
this.fencedAuthorityConfirmed &&
|
|
193
|
+
!this.transportRecoveryAttempted &&
|
|
194
|
+
!this.sequenceGapDetected &&
|
|
195
|
+
this.pendingHandlerMessages === 0 &&
|
|
196
|
+
this.unsettledStatefulOperations.size === 0;
|
|
184
197
|
this.socket = null;
|
|
185
198
|
this.socketGeneration += 1;
|
|
186
199
|
this.stopHeartbeat();
|
|
@@ -194,12 +207,24 @@ export class ControlChannel {
|
|
|
194
207
|
this.pendingHandlerMessages = 0;
|
|
195
208
|
this.rejectPendingConnection(socket, generation, new Error("The AI Guide control channel closed before connecting."));
|
|
196
209
|
if (!wasIntentionallyClosed && !signal.aborted) {
|
|
210
|
+
try {
|
|
211
|
+
if (canRecover && this.transportRecovery?.canRecover()) {
|
|
212
|
+
void this.recoverTransport(signal);
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
/* A failed eligibility check uses full recovery. */
|
|
218
|
+
}
|
|
197
219
|
this.notifyUnsafeDisconnect();
|
|
198
220
|
}
|
|
199
221
|
});
|
|
200
222
|
});
|
|
201
223
|
}
|
|
202
224
|
send(type, payload, stableMessageId) {
|
|
225
|
+
if (this.recoveringTransport && type !== "hello" && type !== "heartbeat") {
|
|
226
|
+
throw new Error("The AI Guide control channel is reconnecting.");
|
|
227
|
+
}
|
|
203
228
|
if (!this.socket || this.socket.readyState !== WebSocket.OPEN) {
|
|
204
229
|
throw new Error("The AI Guide control channel is not connected.");
|
|
205
230
|
}
|
|
@@ -510,6 +535,59 @@ export class ControlChannel {
|
|
|
510
535
|
this.onDisconnect();
|
|
511
536
|
}
|
|
512
537
|
}
|
|
538
|
+
async recoverTransport(signal) {
|
|
539
|
+
this.transportRecoveryAttempted = true;
|
|
540
|
+
this.recoveringTransport = true;
|
|
541
|
+
const controller = new AbortController();
|
|
542
|
+
const abort = () => controller.abort();
|
|
543
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
544
|
+
let timeout;
|
|
545
|
+
const deadline = new Promise((_resolve, reject) => {
|
|
546
|
+
timeout = setTimeout(() => {
|
|
547
|
+
controller.abort();
|
|
548
|
+
reject(new Error("The control reconnect timed out."));
|
|
549
|
+
}, TRANSPORT_RECOVERY_TIMEOUT_MS);
|
|
550
|
+
});
|
|
551
|
+
const fencingToken = this.fencingToken;
|
|
552
|
+
let recovered = false;
|
|
553
|
+
try {
|
|
554
|
+
await Promise.race([
|
|
555
|
+
deadline,
|
|
556
|
+
(async () => {
|
|
557
|
+
if (signal.aborted)
|
|
558
|
+
throw new Error("The control connection was retired.");
|
|
559
|
+
await this.connect(controller.signal, true);
|
|
560
|
+
const hello = await this.waitForHelloOutcome(2000);
|
|
561
|
+
if (hello.status !== "ack" || controller.signal.aborted) {
|
|
562
|
+
throw new Error("The control reconnect was not acknowledged.");
|
|
563
|
+
}
|
|
564
|
+
const authority = await this.proveFencedAuthority(2000);
|
|
565
|
+
if (authority.status !== "ack" ||
|
|
566
|
+
controller.signal.aborted ||
|
|
567
|
+
this.fencingToken !== fencingToken ||
|
|
568
|
+
!this.transportRecovery?.canRecover()) {
|
|
569
|
+
throw new Error("The control reconnect no longer owns this session.");
|
|
570
|
+
}
|
|
571
|
+
this.recoveringTransport = false;
|
|
572
|
+
await this.transportRecovery.onRecovered();
|
|
573
|
+
recovered = !controller.signal.aborted && this.hasFencedAuthority;
|
|
574
|
+
})(),
|
|
575
|
+
]);
|
|
576
|
+
}
|
|
577
|
+
catch {
|
|
578
|
+
// The existing full-resume path revokes local executable authority and
|
|
579
|
+
// handles expired credentials, changed ownership, and longer outages.
|
|
580
|
+
}
|
|
581
|
+
finally {
|
|
582
|
+
clearTimeout(timeout);
|
|
583
|
+
signal.removeEventListener("abort", abort);
|
|
584
|
+
this.recoveringTransport = false;
|
|
585
|
+
if (!recovered && !signal.aborted && !this.intentionallyClosed) {
|
|
586
|
+
this.retireSocket("control reconnect failed");
|
|
587
|
+
this.notifyUnsafeDisconnect();
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
}
|
|
513
591
|
enqueueHandlers(message, socket, generation) {
|
|
514
592
|
const isCurrent = () => this.isMessageSourceCurrent(socket, generation);
|
|
515
593
|
if (!isCurrent())
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type LiveDuplexMediaOptions, type LiveDuplexMediaDependencies } from './liveDuplexMedia.js';
|
|
2
|
-
export declare const LIVE_CONTROL_CLIENT_VERSION = "7.5.
|
|
2
|
+
export declare const LIVE_CONTROL_CLIENT_VERSION = "7.5.8";
|
|
3
3
|
export declare const LIVE_CONTROL_CLIENT_CAPABILITIES: readonly ["live_duplex_v1", "live_local_wake_v1", "live_fragment_captions_v1"];
|
|
4
4
|
declare const phases: readonly ["voice_off", "local_listening", "wake_buffering", "starting", "active", "tentative_idle", "closing", "uncertain", "muted", "suspended", "budget_blocked", "ended"];
|
|
5
5
|
export type LiveControlState = {
|
package/lib/media.d.ts
CHANGED
|
@@ -36,6 +36,7 @@ export declare class RealtimeMedia {
|
|
|
36
36
|
connect(result: LaunchResult, mode: AssistMode, signal: AbortSignal, offerIdempotencyKey?: string, expectedSessionState?: "active_text" | "active_voice"): Promise<number>;
|
|
37
37
|
setMuted(muted: boolean): void;
|
|
38
38
|
get isMuted(): boolean;
|
|
39
|
+
get isConnected(): boolean;
|
|
39
40
|
setAssistantPlaybackAudible(audible: boolean): void;
|
|
40
41
|
setManualTurn(manual: boolean): void;
|
|
41
42
|
beginPushToTalk(): void;
|
package/lib/media.js
CHANGED
|
@@ -134,6 +134,9 @@ export class RealtimeMedia {
|
|
|
134
134
|
get isMuted() {
|
|
135
135
|
return this.muted;
|
|
136
136
|
}
|
|
137
|
+
get isConnected() {
|
|
138
|
+
return !this.closed && this.connection?.connectionState === "connected";
|
|
139
|
+
}
|
|
137
140
|
setAssistantPlaybackAudible(audible) {
|
|
138
141
|
if (this.closed)
|
|
139
142
|
return;
|
package/lib/ui.js
CHANGED
|
@@ -6790,7 +6790,9 @@ export class WidgetView {
|
|
|
6790
6790
|
this.setActionStateRunning(status === "running");
|
|
6791
6791
|
if (this.actionStateRunning) {
|
|
6792
6792
|
this.clearAssistantReadyVisualTimer();
|
|
6793
|
+
const previousState = this.assistantState;
|
|
6793
6794
|
this.assistantState = "thinking";
|
|
6795
|
+
this.updateRemoteAudioEvidenceExpectation(previousState, "thinking");
|
|
6794
6796
|
const assistantCaption = this.captions.querySelector('[data-caption="assistant-partial"]');
|
|
6795
6797
|
if (!assistantCaption || assistantCaption.classList.contains("pending")) {
|
|
6796
6798
|
this.ensurePendingCaption("assistant", this.messages.thinking);
|
package/lib/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const VERSION = "7.5.
|
|
1
|
+
export declare const VERSION = "7.5.8";
|
package/lib/version.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const VERSION = '7.5.
|
|
1
|
+
export const VERSION = '7.5.8';
|