@fenixforce/edition-mobile 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +230 -0
- package/dist/lib/agent.d.ts +44 -0
- package/dist/lib/app-context.d.ts +31 -0
- package/dist/lib/boot.d.ts +22 -0
- package/dist/lib/entry.d.ts +48 -0
- package/dist/lib/providers/hybrid-router.d.ts +60 -0
- package/dist/lib/providers/local-inference.d.ts +31 -0
- package/dist/lib/storage/identity-store.d.ts +13 -0
- package/dist/lib/storage/memory-search.d.ts +30 -0
- package/dist/lib/storage/sqlite-adapter.d.ts +22 -0
- package/dist/lib/sync/client.d.ts +81 -0
- package/dist/lib/sync/triggers.d.ts +59 -0
- package/dist/lib/tools/calendar.d.ts +76 -0
- package/dist/lib/tools/camera.d.ts +40 -0
- package/dist/lib/tools/contacts.d.ts +59 -0
- package/dist/lib/tools/location.d.ts +58 -0
- package/dist/lib/tools/notifications.d.ts +79 -0
- package/dist/lib/tools/registry.d.ts +26 -0
- package/dist/lib/tools/reminders.d.ts +65 -0
- package/dist/platforms/android-xr/XRAgentService.d.ts +27 -0
- package/dist/platforms/android-xr/XRContextCapture.d.ts +47 -0
- package/dist/platforms/android-xr/XRGestureHandler.d.ts +41 -0
- package/dist/platforms/android-xr/XRNotificationBridge.d.ts +35 -0
- package/dist/platforms/android-xr/XRVoiceInterface.d.ts +72 -0
- package/dist/platforms/android-xr/display/ConversationOverlay.d.ts +20 -0
- package/dist/platforms/android-xr/display/DisplayManager.d.ts +44 -0
- package/dist/platforms/android-xr/display/GlanceCard.d.ts +21 -0
- package/dist/platforms/android-xr/display/MemoryHUD.d.ts +22 -0
- package/dist/platforms/android-xr/display/NotificationPill.d.ts +13 -0
- package/dist/platforms/android-xr/xr-entry.d.ts +23 -0
- package/dist/platforms/quest/QuestAdapter.d.ts +66 -0
- package/dist/platforms/quest/QuestPanelConfig.d.ts +45 -0
- package/dist/platforms/quest/quest-entry.d.ts +29 -0
- package/dist/platforms/visionos/AgentBridge.d.ts +39 -0
- package/dist/platforms/visionos/VisionToolAdapters.d.ts +53 -0
- package/dist/platforms/visionos/VisionVoiceInterface.d.ts +32 -0
- package/dist/platforms/visionos/visionos-entry.d.ts +24 -0
- package/dist/src/hil/push-notifier.d.ts +25 -0
- package/dist/src/index.d.ts +72 -0
- package/dist/watch/WatchBridge.d.ts +63 -0
- package/package.json +51 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* XR Notification Bridge — routes system notifications through
|
|
3
|
+
* the agent for contextual awareness and spoken summaries.
|
|
4
|
+
*/
|
|
5
|
+
export interface SystemNotification {
|
|
6
|
+
id: string;
|
|
7
|
+
app: string;
|
|
8
|
+
title: string;
|
|
9
|
+
body: string;
|
|
10
|
+
timestamp: string;
|
|
11
|
+
priority: "low" | "default" | "high" | "urgent";
|
|
12
|
+
}
|
|
13
|
+
export interface NotificationBridgeConfig {
|
|
14
|
+
/** Called when a notification should be spoken. */
|
|
15
|
+
onSpeak?: (text: string) => Promise<void>;
|
|
16
|
+
/** Called when a notification should be displayed. */
|
|
17
|
+
onDisplay?: (notification: SystemNotification) => void;
|
|
18
|
+
/** Filter: which apps' notifications to process. Null = all. */
|
|
19
|
+
allowedApps?: string[] | null;
|
|
20
|
+
/** Summarize multiple notifications into one spoken message. */
|
|
21
|
+
summarize?: (notifications: SystemNotification[]) => Promise<string>;
|
|
22
|
+
}
|
|
23
|
+
export interface NotificationBridge {
|
|
24
|
+
/** Process an incoming system notification. */
|
|
25
|
+
handleNotification(notification: SystemNotification): Promise<void>;
|
|
26
|
+
/** Get unread notifications. */
|
|
27
|
+
getUnread(): SystemNotification[];
|
|
28
|
+
/** Mark a notification as read. */
|
|
29
|
+
markRead(id: string): void;
|
|
30
|
+
/** Clear all notifications. */
|
|
31
|
+
clearAll(): void;
|
|
32
|
+
/** Speak a summary of pending notifications. */
|
|
33
|
+
speakSummary(): Promise<void>;
|
|
34
|
+
}
|
|
35
|
+
export declare function createNotificationBridge(config: NotificationBridgeConfig): NotificationBridge;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* XR Voice Interface — continuous voice I/O pipeline for Android XR glasses.
|
|
3
|
+
* STT → Agent Loop → TTS. Supports wake word ("Hey Fenix") and push-to-talk.
|
|
4
|
+
*/
|
|
5
|
+
export interface STTProvider {
|
|
6
|
+
/** Transcribe audio buffer to text. */
|
|
7
|
+
transcribe(audio: AudioChunk): Promise<string>;
|
|
8
|
+
/** Start continuous listening (streaming mode). */
|
|
9
|
+
startListening(onPartial: (text: string) => void): Promise<void>;
|
|
10
|
+
/** Stop listening. */
|
|
11
|
+
stopListening(): Promise<string>;
|
|
12
|
+
/** Whether this provider works offline. */
|
|
13
|
+
isOffline: boolean;
|
|
14
|
+
}
|
|
15
|
+
export interface TTSProvider {
|
|
16
|
+
/** Convert text to audio and play through speakers. */
|
|
17
|
+
speak(text: string, options?: TTSOptions): Promise<void>;
|
|
18
|
+
/** Stop any ongoing speech. */
|
|
19
|
+
stop(): void;
|
|
20
|
+
/** Whether this provider works offline. */
|
|
21
|
+
isOffline: boolean;
|
|
22
|
+
}
|
|
23
|
+
export interface AudioChunk {
|
|
24
|
+
data: string;
|
|
25
|
+
sampleRate: number;
|
|
26
|
+
channels: number;
|
|
27
|
+
duration: number;
|
|
28
|
+
}
|
|
29
|
+
export interface TTSOptions {
|
|
30
|
+
/** Speech rate (0.5-2.0). Default 1.0. */
|
|
31
|
+
rate?: number;
|
|
32
|
+
/** Voice pitch (0.5-2.0). Default 1.0. */
|
|
33
|
+
pitch?: number;
|
|
34
|
+
/** Spatial audio position (for glasses with spatial audio). */
|
|
35
|
+
spatialPosition?: {
|
|
36
|
+
x: number;
|
|
37
|
+
y: number;
|
|
38
|
+
z: number;
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
export interface WakeWordDetector {
|
|
42
|
+
/** Start listening for the wake word. */
|
|
43
|
+
start(onDetected: () => void): void;
|
|
44
|
+
/** Stop wake word detection. */
|
|
45
|
+
stop(): void;
|
|
46
|
+
/** Whether currently listening. */
|
|
47
|
+
isActive(): boolean;
|
|
48
|
+
}
|
|
49
|
+
export interface VoiceInterfaceConfig {
|
|
50
|
+
sttProvider: STTProvider;
|
|
51
|
+
ttsProvider: TTSProvider;
|
|
52
|
+
wakeWordDetector?: WakeWordDetector;
|
|
53
|
+
onTranscription: (text: string) => Promise<string>;
|
|
54
|
+
/** Offline fallback STT (e.g. on-device Whisper). */
|
|
55
|
+
offlineSTT?: STTProvider;
|
|
56
|
+
/** Offline fallback TTS (e.g. edge TTS model). */
|
|
57
|
+
offlineTTS?: TTSProvider;
|
|
58
|
+
isOnline?: () => boolean;
|
|
59
|
+
}
|
|
60
|
+
export interface VoiceInterface {
|
|
61
|
+
/** Start the voice pipeline (wake word + push-to-talk ready). */
|
|
62
|
+
start(): void;
|
|
63
|
+
/** Stop everything. */
|
|
64
|
+
stop(): void;
|
|
65
|
+
/** Manually trigger listening (push-to-talk). */
|
|
66
|
+
pushToTalk(): Promise<string>;
|
|
67
|
+
/** Speak a response through TTS. */
|
|
68
|
+
speak(text: string): Promise<void>;
|
|
69
|
+
/** Whether currently processing voice. */
|
|
70
|
+
isBusy(): boolean;
|
|
71
|
+
}
|
|
72
|
+
export declare function createVoiceInterface(config: VoiceInterfaceConfig): VoiceInterface;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ConversationOverlay — scrollable conversation for binocular displays.
|
|
3
|
+
* Anchored in space, supports extended reading.
|
|
4
|
+
*/
|
|
5
|
+
export interface OverlayMessage {
|
|
6
|
+
id: string;
|
|
7
|
+
role: "user" | "assistant";
|
|
8
|
+
content: string;
|
|
9
|
+
timestamp: string;
|
|
10
|
+
}
|
|
11
|
+
export interface ConversationOverlayState {
|
|
12
|
+
messages: OverlayMessage[];
|
|
13
|
+
visible: boolean;
|
|
14
|
+
scrollOffset: number;
|
|
15
|
+
maxVisibleMessages: number;
|
|
16
|
+
}
|
|
17
|
+
export declare function createConversationOverlay(maxVisible?: number): ConversationOverlayState;
|
|
18
|
+
export declare function appendMessage(state: ConversationOverlayState, role: "user" | "assistant", content: string): ConversationOverlayState;
|
|
19
|
+
export declare function scrollOverlay(state: ConversationOverlayState, direction: "up" | "down"): ConversationOverlayState;
|
|
20
|
+
export declare function getVisibleMessages(state: ConversationOverlayState): OverlayMessage[];
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Display Manager — layout manager for Android XR glasses displays.
|
|
3
|
+
* Adapts output to audio-only, monocular, or binocular display tiers.
|
|
4
|
+
*/
|
|
5
|
+
import { type GlanceCardData } from "./GlanceCard.js";
|
|
6
|
+
import { type NotificationPillData } from "./NotificationPill.js";
|
|
7
|
+
import { type ConversationOverlayState } from "./ConversationOverlay.js";
|
|
8
|
+
import { type MemoryHUDState } from "./MemoryHUD.js";
|
|
9
|
+
export type DisplayType = "none" | "monocular" | "binocular";
|
|
10
|
+
export interface ResponseMetadata {
|
|
11
|
+
conversationId?: string;
|
|
12
|
+
toolsUsed?: string[];
|
|
13
|
+
}
|
|
14
|
+
export interface DisplayAdapter {
|
|
15
|
+
showResponse(text: string, metadata?: ResponseMetadata): void;
|
|
16
|
+
showNotification(title: string, body: string, priority: "low" | "high"): void;
|
|
17
|
+
showMemoryContext(facts: Array<{
|
|
18
|
+
id: string;
|
|
19
|
+
content: string;
|
|
20
|
+
confidence: number;
|
|
21
|
+
}>): void;
|
|
22
|
+
dismiss(): void;
|
|
23
|
+
getDisplayType(): DisplayType;
|
|
24
|
+
/** Get current display state for rendering. */
|
|
25
|
+
getState(): DisplayState;
|
|
26
|
+
}
|
|
27
|
+
export interface DisplayState {
|
|
28
|
+
currentCard: GlanceCardData | null;
|
|
29
|
+
notifications: NotificationPillData[];
|
|
30
|
+
conversation: ConversationOverlayState;
|
|
31
|
+
memory: MemoryHUDState;
|
|
32
|
+
}
|
|
33
|
+
export interface DisplayManagerConfig {
|
|
34
|
+
displayType: DisplayType;
|
|
35
|
+
/** Compress long responses. Null = truncate only. */
|
|
36
|
+
compressResponse?: (text: string, maxWords: number) => Promise<string>;
|
|
37
|
+
/** Word budget for monocular. Default 50. */
|
|
38
|
+
monocularWordBudget?: number;
|
|
39
|
+
/** Word budget for binocular. Default 150. */
|
|
40
|
+
binocularWordBudget?: number;
|
|
41
|
+
/** Called when TTS should speak (for audio-only or supplement). */
|
|
42
|
+
onSpeak?: (text: string) => Promise<void>;
|
|
43
|
+
}
|
|
44
|
+
export declare function createDisplayManager(config: DisplayManagerConfig): DisplayAdapter;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GlanceCard — compact response card for monocular XR displays.
|
|
3
|
+
* Title + 2-3 lines of text, auto-dismiss after timeout.
|
|
4
|
+
*/
|
|
5
|
+
export interface GlanceCardData {
|
|
6
|
+
id: string;
|
|
7
|
+
title: string;
|
|
8
|
+
body: string;
|
|
9
|
+
timestamp: string;
|
|
10
|
+
pinned: boolean;
|
|
11
|
+
autoDismissMs: number;
|
|
12
|
+
}
|
|
13
|
+
export interface GlanceCardConfig {
|
|
14
|
+
/** Max body length in characters for monocular. Default 120. */
|
|
15
|
+
maxBodyLength?: number;
|
|
16
|
+
/** Auto-dismiss timeout in ms. Default 8000. */
|
|
17
|
+
autoDismissMs?: number;
|
|
18
|
+
}
|
|
19
|
+
export declare function createGlanceCard(title: string, body: string, config?: GlanceCardConfig): GlanceCardData;
|
|
20
|
+
export declare function pinCard(card: GlanceCardData): GlanceCardData;
|
|
21
|
+
export declare function unpinCard(card: GlanceCardData): GlanceCardData;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MemoryHUD — "I remember..." context display for XR glasses.
|
|
3
|
+
* Shows relevant memory facts as floating contextual annotations.
|
|
4
|
+
*/
|
|
5
|
+
export interface MemoryAnnotation {
|
|
6
|
+
id: string;
|
|
7
|
+
content: string;
|
|
8
|
+
relevance: number;
|
|
9
|
+
timestamp: string;
|
|
10
|
+
}
|
|
11
|
+
export interface MemoryHUDState {
|
|
12
|
+
annotations: MemoryAnnotation[];
|
|
13
|
+
visible: boolean;
|
|
14
|
+
maxAnnotations: number;
|
|
15
|
+
}
|
|
16
|
+
export declare function createMemoryHUD(maxAnnotations?: number): MemoryHUDState;
|
|
17
|
+
export declare function updateAnnotations(state: MemoryHUDState, facts: Array<{
|
|
18
|
+
id: string;
|
|
19
|
+
content: string;
|
|
20
|
+
confidence: number;
|
|
21
|
+
}>): MemoryHUDState;
|
|
22
|
+
export declare function dismissHUD(state: MemoryHUDState): MemoryHUDState;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NotificationPill — minimal pill overlay for alerts.
|
|
3
|
+
* Shows app icon + short text, auto-dismiss.
|
|
4
|
+
*/
|
|
5
|
+
export interface NotificationPillData {
|
|
6
|
+
id: string;
|
|
7
|
+
app: string;
|
|
8
|
+
text: string;
|
|
9
|
+
priority: "low" | "high";
|
|
10
|
+
timestamp: string;
|
|
11
|
+
autoDismissMs: number;
|
|
12
|
+
}
|
|
13
|
+
export declare function createNotificationPill(app: string, text: string, priority?: "low" | "high"): NotificationPillData;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Android XR Entry Point — initializes Fenix for Android XR glasses.
|
|
3
|
+
* Wires the standard mobile entry point with XR-specific services
|
|
4
|
+
* (voice, vision, gestures, notifications, display).
|
|
5
|
+
*/
|
|
6
|
+
import { type FenixMobileConfig, type FenixMobileContext } from "../../lib/entry.js";
|
|
7
|
+
import { type XRAgentService } from "./XRAgentService.js";
|
|
8
|
+
import type { VoiceInterfaceConfig } from "./XRVoiceInterface.js";
|
|
9
|
+
import type { ContextCaptureConfig } from "./XRContextCapture.js";
|
|
10
|
+
import type { GestureHandlerConfig } from "./XRGestureHandler.js";
|
|
11
|
+
import type { NotificationBridgeConfig } from "./XRNotificationBridge.js";
|
|
12
|
+
export interface AndroidXRConfig extends FenixMobileConfig {
|
|
13
|
+
voice: VoiceInterfaceConfig;
|
|
14
|
+
context: ContextCaptureConfig;
|
|
15
|
+
gesture?: GestureHandlerConfig;
|
|
16
|
+
notification?: NotificationBridgeConfig;
|
|
17
|
+
}
|
|
18
|
+
export interface AndroidXRContext {
|
|
19
|
+
fenix: FenixMobileContext;
|
|
20
|
+
xr: XRAgentService;
|
|
21
|
+
dispose(): void;
|
|
22
|
+
}
|
|
23
|
+
export declare function initializeAndroidXR(config: AndroidXRConfig): Promise<AndroidXRContext>;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Quest Adapter — voice input and controller mapping for Meta Quest.
|
|
3
|
+
*
|
|
4
|
+
* Quest controllers provide:
|
|
5
|
+
* - Trigger (index finger): mapped to "send message"
|
|
6
|
+
* - Grip (middle finger): mapped to "push to talk" for voice
|
|
7
|
+
* - Thumbstick click: mapped to "cancel/back"
|
|
8
|
+
* - Standard touch/pointer: works automatically on 2D panels
|
|
9
|
+
*
|
|
10
|
+
* Voice is captured via standard Android audio APIs (RECORD_AUDIO permission).
|
|
11
|
+
*/
|
|
12
|
+
export type ControllerButton = "trigger" | "grip" | "thumbstick_click" | "a" | "b" | "x" | "y";
|
|
13
|
+
export type ControllerHand = "left" | "right";
|
|
14
|
+
export interface ControllerEvent {
|
|
15
|
+
button: ControllerButton;
|
|
16
|
+
hand: ControllerHand;
|
|
17
|
+
pressed: boolean;
|
|
18
|
+
timestamp: number;
|
|
19
|
+
}
|
|
20
|
+
export interface ControllerMapping {
|
|
21
|
+
/** Action when trigger is pressed. */
|
|
22
|
+
trigger: "send_message" | "confirm" | "none";
|
|
23
|
+
/** Action when grip is held. */
|
|
24
|
+
grip: "push_to_talk" | "none";
|
|
25
|
+
/** Action when thumbstick is clicked. */
|
|
26
|
+
thumbstick_click: "cancel" | "back" | "none";
|
|
27
|
+
}
|
|
28
|
+
export interface VoiceCapture {
|
|
29
|
+
/** Start recording audio from the Quest microphone. */
|
|
30
|
+
startRecording(): Promise<void>;
|
|
31
|
+
/** Stop recording and return the audio buffer. */
|
|
32
|
+
stopRecording(): Promise<AudioBuffer>;
|
|
33
|
+
/** Check if recording is in progress. */
|
|
34
|
+
isRecording(): boolean;
|
|
35
|
+
}
|
|
36
|
+
export interface AudioBuffer {
|
|
37
|
+
/** PCM audio data as base64 string. */
|
|
38
|
+
data: string;
|
|
39
|
+
/** Sample rate in Hz. */
|
|
40
|
+
sampleRate: number;
|
|
41
|
+
/** Duration in seconds. */
|
|
42
|
+
duration: number;
|
|
43
|
+
}
|
|
44
|
+
export interface QuestAdapterConfig {
|
|
45
|
+
/** Controller button mapping. Uses defaults if not provided. */
|
|
46
|
+
mapping?: Partial<ControllerMapping>;
|
|
47
|
+
/** Voice capture implementation (injected from native module). */
|
|
48
|
+
voiceCapture?: VoiceCapture;
|
|
49
|
+
/** Called when a controller action is triggered. */
|
|
50
|
+
onAction?: (action: string, event: ControllerEvent) => void;
|
|
51
|
+
}
|
|
52
|
+
export interface QuestAdapter {
|
|
53
|
+
/** Process a controller event and dispatch the mapped action. */
|
|
54
|
+
handleControllerEvent(event: ControllerEvent): string | null;
|
|
55
|
+
/** Start push-to-talk recording. */
|
|
56
|
+
startVoice(): Promise<void>;
|
|
57
|
+
/** Stop push-to-talk and return audio. */
|
|
58
|
+
stopVoice(): Promise<AudioBuffer | null>;
|
|
59
|
+
/** Check if currently recording voice. */
|
|
60
|
+
isRecordingVoice(): boolean;
|
|
61
|
+
/** Get the current controller mapping. */
|
|
62
|
+
getMapping(): ControllerMapping;
|
|
63
|
+
/** Update controller mapping. */
|
|
64
|
+
setMapping(mapping: Partial<ControllerMapping>): void;
|
|
65
|
+
}
|
|
66
|
+
export declare function createQuestAdapter(config?: QuestAdapterConfig): QuestAdapter;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Quest Panel Configuration — dimensions, placement, and display
|
|
3
|
+
* settings for the Fenix 2D panel in Meta Horizon OS.
|
|
4
|
+
*
|
|
5
|
+
* Quest renders standard Android apps as floating 2D panels.
|
|
6
|
+
* These settings configure the default panel appearance.
|
|
7
|
+
*/
|
|
8
|
+
export interface PanelConfig {
|
|
9
|
+
/** Logical pixel width of the panel. Default 1024. */
|
|
10
|
+
width: number;
|
|
11
|
+
/** Logical pixel height of the panel. Default 768. */
|
|
12
|
+
height: number;
|
|
13
|
+
/** Panel background opacity (0-1). Default 0.95 (slight transparency). */
|
|
14
|
+
backgroundOpacity: number;
|
|
15
|
+
/** Font scale multiplier for VR readability. Default 1.2. */
|
|
16
|
+
fontScale: number;
|
|
17
|
+
/** Minimum touch target size in dp (larger for controller pointer). Default 48. */
|
|
18
|
+
minTouchTarget: number;
|
|
19
|
+
/** Whether to show the system keyboard on text input focus. Default true. */
|
|
20
|
+
showSystemKeyboard: boolean;
|
|
21
|
+
}
|
|
22
|
+
export declare const DEFAULT_PANEL_CONFIG: PanelConfig;
|
|
23
|
+
/**
|
|
24
|
+
* Compute a style override object for React Native StyleSheet
|
|
25
|
+
* that adjusts text sizes and touch targets for Quest readability.
|
|
26
|
+
*/
|
|
27
|
+
export declare function getQuestStyleOverrides(config?: PanelConfig): {
|
|
28
|
+
/** Base font size multiplied by scale factor. */
|
|
29
|
+
baseFontSize: number;
|
|
30
|
+
/** Heading font size. */
|
|
31
|
+
headingFontSize: number;
|
|
32
|
+
/** Minimum touchable area dimension. */
|
|
33
|
+
minTouchTarget: number;
|
|
34
|
+
/** Panel dimensions. */
|
|
35
|
+
panelWidth: number;
|
|
36
|
+
panelHeight: number;
|
|
37
|
+
/** Background opacity for the main container. */
|
|
38
|
+
backgroundOpacity: number;
|
|
39
|
+
};
|
|
40
|
+
/**
|
|
41
|
+
* Check if the app is running inside a Quest headset.
|
|
42
|
+
* Uses the Android Build.MANUFACTURER check.
|
|
43
|
+
*/
|
|
44
|
+
export declare function isQuestDevice(): boolean;
|
|
45
|
+
export declare function setIsQuestDevice(value: boolean): void;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Quest-specific initialization — called instead of the standard
|
|
3
|
+
* mobile entry point when running on a Meta Quest headset.
|
|
4
|
+
*
|
|
5
|
+
* Applies Quest panel configuration and controller input mapping
|
|
6
|
+
* on top of the standard Fenix Mobile initialization.
|
|
7
|
+
*/
|
|
8
|
+
import { type FenixMobileConfig, type FenixMobileContext } from "../../lib/entry.js";
|
|
9
|
+
import { type QuestAdapter, type QuestAdapterConfig } from "./QuestAdapter.js";
|
|
10
|
+
import { getQuestStyleOverrides, type PanelConfig } from "./QuestPanelConfig.js";
|
|
11
|
+
export interface QuestConfig extends FenixMobileConfig {
|
|
12
|
+
/** Quest panel configuration overrides. */
|
|
13
|
+
panel?: Partial<PanelConfig>;
|
|
14
|
+
/** Quest controller adapter configuration. */
|
|
15
|
+
controller?: QuestAdapterConfig;
|
|
16
|
+
}
|
|
17
|
+
export interface QuestContext {
|
|
18
|
+
/** Standard Fenix Mobile context. */
|
|
19
|
+
fenix: FenixMobileContext;
|
|
20
|
+
/** Quest controller adapter. */
|
|
21
|
+
controller: QuestAdapter;
|
|
22
|
+
/** Quest panel style overrides. */
|
|
23
|
+
styles: ReturnType<typeof getQuestStyleOverrides>;
|
|
24
|
+
/** Panel configuration. */
|
|
25
|
+
panel: PanelConfig;
|
|
26
|
+
/** Cleanup. */
|
|
27
|
+
dispose(): void;
|
|
28
|
+
}
|
|
29
|
+
export declare function initializeQuestApp(config: QuestConfig): Promise<QuestContext>;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AgentBridge — TypeScript bridge between visionOS SwiftUI and the Fenix agent.
|
|
3
|
+
* Exposes agent capabilities over a JSON-based message protocol for the native
|
|
4
|
+
* Swift layer to call via JavaScriptCore or WebSocket.
|
|
5
|
+
*/
|
|
6
|
+
import type { MobileAgent } from "../../lib/agent.js";
|
|
7
|
+
import type { IdentityStore } from "../../lib/storage/identity-store.js";
|
|
8
|
+
export type BridgeMessageType = "query" | "cancel" | "memory_search" | "identity_read" | "identity_write" | "status";
|
|
9
|
+
export interface BridgeRequest {
|
|
10
|
+
id: string;
|
|
11
|
+
type: BridgeMessageType;
|
|
12
|
+
payload: Record<string, unknown>;
|
|
13
|
+
}
|
|
14
|
+
export interface BridgeResponse {
|
|
15
|
+
id: string;
|
|
16
|
+
type: "result" | "error" | "stream_token" | "stream_end";
|
|
17
|
+
payload: Record<string, unknown>;
|
|
18
|
+
}
|
|
19
|
+
export interface AgentBridgeConfig {
|
|
20
|
+
agent: MobileAgent;
|
|
21
|
+
identityStore: IdentityStore;
|
|
22
|
+
memorySearch: {
|
|
23
|
+
searchFTS: (query: string, limit?: number) => Array<{
|
|
24
|
+
id: string;
|
|
25
|
+
content: string;
|
|
26
|
+
category: string;
|
|
27
|
+
confidence: number;
|
|
28
|
+
score: number;
|
|
29
|
+
}>;
|
|
30
|
+
};
|
|
31
|
+
onResponse: (response: BridgeResponse) => void;
|
|
32
|
+
}
|
|
33
|
+
export interface AgentBridge {
|
|
34
|
+
handleRequest(request: BridgeRequest): Promise<void>;
|
|
35
|
+
cancelRequest(requestId: string): void;
|
|
36
|
+
getActiveRequests(): string[];
|
|
37
|
+
dispose(): void;
|
|
38
|
+
}
|
|
39
|
+
export declare function createAgentBridge(config: AgentBridgeConfig): AgentBridge;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VisionToolAdapters — visionOS-specific tool adapters.
|
|
3
|
+
* Wraps ARKit scene understanding, spatial anchors, hand tracking,
|
|
4
|
+
* and SharePlay for the tool registry.
|
|
5
|
+
*/
|
|
6
|
+
import type { MobileTool } from "../../lib/tools/registry.js";
|
|
7
|
+
export interface SpatialAnchor {
|
|
8
|
+
id: string;
|
|
9
|
+
label: string;
|
|
10
|
+
position: {
|
|
11
|
+
x: number;
|
|
12
|
+
y: number;
|
|
13
|
+
z: number;
|
|
14
|
+
};
|
|
15
|
+
timestamp: string;
|
|
16
|
+
}
|
|
17
|
+
export interface SceneObject {
|
|
18
|
+
label: string;
|
|
19
|
+
confidence: number;
|
|
20
|
+
boundingBox: {
|
|
21
|
+
x: number;
|
|
22
|
+
y: number;
|
|
23
|
+
z: number;
|
|
24
|
+
width: number;
|
|
25
|
+
height: number;
|
|
26
|
+
depth: number;
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
export interface HandPose {
|
|
30
|
+
hand: "left" | "right";
|
|
31
|
+
gesture: string;
|
|
32
|
+
confidence: number;
|
|
33
|
+
}
|
|
34
|
+
/** Native visionOS APIs injected from Swift. */
|
|
35
|
+
export interface VisionOSNativeAPIs {
|
|
36
|
+
getSceneObjects(): Promise<SceneObject[]>;
|
|
37
|
+
placeAnchor(label: string, position: {
|
|
38
|
+
x: number;
|
|
39
|
+
y: number;
|
|
40
|
+
z: number;
|
|
41
|
+
}): Promise<SpatialAnchor>;
|
|
42
|
+
getAnchors(): Promise<SpatialAnchor[]>;
|
|
43
|
+
removeAnchor(id: string): Promise<boolean>;
|
|
44
|
+
getHandPoses(): Promise<HandPose[]>;
|
|
45
|
+
startSharePlay?(sessionId: string): Promise<boolean>;
|
|
46
|
+
endSharePlay?(): Promise<void>;
|
|
47
|
+
}
|
|
48
|
+
export declare function createSceneUnderstandingTool(native: VisionOSNativeAPIs): MobileTool;
|
|
49
|
+
export declare function createSpatialAnchorTool(native: VisionOSNativeAPIs): MobileTool;
|
|
50
|
+
export declare function createHandTrackingTool(native: VisionOSNativeAPIs): MobileTool;
|
|
51
|
+
export declare function createSharePlayTool(native: VisionOSNativeAPIs): MobileTool;
|
|
52
|
+
/** Create all visionOS spatial tools. */
|
|
53
|
+
export declare function createVisionOSTools(native: VisionOSNativeAPIs): MobileTool[];
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VisionVoiceInterface — voice input/output for visionOS.
|
|
3
|
+
* Uses Apple Speech framework for STT and AVSpeechSynthesizer for TTS.
|
|
4
|
+
* Designed to be called from the Swift layer via the AgentBridge.
|
|
5
|
+
*/
|
|
6
|
+
export interface VisionSTTProvider {
|
|
7
|
+
startListening(): Promise<void>;
|
|
8
|
+
stopListening(): Promise<string>;
|
|
9
|
+
isListening(): boolean;
|
|
10
|
+
}
|
|
11
|
+
export interface VisionTTSProvider {
|
|
12
|
+
speak(text: string): Promise<void>;
|
|
13
|
+
stop(): void;
|
|
14
|
+
isSpeaking(): boolean;
|
|
15
|
+
}
|
|
16
|
+
export interface VisionVoiceConfig {
|
|
17
|
+
stt: VisionSTTProvider;
|
|
18
|
+
tts: VisionTTSProvider;
|
|
19
|
+
/** Called when transcription completes. Returns agent response text. */
|
|
20
|
+
onTranscription: (text: string) => Promise<string>;
|
|
21
|
+
/** Locale for STT/TTS. Default "en-US". */
|
|
22
|
+
locale?: string;
|
|
23
|
+
}
|
|
24
|
+
export interface VisionVoiceInterface {
|
|
25
|
+
startListening(): Promise<void>;
|
|
26
|
+
stopAndProcess(): Promise<string>;
|
|
27
|
+
speak(text: string): Promise<void>;
|
|
28
|
+
cancelSpeech(): void;
|
|
29
|
+
isListening(): boolean;
|
|
30
|
+
isSpeaking(): boolean;
|
|
31
|
+
}
|
|
32
|
+
export declare function createVisionVoiceInterface(config: VisionVoiceConfig): VisionVoiceInterface;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* visionOS Entry Point — initializes Fenix for Apple Vision Pro.
|
|
3
|
+
* Two runtime modes: standalone (full agent) and companion (paired with iPhone).
|
|
4
|
+
* SwiftUI spatial window in Shared Space.
|
|
5
|
+
*/
|
|
6
|
+
import { type FenixMobileConfig, type FenixMobileContext } from "../../lib/entry.js";
|
|
7
|
+
import { type AgentBridge, type BridgeResponse } from "./AgentBridge.js";
|
|
8
|
+
import { type VisionVoiceConfig, type VisionVoiceInterface } from "./VisionVoiceInterface.js";
|
|
9
|
+
import { type VisionOSNativeAPIs } from "./VisionToolAdapters.js";
|
|
10
|
+
export type VisionOSMode = "standalone" | "companion";
|
|
11
|
+
export interface VisionOSConfig extends FenixMobileConfig {
|
|
12
|
+
mode: VisionOSMode;
|
|
13
|
+
voice?: VisionVoiceConfig;
|
|
14
|
+
nativeAPIs?: VisionOSNativeAPIs;
|
|
15
|
+
onBridgeResponse?: (response: BridgeResponse) => void;
|
|
16
|
+
}
|
|
17
|
+
export interface VisionOSContext {
|
|
18
|
+
fenix: FenixMobileContext;
|
|
19
|
+
bridge: AgentBridge;
|
|
20
|
+
voice: VisionVoiceInterface | null;
|
|
21
|
+
mode: VisionOSMode;
|
|
22
|
+
dispose(): void;
|
|
23
|
+
}
|
|
24
|
+
export declare function initializeVisionOS(config: VisionOSConfig): Promise<VisionOSContext>;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { PushNotificationSender } from "@fenixforce/kernel";
|
|
2
|
+
export interface PushNotifierConfig {
|
|
3
|
+
/** Expo push endpoint. Default: https://exp.host/--/api/v2/push/send */
|
|
4
|
+
endpoint?: string;
|
|
5
|
+
/** Function to retrieve push tokens for a user/session. */
|
|
6
|
+
getDeviceTokens: (sessionId: string) => Promise<string[]>;
|
|
7
|
+
/** Optional fetch implementation. */
|
|
8
|
+
fetch?: typeof globalThis.fetch;
|
|
9
|
+
}
|
|
10
|
+
export interface ExpoPushMessage {
|
|
11
|
+
to: string;
|
|
12
|
+
title: string;
|
|
13
|
+
body: string;
|
|
14
|
+
data?: Record<string, unknown>;
|
|
15
|
+
sound?: string;
|
|
16
|
+
priority?: "default" | "normal" | "high";
|
|
17
|
+
}
|
|
18
|
+
export interface ExpoPushResponse {
|
|
19
|
+
data: Array<{
|
|
20
|
+
id?: string;
|
|
21
|
+
status: "ok" | "error";
|
|
22
|
+
message?: string;
|
|
23
|
+
}>;
|
|
24
|
+
}
|
|
25
|
+
export declare function createPushNotifier(config: PushNotifierConfig): PushNotificationSender;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fenix Mobile Edition — Lightweight agent for mobile and
|
|
3
|
+
* resource-constrained environments. SQLite for local persistence,
|
|
4
|
+
* restricted tool categories.
|
|
5
|
+
*/
|
|
6
|
+
export { EDITION, MOBILE_MANIFEST, MOBILE_KERNEL_CONFIG, startMobile, bootMobile, createMobileAgent, createKernel, bootKernel, KERNEL_VERSION, } from "../lib/agent.js";
|
|
7
|
+
export type { MobileAgentConfig, MobileAgent } from "../lib/agent.js";
|
|
8
|
+
export { createSQLiteAdapter } from "../lib/storage/sqlite-adapter.js";
|
|
9
|
+
export type { SQLiteDB } from "../lib/storage/sqlite-adapter.js";
|
|
10
|
+
export { createMemorySearch } from "../lib/storage/memory-search.js";
|
|
11
|
+
export type { MemorySearchResult, MemorySearchConfig } from "../lib/storage/memory-search.js";
|
|
12
|
+
export { createHybridRouter } from "../lib/providers/hybrid-router.js";
|
|
13
|
+
export type { HybridRouterConfig, LLMProvider } from "../lib/providers/hybrid-router.js";
|
|
14
|
+
export { createCalendarTool } from "../lib/tools/calendar.js";
|
|
15
|
+
export { createContactsTool } from "../lib/tools/contacts.js";
|
|
16
|
+
export { createCameraTool } from "../lib/tools/camera.js";
|
|
17
|
+
export { createLocationTool } from "../lib/tools/location.js";
|
|
18
|
+
export { createRemindersTool } from "../lib/tools/reminders.js";
|
|
19
|
+
export { createNotificationsTool } from "../lib/tools/notifications.js";
|
|
20
|
+
export { createMobileToolRegistry } from "../lib/tools/registry.js";
|
|
21
|
+
export type { MobileTool, MobileToolRegistryConfig } from "../lib/tools/registry.js";
|
|
22
|
+
export { createIdentityStore } from "../lib/storage/identity-store.js";
|
|
23
|
+
export type { IdentityStore } from "../lib/storage/identity-store.js";
|
|
24
|
+
export type { AppServices } from "../lib/app-context.js";
|
|
25
|
+
export { bootApp } from "../lib/boot.js";
|
|
26
|
+
export type { BootOptions } from "../lib/boot.js";
|
|
27
|
+
export { initializeFenixMobile } from "../lib/entry.js";
|
|
28
|
+
export type { FenixMobileConfig, FenixMobileContext } from "../lib/entry.js";
|
|
29
|
+
export { createSyncClient } from "../lib/sync/client.js";
|
|
30
|
+
export type { SyncClient, SyncStatus, SyncClientConfig, LocalChanges, PushResult, PullResult, ConflictRecord } from "../lib/sync/client.js";
|
|
31
|
+
export { createSyncTriggers, SYNC_TASK_NAME } from "../lib/sync/triggers.js";
|
|
32
|
+
export type { SyncTriggerConfig, SyncTriggers, SyncResult, TriggerSource } from "../lib/sync/triggers.js";
|
|
33
|
+
export { createLocalProvider } from "../lib/providers/local-inference.js";
|
|
34
|
+
export type { LocalModelConfig, EmbeddedInference } from "../lib/providers/local-inference.js";
|
|
35
|
+
export { createWatchBridge, generateSuggestions } from "../watch/WatchBridge.js";
|
|
36
|
+
export type { WatchBridge, WatchBridgeConfig, WatchConnectivity, WatchMessage, PhoneResponse, AgentStatus } from "../watch/WatchBridge.js";
|
|
37
|
+
export { createQuestAdapter } from "../platforms/quest/QuestAdapter.js";
|
|
38
|
+
export type { QuestAdapter, ControllerEvent, ControllerMapping, ControllerButton, AudioBuffer } from "../platforms/quest/QuestAdapter.js";
|
|
39
|
+
export { DEFAULT_PANEL_CONFIG, getQuestStyleOverrides } from "../platforms/quest/QuestPanelConfig.js";
|
|
40
|
+
export type { PanelConfig } from "../platforms/quest/QuestPanelConfig.js";
|
|
41
|
+
export { initializeQuestApp } from "../platforms/quest/quest-entry.js";
|
|
42
|
+
export type { QuestConfig, QuestContext } from "../platforms/quest/quest-entry.js";
|
|
43
|
+
export { createVoiceInterface } from "../platforms/android-xr/XRVoiceInterface.js";
|
|
44
|
+
export type { VoiceInterface, VoiceInterfaceConfig, STTProvider, TTSProvider, WakeWordDetector, AudioChunk, TTSOptions } from "../platforms/android-xr/XRVoiceInterface.js";
|
|
45
|
+
export { createContextCapture } from "../platforms/android-xr/XRContextCapture.js";
|
|
46
|
+
export type { ContextCapture, ContextCaptureConfig, CameraFrame, VisionProvider, CameraAccess } from "../platforms/android-xr/XRContextCapture.js";
|
|
47
|
+
export { createGestureHandler } from "../platforms/android-xr/XRGestureHandler.js";
|
|
48
|
+
export type { GestureHandler, GestureHandlerConfig, GestureMapping, XRInputEvent, XRAction, TouchpadGesture, HeadGesture, HardwareButton } from "../platforms/android-xr/XRGestureHandler.js";
|
|
49
|
+
export { createNotificationBridge } from "../platforms/android-xr/XRNotificationBridge.js";
|
|
50
|
+
export type { NotificationBridge, NotificationBridgeConfig, SystemNotification } from "../platforms/android-xr/XRNotificationBridge.js";
|
|
51
|
+
export { createXRAgentService } from "../platforms/android-xr/XRAgentService.js";
|
|
52
|
+
export type { XRAgentService, XRAgentServiceConfig } from "../platforms/android-xr/XRAgentService.js";
|
|
53
|
+
export { initializeAndroidXR } from "../platforms/android-xr/xr-entry.js";
|
|
54
|
+
export type { AndroidXRConfig, AndroidXRContext } from "../platforms/android-xr/xr-entry.js";
|
|
55
|
+
export { createGlanceCard, pinCard, unpinCard } from "../platforms/android-xr/display/GlanceCard.js";
|
|
56
|
+
export type { GlanceCardData, GlanceCardConfig } from "../platforms/android-xr/display/GlanceCard.js";
|
|
57
|
+
export { createNotificationPill } from "../platforms/android-xr/display/NotificationPill.js";
|
|
58
|
+
export type { NotificationPillData } from "../platforms/android-xr/display/NotificationPill.js";
|
|
59
|
+
export { createConversationOverlay, appendMessage, scrollOverlay, getVisibleMessages } from "../platforms/android-xr/display/ConversationOverlay.js";
|
|
60
|
+
export type { ConversationOverlayState, OverlayMessage } from "../platforms/android-xr/display/ConversationOverlay.js";
|
|
61
|
+
export { createMemoryHUD, updateAnnotations, dismissHUD } from "../platforms/android-xr/display/MemoryHUD.js";
|
|
62
|
+
export type { MemoryHUDState, MemoryAnnotation } from "../platforms/android-xr/display/MemoryHUD.js";
|
|
63
|
+
export { createDisplayManager } from "../platforms/android-xr/display/DisplayManager.js";
|
|
64
|
+
export type { DisplayAdapter, DisplayManagerConfig, DisplayState, DisplayType, ResponseMetadata } from "../platforms/android-xr/display/DisplayManager.js";
|
|
65
|
+
export { createAgentBridge } from "../platforms/visionos/AgentBridge.js";
|
|
66
|
+
export type { AgentBridge, AgentBridgeConfig, BridgeRequest, BridgeResponse, BridgeMessageType } from "../platforms/visionos/AgentBridge.js";
|
|
67
|
+
export { createVisionVoiceInterface } from "../platforms/visionos/VisionVoiceInterface.js";
|
|
68
|
+
export type { VisionVoiceInterface, VisionVoiceConfig, VisionSTTProvider, VisionTTSProvider } from "../platforms/visionos/VisionVoiceInterface.js";
|
|
69
|
+
export { createVisionOSTools, createSceneUnderstandingTool, createSpatialAnchorTool, createHandTrackingTool, createSharePlayTool } from "../platforms/visionos/VisionToolAdapters.js";
|
|
70
|
+
export type { VisionOSNativeAPIs, SpatialAnchor, SceneObject, HandPose } from "../platforms/visionos/VisionToolAdapters.js";
|
|
71
|
+
export { initializeVisionOS } from "../platforms/visionos/visionos-entry.js";
|
|
72
|
+
export type { VisionOSConfig, VisionOSContext, VisionOSMode } from "../platforms/visionos/visionos-entry.js";
|