@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +83 -0
- package/package.json +82 -15
- package/src/actions/generate-media.d.ts +59 -0
- package/src/actions/generate-media.d.ts.map +1 -0
- package/src/actions/generate-media.ts +647 -0
- package/src/actions/identify-speaker.d.ts +23 -0
- package/src/actions/identify-speaker.d.ts.map +1 -0
- package/src/actions/identify-speaker.ts +171 -0
- package/src/actions/transcription-control.d.ts +29 -0
- package/src/actions/transcription-control.d.ts.map +1 -0
- package/src/actions/transcription-control.test.ts +100 -0
- package/src/actions/transcription-control.ts +127 -0
- package/src/adapters/capacitor-llama/__tests__/compat-behavior.test.ts +218 -0
- package/src/adapters/capacitor-llama/__tests__/index.test.ts +68 -0
- package/src/adapters/capacitor-llama/__tests__/structured-output.test.ts +215 -0
- package/src/adapters/capacitor-llama/__tests__/text-streaming.test.ts +174 -0
- package/src/adapters/capacitor-llama/environment.ts +71 -0
- package/src/adapters/capacitor-llama/index.browser.ts +83 -0
- package/src/adapters/capacitor-llama/index.ts +807 -0
- package/src/adapters/capacitor-llama/loader.ts +109 -0
- package/src/adapters/capacitor-llama/structured-output.ts +165 -0
- package/src/adapters/capacitor-llama/text-streaming.ts +227 -0
- package/src/adapters/capacitor-llama/types.ts +374 -0
- package/src/backends/apple-foundation.ts +127 -0
- package/src/index.d.ts +8 -0
- package/src/index.d.ts.map +1 -0
- package/src/index.ts +62 -0
- package/src/local-inference-routes.d.ts +38 -0
- package/src/local-inference-routes.d.ts.map +1 -0
- package/src/local-inference-routes.test.ts +344 -0
- package/src/local-inference-routes.ts +1543 -0
- package/src/provider.d.ts +21 -0
- package/src/provider.d.ts.map +1 -0
- package/src/provider.ts +1082 -0
- package/src/routes/compat-helpers.d.ts +18 -0
- package/src/routes/compat-helpers.d.ts.map +1 -0
- package/src/routes/compat-helpers.ts +274 -0
- package/src/routes/family-member-route.d.ts +62 -0
- package/src/routes/family-member-route.d.ts.map +1 -0
- package/src/routes/family-member-route.ts +353 -0
- package/src/routes/index.d.ts +19 -0
- package/src/routes/index.d.ts.map +1 -0
- package/src/routes/index.ts +60 -0
- package/src/routes/live-diarization-route.d.ts +26 -0
- package/src/routes/live-diarization-route.d.ts.map +1 -0
- package/src/routes/live-diarization-route.test.ts +213 -0
- package/src/routes/live-diarization-route.ts +122 -0
- package/src/routes/local-inference-asr-route.d.ts +4 -0
- package/src/routes/local-inference-asr-route.d.ts.map +1 -0
- package/src/routes/local-inference-asr-route.test.ts +205 -0
- package/src/routes/local-inference-asr-route.ts +163 -0
- package/src/routes/local-inference-asr-transcribe.d.ts +20 -0
- package/src/routes/local-inference-asr-transcribe.d.ts.map +1 -0
- package/src/routes/local-inference-asr-transcribe.test.ts +118 -0
- package/src/routes/local-inference-asr-transcribe.ts +97 -0
- package/src/routes/local-inference-compat-routes.d.ts +16 -0
- package/src/routes/local-inference-compat-routes.d.ts.map +1 -0
- package/src/routes/local-inference-compat-routes.test.ts +485 -0
- package/src/routes/local-inference-compat-routes.ts +808 -0
- package/src/routes/local-inference-tts-route.d.ts +7 -0
- package/src/routes/local-inference-tts-route.d.ts.map +1 -0
- package/src/routes/local-inference-tts-route.test.ts +179 -0
- package/src/routes/local-inference-tts-route.ts +230 -0
- package/src/routes/transcript-audio-store.d.ts +15 -0
- package/src/routes/transcript-audio-store.d.ts.map +1 -0
- package/src/routes/transcript-audio-store.ts +27 -0
- package/src/routes/transcripts-routes.d.ts +36 -0
- package/src/routes/transcripts-routes.d.ts.map +1 -0
- package/src/routes/transcripts-routes.test.ts +144 -0
- package/src/routes/transcripts-routes.ts +159 -0
- package/src/routes/voice-first-run-routes.d.ts +62 -0
- package/src/routes/voice-first-run-routes.d.ts.map +1 -0
- package/src/routes/voice-first-run-routes.ts +524 -0
- package/src/routes/voice-models-routes.d.ts +62 -0
- package/src/routes/voice-models-routes.d.ts.map +1 -0
- package/src/routes/voice-models-routes.ts +554 -0
- package/src/routes/voice-profile-plugin-routes.d.ts +19 -0
- package/src/routes/voice-profile-plugin-routes.d.ts.map +1 -0
- package/src/routes/voice-profile-plugin-routes.ts +138 -0
- package/src/routes/voice-profiles-management-routes.d.ts +52 -0
- package/src/routes/voice-profiles-management-routes.d.ts.map +1 -0
- package/src/routes/voice-profiles-management-routes.ts +476 -0
- package/src/routes/voice-speaker-profile-routes.d.ts +57 -0
- package/src/routes/voice-speaker-profile-routes.d.ts.map +1 -0
- package/src/routes/voice-speaker-profile-routes.ts +199 -0
- package/src/runtime/aosp-llama-loader-selection.test.ts +80 -0
- package/src/runtime/capacitor-llama.d.ts +25 -0
- package/src/runtime/embedding-manager-support.d.ts +77 -0
- package/src/runtime/embedding-manager-support.d.ts.map +1 -0
- package/src/runtime/embedding-manager-support.ts +497 -0
- package/src/runtime/embedding-presets.d.ts +16 -0
- package/src/runtime/embedding-presets.d.ts.map +1 -0
- package/src/runtime/embedding-presets.ts +81 -0
- package/src/runtime/embedding-warmup-policy.d.ts +14 -0
- package/src/runtime/embedding-warmup-policy.d.ts.map +1 -0
- package/src/runtime/embedding-warmup-policy.test.ts +53 -0
- package/src/runtime/embedding-warmup-policy.ts +48 -0
- package/src/runtime/ensure-local-inference-handler.d.ts +62 -0
- package/src/runtime/ensure-local-inference-handler.d.ts.map +1 -0
- package/src/runtime/ensure-local-inference-handler.test.ts +528 -0
- package/src/runtime/ensure-local-inference-handler.ts +1448 -0
- package/src/runtime/index.d.ts +15 -0
- package/src/runtime/index.d.ts.map +1 -0
- package/src/runtime/index.ts +33 -0
- package/src/runtime/mobile-local-inference-gate.d.ts +31 -0
- package/src/runtime/mobile-local-inference-gate.d.ts.map +1 -0
- package/src/runtime/mobile-local-inference-gate.test.ts +69 -0
- package/src/runtime/mobile-local-inference-gate.ts +44 -0
- package/src/runtime/voice-entity-binding.d.ts +103 -0
- package/src/runtime/voice-entity-binding.d.ts.map +1 -0
- package/src/runtime/voice-entity-binding.transcript.test.ts +69 -0
- package/src/runtime/voice-entity-binding.ts +328 -0
- package/src/services/README.md +71 -0
- package/src/services/__tests__/backend-selector.test.ts +101 -0
- package/src/services/__tests__/checkpoint-manager.test.ts +376 -0
- package/src/services/__tests__/gpu-autotune.test.ts +400 -0
- package/src/services/__tests__/llm-streaming-binding.test.ts +85 -0
- package/src/services/__tests__/planner-grammar.test.ts +372 -0
- package/src/services/__tests__/runtime-target.test.ts +176 -0
- package/src/services/active-model-switch-rollback.test.ts +183 -0
- package/src/services/active-model.d.ts +282 -0
- package/src/services/active-model.d.ts.map +1 -0
- package/src/services/active-model.ts +1213 -0
- package/src/services/assignments.d.ts +71 -0
- package/src/services/assignments.d.ts.map +1 -0
- package/src/services/assignments.test.ts +80 -0
- package/src/services/assignments.ts +230 -0
- package/src/services/backend-selector.ts +95 -0
- package/src/services/backend.d.ts +346 -0
- package/src/services/backend.d.ts.map +1 -0
- package/src/services/backend.ts +612 -0
- package/src/services/bionic-host-loader.d.ts +46 -0
- package/src/services/bionic-host-loader.d.ts.map +1 -0
- package/src/services/bionic-host-loader.test.ts +133 -0
- package/src/services/bionic-host-loader.ts +180 -0
- package/src/services/bundled-models.d.ts +34 -0
- package/src/services/bundled-models.d.ts.map +1 -0
- package/src/services/bundled-models.ts +129 -0
- package/src/services/cache-bridge.d.ts +206 -0
- package/src/services/cache-bridge.d.ts.map +1 -0
- package/src/services/cache-bridge.test.ts +516 -0
- package/src/services/cache-bridge.ts +423 -0
- package/src/services/catalog.d.ts +10 -0
- package/src/services/catalog.d.ts.map +1 -0
- package/src/services/catalog.test.ts +238 -0
- package/src/services/catalog.ts +27 -0
- package/src/services/checkpoint-client.d.ts +109 -0
- package/src/services/checkpoint-client.d.ts.map +1 -0
- package/src/services/checkpoint-client.ts +258 -0
- package/src/services/checkpoint-manager.ts +474 -0
- package/src/services/cloud-fallback.d.ts +102 -0
- package/src/services/cloud-fallback.d.ts.map +1 -0
- package/src/services/cloud-fallback.ts +230 -0
- package/src/services/conversation-registry.d.ts +142 -0
- package/src/services/conversation-registry.d.ts.map +1 -0
- package/src/services/conversation-registry.test.ts +235 -0
- package/src/services/conversation-registry.ts +264 -0
- package/src/services/desktop-fused-ffi-backend-runtime.d.ts +95 -0
- package/src/services/desktop-fused-ffi-backend-runtime.d.ts.map +1 -0
- package/src/services/desktop-fused-ffi-backend-runtime.ts +339 -0
- package/src/services/device-bridge.d.ts +188 -0
- package/src/services/device-bridge.d.ts.map +1 -0
- package/src/services/device-bridge.ts +1237 -0
- package/src/services/device-resource-metrics.d.ts +149 -0
- package/src/services/device-resource-metrics.d.ts.map +1 -0
- package/src/services/device-resource-metrics.test.ts +98 -0
- package/src/services/device-resource-metrics.ts +346 -0
- package/src/services/device-tier.d.ts +115 -0
- package/src/services/device-tier.d.ts.map +1 -0
- package/src/services/device-tier.test.ts +371 -0
- package/src/services/device-tier.ts +410 -0
- package/src/services/downloader.d.ts +82 -0
- package/src/services/downloader.d.ts.map +1 -0
- package/src/services/downloader.test.ts +747 -0
- package/src/services/downloader.ts +925 -0
- package/src/services/engine-direct-bundle.test.ts +58 -0
- package/src/services/engine-streaming.test.ts +80 -0
- package/src/services/engine.d.ts +540 -0
- package/src/services/engine.d.ts.map +1 -0
- package/src/services/engine.ts +1909 -0
- package/src/services/ensure-local-artifacts.integration.test.ts +273 -0
- package/src/services/ensure-local-artifacts.test.ts +368 -0
- package/src/services/ensure-local-artifacts.ts +351 -0
- package/src/services/external-scanner.d.ts +17 -0
- package/src/services/external-scanner.d.ts.map +1 -0
- package/src/services/external-scanner.ts +312 -0
- package/src/services/ffi-llm-mock.ts +354 -0
- package/src/services/ffi-llm-streaming-abi.ts +442 -0
- package/src/services/ffi-streaming-backend.d.ts +180 -0
- package/src/services/ffi-streaming-backend.d.ts.map +1 -0
- package/src/services/ffi-streaming-backend.ts +382 -0
- package/src/services/ffi-streaming-runner.d.ts +122 -0
- package/src/services/ffi-streaming-runner.d.ts.map +1 -0
- package/src/services/ffi-streaming-runner.test.ts +60 -0
- package/src/services/ffi-streaming-runner.ts +354 -0
- package/src/services/ffi-unload-ordering.test.ts +162 -0
- package/src/services/gpu-autotune.ts +534 -0
- package/src/services/gpu-detect.d.ts +56 -0
- package/src/services/gpu-detect.d.ts.map +1 -0
- package/src/services/gpu-detect.ts +139 -0
- package/src/services/handler-registry.d.ts +72 -0
- package/src/services/handler-registry.d.ts.map +1 -0
- package/src/services/handler-registry.ts +240 -0
- package/src/services/hardware.d.ts +63 -0
- package/src/services/hardware.d.ts.map +1 -0
- package/src/services/hardware.test.ts +231 -0
- package/src/services/hardware.ts +410 -0
- package/src/services/hf-search.d.ts +26 -0
- package/src/services/hf-search.d.ts.map +1 -0
- package/src/services/hf-search.test.ts +69 -0
- package/src/services/hf-search.ts +420 -0
- package/src/services/image-description-runtime.d.ts +14 -0
- package/src/services/image-description-runtime.d.ts.map +1 -0
- package/src/services/image-description-runtime.test.ts +61 -0
- package/src/services/image-description-runtime.ts +118 -0
- package/src/services/imagegen/aosp-unavailable.d.ts +134 -0
- package/src/services/imagegen/aosp-unavailable.d.ts.map +1 -0
- package/src/services/imagegen/aosp-unavailable.ts +229 -0
- package/src/services/imagegen/backend-selector.d.ts +118 -0
- package/src/services/imagegen/backend-selector.d.ts.map +1 -0
- package/src/services/imagegen/backend-selector.ts +277 -0
- package/src/services/imagegen/coreml-unavailable.d.ts +105 -0
- package/src/services/imagegen/coreml-unavailable.d.ts.map +1 -0
- package/src/services/imagegen/coreml-unavailable.ts +237 -0
- package/src/services/imagegen/errors.d.ts +16 -0
- package/src/services/imagegen/errors.d.ts.map +1 -0
- package/src/services/imagegen/errors.ts +40 -0
- package/src/services/imagegen/index.d.ts +58 -0
- package/src/services/imagegen/index.d.ts.map +1 -0
- package/src/services/imagegen/index.ts +144 -0
- package/src/services/imagegen/mflux.d.ts +74 -0
- package/src/services/imagegen/mflux.d.ts.map +1 -0
- package/src/services/imagegen/mflux.ts +313 -0
- package/src/services/imagegen/sd-cpp.d.ts +180 -0
- package/src/services/imagegen/sd-cpp.d.ts.map +1 -0
- package/src/services/imagegen/sd-cpp.ts +718 -0
- package/src/services/imagegen/tensorrt-unavailable.d.ts +83 -0
- package/src/services/imagegen/tensorrt-unavailable.d.ts.map +1 -0
- package/src/services/imagegen/tensorrt-unavailable.ts +295 -0
- package/src/services/imagegen/types.d.ts +181 -0
- package/src/services/imagegen/types.d.ts.map +1 -0
- package/src/services/imagegen/types.ts +193 -0
- package/src/services/index.d.ts +29 -0
- package/src/services/index.d.ts.map +1 -0
- package/src/services/index.ts +211 -0
- package/src/services/inference-capabilities.d.ts +132 -0
- package/src/services/inference-capabilities.d.ts.map +1 -0
- package/src/services/inference-capabilities.test.ts +75 -0
- package/src/services/inference-capabilities.ts +204 -0
- package/src/services/inference-telemetry.d.ts +59 -0
- package/src/services/inference-telemetry.d.ts.map +1 -0
- package/src/services/inference-telemetry.ts +143 -0
- package/src/services/ios-llama-streaming.ts +248 -0
- package/src/services/kv-spill.d.ts +189 -0
- package/src/services/kv-spill.d.ts.map +1 -0
- package/src/services/kv-spill.test.ts +222 -0
- package/src/services/kv-spill.ts +356 -0
- package/src/services/latency-trace.d.ts +346 -0
- package/src/services/latency-trace.d.ts.map +1 -0
- package/src/services/latency-trace.test.ts +266 -0
- package/src/services/latency-trace.ts +844 -0
- package/src/services/llama-server-metrics.ts +304 -0
- package/src/services/llm-streaming-binding.d.ts +96 -0
- package/src/services/llm-streaming-binding.d.ts.map +1 -0
- package/src/services/llm-streaming-binding.ts +136 -0
- package/src/services/load-args.d.ts +82 -0
- package/src/services/load-args.d.ts.map +1 -0
- package/src/services/load-args.ts +81 -0
- package/src/services/manifest/eliza-1.manifest.v1.json +708 -0
- package/src/services/manifest/index.d.ts +4 -0
- package/src/services/manifest/index.d.ts.map +1 -0
- package/src/services/manifest/index.ts +66 -0
- package/src/services/manifest/manifest.test.ts +689 -0
- package/src/services/manifest/schema.d.ts +713 -0
- package/src/services/manifest/schema.d.ts.map +1 -0
- package/src/services/manifest/schema.ts +653 -0
- package/src/services/manifest/types.d.ts +30 -0
- package/src/services/manifest/types.d.ts.map +1 -0
- package/src/services/manifest/types.ts +55 -0
- package/src/services/manifest/validator.d.ts +66 -0
- package/src/services/manifest/validator.d.ts.map +1 -0
- package/src/services/manifest/validator.ts +567 -0
- package/src/services/memory-arbiter.d.ts +318 -0
- package/src/services/memory-arbiter.d.ts.map +1 -0
- package/src/services/memory-arbiter.test.ts +419 -0
- package/src/services/memory-arbiter.ts +925 -0
- package/src/services/memory-monitor.d.ts +122 -0
- package/src/services/memory-monitor.d.ts.map +1 -0
- package/src/services/memory-monitor.test.ts +208 -0
- package/src/services/memory-monitor.ts +297 -0
- package/src/services/memory-pressure.d.ts +130 -0
- package/src/services/memory-pressure.d.ts.map +1 -0
- package/src/services/memory-pressure.ts +414 -0
- package/src/services/mtp-doctor.d.ts +13 -0
- package/src/services/mtp-doctor.d.ts.map +1 -0
- package/src/services/mtp-doctor.ts +78 -0
- package/src/services/network-policy.d.ts +127 -0
- package/src/services/network-policy.d.ts.map +1 -0
- package/src/services/network-policy.ts +346 -0
- package/src/services/paths.d.ts +6 -0
- package/src/services/paths.d.ts.map +1 -0
- package/src/services/paths.ts +25 -0
- package/src/services/planner-skeleton.d.ts +124 -0
- package/src/services/planner-skeleton.d.ts.map +1 -0
- package/src/services/planner-skeleton.ts +175 -0
- package/src/services/providers.d.ts +38 -0
- package/src/services/providers.d.ts.map +1 -0
- package/src/services/providers.ts +507 -0
- package/src/services/ram-budget-cache.test.ts +163 -0
- package/src/services/ram-budget.d.ts +110 -0
- package/src/services/ram-budget.d.ts.map +1 -0
- package/src/services/ram-budget.ts +0 -0
- package/src/services/readiness.d.ts +9 -0
- package/src/services/readiness.d.ts.map +1 -0
- package/src/services/readiness.test.ts +87 -0
- package/src/services/readiness.ts +238 -0
- package/src/services/recommendation.d.ts +111 -0
- package/src/services/recommendation.d.ts.map +1 -0
- package/src/services/recommendation.ts +671 -0
- package/src/services/registry.d.ts +35 -0
- package/src/services/registry.d.ts.map +1 -0
- package/src/services/registry.ts +151 -0
- package/src/services/router-handler.d.ts +92 -0
- package/src/services/router-handler.d.ts.map +1 -0
- package/src/services/router-handler.test.ts +45 -0
- package/src/services/router-handler.ts +407 -0
- package/src/services/routing-policy.d.ts +69 -0
- package/src/services/routing-policy.d.ts.map +1 -0
- package/src/services/routing-policy.test.ts +164 -0
- package/src/services/routing-policy.ts +297 -0
- package/src/services/routing-preferences.d.ts +8 -0
- package/src/services/routing-preferences.d.ts.map +1 -0
- package/src/services/routing-preferences.ts +17 -0
- package/src/services/runtime-target.d.ts +98 -0
- package/src/services/runtime-target.d.ts.map +1 -0
- package/src/services/runtime-target.ts +154 -0
- package/src/services/service.d.ts +128 -0
- package/src/services/service.d.ts.map +1 -0
- package/src/services/service.test.ts +223 -0
- package/src/services/service.ts +735 -0
- package/src/services/session-pool.d.ts +72 -0
- package/src/services/session-pool.d.ts.map +1 -0
- package/src/services/session-pool.ts +153 -0
- package/src/services/structured-output/deterministic-repair.d.ts +23 -0
- package/src/services/structured-output/deterministic-repair.d.ts.map +1 -0
- package/src/services/structured-output/deterministic-repair.test.ts +169 -0
- package/src/services/structured-output/deterministic-repair.ts +443 -0
- package/src/services/structured-output/index.ts +4 -0
- package/src/services/structured-output.d.ts +311 -0
- package/src/services/structured-output.d.ts.map +1 -0
- package/src/services/structured-output.test.ts +483 -0
- package/src/services/structured-output.ts +712 -0
- package/src/services/system-memory.d.ts +33 -0
- package/src/services/system-memory.d.ts.map +1 -0
- package/src/services/system-memory.test.ts +47 -0
- package/src/services/system-memory.ts +67 -0
- package/src/services/transcription-priority.test.ts +211 -0
- package/src/services/types.d.ts +19 -0
- package/src/services/types.d.ts.map +1 -0
- package/src/services/types.ts +55 -0
- package/src/services/verify-on-device.d.ts +34 -0
- package/src/services/verify-on-device.d.ts.map +1 -0
- package/src/services/verify-on-device.test.ts +87 -0
- package/src/services/verify-on-device.ts +127 -0
- package/src/services/verify.d.ts +8 -0
- package/src/services/verify.d.ts.map +1 -0
- package/src/services/verify.ts +13 -0
- package/src/services/vision/aosp-unavailable.d.ts +115 -0
- package/src/services/vision/aosp-unavailable.d.ts.map +1 -0
- package/src/services/vision/aosp-unavailable.ts +163 -0
- package/src/services/vision/capacitor-llama.d.ts +99 -0
- package/src/services/vision/capacitor-llama.d.ts.map +1 -0
- package/src/services/vision/capacitor-llama.ts +255 -0
- package/src/services/vision/cloud-fallback.d.ts +47 -0
- package/src/services/vision/cloud-fallback.d.ts.map +1 -0
- package/src/services/vision/cloud-fallback.test.ts +243 -0
- package/src/services/vision/cloud-fallback.ts +268 -0
- package/src/services/vision/fallback-chain.test.ts +86 -0
- package/src/services/vision/hash.d.ts +71 -0
- package/src/services/vision/hash.d.ts.map +1 -0
- package/src/services/vision/hash.ts +157 -0
- package/src/services/vision/index.d.ts +95 -0
- package/src/services/vision/index.d.ts.map +1 -0
- package/src/services/vision/index.ts +251 -0
- package/src/services/vision/llama-server.d.ts +73 -0
- package/src/services/vision/llama-server.d.ts.map +1 -0
- package/src/services/vision/llama-server.ts +177 -0
- package/src/services/vision/types.d.ts +153 -0
- package/src/services/vision/types.d.ts.map +1 -0
- package/src/services/vision/types.ts +154 -0
- package/src/services/vision/vast-fallback.d.ts +18 -0
- package/src/services/vision/vast-fallback.d.ts.map +1 -0
- package/src/services/vision/vast-fallback.ts +127 -0
- package/src/services/vision-embedding-cache.d.ts +98 -0
- package/src/services/vision-embedding-cache.d.ts.map +1 -0
- package/src/services/vision-embedding-cache.ts +189 -0
- package/src/services/voice/VOICE_WORKBENCH.md +88 -0
- package/src/services/voice/__test-helpers__/fake-ffi.ts +94 -0
- package/src/services/voice/__test-helpers__/synthetic-speech.ts +124 -0
- package/src/services/voice/__tests__/checkpoint-manager.test.ts +241 -0
- package/src/services/voice/__tests__/checkpoint-policy.test.ts +270 -0
- package/src/services/voice/__tests__/eager-context-builder.test.ts +257 -0
- package/src/services/voice/__tests__/eliza1-eot-scorer.test.ts +288 -0
- package/src/services/voice/__tests__/eot-classifier.test.ts +431 -0
- package/src/services/voice/__tests__/optimistic-rollback.test.ts +312 -0
- package/src/services/voice/__tests__/prefill-client.test.ts +266 -0
- package/src/services/voice/__tests__/prefix-preserving-queue.test.ts +208 -0
- package/src/services/voice/__tests__/streaming-asr.test.ts +450 -0
- package/src/services/voice/__tests__/streaming-transcriber.test.ts +339 -0
- package/src/services/voice/__tests__/turn-detector-resolver.test.ts +195 -0
- package/src/services/voice/__tests__/voice-state-machine-prefill.test.ts +275 -0
- package/src/services/voice/__tests__/voice-state-machine.test.ts +354 -0
- package/src/services/voice/asr-timed.real.test.ts +141 -0
- package/src/services/voice/audio-frame-consumer.d.ts +212 -0
- package/src/services/voice/audio-frame-consumer.d.ts.map +1 -0
- package/src/services/voice/audio-frame-consumer.test.ts +343 -0
- package/src/services/voice/audio-frame-consumer.ts +491 -0
- package/src/services/voice/barge-in.d.ts +112 -0
- package/src/services/voice/barge-in.d.ts.map +1 -0
- package/src/services/voice/barge-in.test.ts +244 -0
- package/src/services/voice/barge-in.ts +336 -0
- package/src/services/voice/cancellation-coordinator.d.ts +127 -0
- package/src/services/voice/cancellation-coordinator.d.ts.map +1 -0
- package/src/services/voice/cancellation-coordinator.test.ts +196 -0
- package/src/services/voice/cancellation-coordinator.ts +269 -0
- package/src/services/voice/checkpoint-manager.d.ts +199 -0
- package/src/services/voice/checkpoint-manager.d.ts.map +1 -0
- package/src/services/voice/checkpoint-manager.ts +401 -0
- package/src/services/voice/checkpoint-policy.ts +336 -0
- package/src/services/voice/composite-eot-classifier.test.ts +59 -0
- package/src/services/voice/e2e-harness.test.ts +182 -0
- package/src/services/voice/e2e-harness.ts +743 -0
- package/src/services/voice/eager-context-builder.d.ts +170 -0
- package/src/services/voice/eager-context-builder.d.ts.map +1 -0
- package/src/services/voice/eager-context-builder.ts +262 -0
- package/src/services/voice/eliza1-eot-scorer.d.ts +124 -0
- package/src/services/voice/eliza1-eot-scorer.d.ts.map +1 -0
- package/src/services/voice/eliza1-eot-scorer.ts +242 -0
- package/src/services/voice/embedding-server.ts +200 -0
- package/src/services/voice/embedding.d.ts +133 -0
- package/src/services/voice/embedding.d.ts.map +1 -0
- package/src/services/voice/embedding.test.ts +131 -0
- package/src/services/voice/embedding.ts +243 -0
- package/src/services/voice/emotion-attribution.d.ts +68 -0
- package/src/services/voice/emotion-attribution.d.ts.map +1 -0
- package/src/services/voice/emotion-attribution.test.ts +129 -0
- package/src/services/voice/emotion-attribution.ts +361 -0
- package/src/services/voice/engine-bridge-cancellation.test.ts +422 -0
- package/src/services/voice/engine-bridge.d.ts +759 -0
- package/src/services/voice/engine-bridge.d.ts.map +1 -0
- package/src/services/voice/engine-bridge.test.ts +384 -0
- package/src/services/voice/engine-bridge.ts +2302 -0
- package/src/services/voice/eot-classifier-ggml.d.ts +179 -0
- package/src/services/voice/eot-classifier-ggml.d.ts.map +1 -0
- package/src/services/voice/eot-classifier-ggml.ts +566 -0
- package/src/services/voice/eot-classifier.d.ts +214 -0
- package/src/services/voice/eot-classifier.d.ts.map +1 -0
- package/src/services/voice/eot-classifier.ts +533 -0
- package/src/services/voice/errors.d.ts +20 -0
- package/src/services/voice/errors.d.ts.map +1 -0
- package/src/services/voice/errors.ts +32 -0
- package/src/services/voice/expressive-tags.d.ts +158 -0
- package/src/services/voice/expressive-tags.d.ts.map +1 -0
- package/src/services/voice/expressive-tags.ts +405 -0
- package/src/services/voice/ffi-bindings.d.ts +674 -0
- package/src/services/voice/ffi-bindings.d.ts.map +1 -0
- package/src/services/voice/ffi-bindings.test.ts +728 -0
- package/src/services/voice/ffi-bindings.ts +3225 -0
- package/src/services/voice/first-line-cache.d.ts +181 -0
- package/src/services/voice/first-line-cache.d.ts.map +1 -0
- package/src/services/voice/first-line-cache.ts +725 -0
- package/src/services/voice/fused-eot-scorer.d.ts +51 -0
- package/src/services/voice/fused-eot-scorer.d.ts.map +1 -0
- package/src/services/voice/fused-eot-scorer.ts +135 -0
- package/src/services/voice/index.d.ts +91 -0
- package/src/services/voice/index.d.ts.map +1 -0
- package/src/services/voice/index.ts +481 -0
- package/src/services/voice/kokoro/__tests__/kokoro-backend.test.ts +151 -0
- package/src/services/voice/kokoro/__tests__/kokoro-engine-bridge.real.test.ts +151 -0
- package/src/services/voice/kokoro/__tests__/kokoro-engine-bridge.test.ts +60 -0
- package/src/services/voice/kokoro/__tests__/kokoro-engine-discovery.test.ts +277 -0
- package/src/services/voice/kokoro/__tests__/kokoro-ffi-runtime.test.ts +235 -0
- package/src/services/voice/kokoro/__tests__/kokoro-runtime.test.ts +95 -0
- package/src/services/voice/kokoro/__tests__/phonemizer.test.ts +53 -0
- package/src/services/voice/kokoro/__tests__/runtime-selection.test.ts +231 -0
- package/src/services/voice/kokoro/__tests__/voices.test.ts +57 -0
- package/src/services/voice/kokoro/index.ts +79 -0
- package/src/services/voice/kokoro/kokoro-backend.d.ts +72 -0
- package/src/services/voice/kokoro/kokoro-backend.d.ts.map +1 -0
- package/src/services/voice/kokoro/kokoro-backend.ts +207 -0
- package/src/services/voice/kokoro/kokoro-engine-discovery.d.ts +58 -0
- package/src/services/voice/kokoro/kokoro-engine-discovery.d.ts.map +1 -0
- package/src/services/voice/kokoro/kokoro-engine-discovery.ts +177 -0
- package/src/services/voice/kokoro/kokoro-ffi-runtime.d.ts +75 -0
- package/src/services/voice/kokoro/kokoro-ffi-runtime.d.ts.map +1 -0
- package/src/services/voice/kokoro/kokoro-ffi-runtime.ts +233 -0
- package/src/services/voice/kokoro/kokoro-runtime.d.ts +100 -0
- package/src/services/voice/kokoro/kokoro-runtime.d.ts.map +1 -0
- package/src/services/voice/kokoro/kokoro-runtime.ts +170 -0
- package/src/services/voice/kokoro/phoneme-stream.ts +123 -0
- package/src/services/voice/kokoro/phonemizer.d.ts +50 -0
- package/src/services/voice/kokoro/phonemizer.d.ts.map +1 -0
- package/src/services/voice/kokoro/phonemizer.ts +344 -0
- package/src/services/voice/kokoro/pick-runtime.d.ts +61 -0
- package/src/services/voice/kokoro/pick-runtime.d.ts.map +1 -0
- package/src/services/voice/kokoro/pick-runtime.test.ts +91 -0
- package/src/services/voice/kokoro/pick-runtime.ts +130 -0
- package/src/services/voice/kokoro/runtime-selection.d.ts +92 -0
- package/src/services/voice/kokoro/runtime-selection.d.ts.map +1 -0
- package/src/services/voice/kokoro/runtime-selection.ts +237 -0
- package/src/services/voice/kokoro/types.d.ts +82 -0
- package/src/services/voice/kokoro/types.d.ts.map +1 -0
- package/src/services/voice/kokoro/types.ts +95 -0
- package/src/services/voice/kokoro/voice-presets.d.ts +23 -0
- package/src/services/voice/kokoro/voice-presets.d.ts.map +1 -0
- package/src/services/voice/kokoro/voice-presets.ts +129 -0
- package/src/services/voice/kokoro/voices.d.ts +30 -0
- package/src/services/voice/kokoro/voices.d.ts.map +1 -0
- package/src/services/voice/kokoro/voices.ts +64 -0
- package/src/services/voice/lifecycle.d.ts +135 -0
- package/src/services/voice/lifecycle.d.ts.map +1 -0
- package/src/services/voice/lifecycle.test.ts +315 -0
- package/src/services/voice/lifecycle.ts +301 -0
- package/src/services/voice/live-diarization-session.d.ts +96 -0
- package/src/services/voice/live-diarization-session.d.ts.map +1 -0
- package/src/services/voice/live-diarization-session.ts +289 -0
- package/src/services/voice/mic-source.d.ts +136 -0
- package/src/services/voice/mic-source.d.ts.map +1 -0
- package/src/services/voice/mic-source.test.ts +210 -0
- package/src/services/voice/mic-source.ts +503 -0
- package/src/services/voice/optimistic-policy.d.ts +109 -0
- package/src/services/voice/optimistic-policy.d.ts.map +1 -0
- package/src/services/voice/optimistic-policy.test.ts +101 -0
- package/src/services/voice/optimistic-policy.ts +192 -0
- package/src/services/voice/optimistic-rollback.ts +343 -0
- package/src/services/voice/partial-stabilizer.d.ts +73 -0
- package/src/services/voice/partial-stabilizer.d.ts.map +1 -0
- package/src/services/voice/partial-stabilizer.test.ts +68 -0
- package/src/services/voice/partial-stabilizer.ts +140 -0
- package/src/services/voice/phoneme-tokenizer.d.ts +49 -0
- package/src/services/voice/phoneme-tokenizer.d.ts.map +1 -0
- package/src/services/voice/phoneme-tokenizer.ts +158 -0
- package/src/services/voice/phrase-cache.d.ts +76 -0
- package/src/services/voice/phrase-cache.d.ts.map +1 -0
- package/src/services/voice/phrase-cache.test.ts +242 -0
- package/src/services/voice/phrase-cache.ts +186 -0
- package/src/services/voice/phrase-chunker.d.ts +62 -0
- package/src/services/voice/phrase-chunker.d.ts.map +1 -0
- package/src/services/voice/phrase-chunker.test.ts +239 -0
- package/src/services/voice/phrase-chunker.ts +281 -0
- package/src/services/voice/pipeline-impls.d.ts +151 -0
- package/src/services/voice/pipeline-impls.d.ts.map +1 -0
- package/src/services/voice/pipeline-impls.l6.test.ts +110 -0
- package/src/services/voice/pipeline-impls.test.ts +292 -0
- package/src/services/voice/pipeline-impls.ts +315 -0
- package/src/services/voice/pipeline.d.ts +216 -0
- package/src/services/voice/pipeline.d.ts.map +1 -0
- package/src/services/voice/pipeline.ts +505 -0
- package/src/services/voice/prefill-client.d.ts +123 -0
- package/src/services/voice/prefill-client.d.ts.map +1 -0
- package/src/services/voice/prefill-client.ts +316 -0
- package/src/services/voice/prefix-preserving-queue.d.ts +113 -0
- package/src/services/voice/prefix-preserving-queue.d.ts.map +1 -0
- package/src/services/voice/prefix-preserving-queue.ts +162 -0
- package/src/services/voice/profile-store.d.ts +248 -0
- package/src/services/voice/profile-store.d.ts.map +1 -0
- package/src/services/voice/profile-store.ts +887 -0
- package/src/services/voice/real-audio-decode.test.ts +148 -0
- package/src/services/voice/ring-buffer.d.ts +40 -0
- package/src/services/voice/ring-buffer.d.ts.map +1 -0
- package/src/services/voice/ring-buffer.test.ts +129 -0
- package/src/services/voice/ring-buffer.ts +123 -0
- package/src/services/voice/rollback-queue.d.ts +24 -0
- package/src/services/voice/rollback-queue.d.ts.map +1 -0
- package/src/services/voice/rollback-queue.ts +74 -0
- package/src/services/voice/samantha-preset-placeholder.d.ts +67 -0
- package/src/services/voice/samantha-preset-placeholder.d.ts.map +1 -0
- package/src/services/voice/samantha-preset-placeholder.test.ts +97 -0
- package/src/services/voice/samantha-preset-placeholder.ts +148 -0
- package/src/services/voice/samantha-preset-regenerator.d.ts +87 -0
- package/src/services/voice/samantha-preset-regenerator.d.ts.map +1 -0
- package/src/services/voice/samantha-preset-regenerator.ts +393 -0
- package/src/services/voice/scheduler.d.ts +146 -0
- package/src/services/voice/scheduler.d.ts.map +1 -0
- package/src/services/voice/scheduler.t2.test.ts +141 -0
- package/src/services/voice/scheduler.ts +927 -0
- package/src/services/voice/shared-resources.d.ts +190 -0
- package/src/services/voice/shared-resources.d.ts.map +1 -0
- package/src/services/voice/shared-resources.ts +320 -0
- package/src/services/voice/speaker/attribution-pipeline.d.ts +74 -0
- package/src/services/voice/speaker/attribution-pipeline.d.ts.map +1 -0
- package/src/services/voice/speaker/attribution-pipeline.ts +386 -0
- package/src/services/voice/speaker/diarizer-fused.d.ts +59 -0
- package/src/services/voice/speaker/diarizer-fused.d.ts.map +1 -0
- package/src/services/voice/speaker/diarizer-fused.real.test.ts +100 -0
- package/src/services/voice/speaker/diarizer-fused.ts +154 -0
- package/src/services/voice/speaker/diarizer.d.ts +75 -0
- package/src/services/voice/speaker/diarizer.d.ts.map +1 -0
- package/src/services/voice/speaker/diarizer.ts +218 -0
- package/src/services/voice/speaker/encoder-fused.d.ts +60 -0
- package/src/services/voice/speaker/encoder-fused.d.ts.map +1 -0
- package/src/services/voice/speaker/encoder-fused.real.test.ts +113 -0
- package/src/services/voice/speaker/encoder-fused.ts +138 -0
- package/src/services/voice/speaker/encoder-ggml.d.ts +33 -0
- package/src/services/voice/speaker/encoder-ggml.d.ts.map +1 -0
- package/src/services/voice/speaker/encoder-ggml.ts +79 -0
- package/src/services/voice/speaker/encoder.d.ts +37 -0
- package/src/services/voice/speaker/encoder.d.ts.map +1 -0
- package/src/services/voice/speaker/encoder.ts +105 -0
- package/src/services/voice/speaker-imprint.d.ts +83 -0
- package/src/services/voice/speaker-imprint.d.ts.map +1 -0
- package/src/services/voice/speaker-imprint.test.ts +185 -0
- package/src/services/voice/speaker-imprint.ts +312 -0
- package/src/services/voice/speaker-preset-cache.d.ts +77 -0
- package/src/services/voice/speaker-preset-cache.d.ts.map +1 -0
- package/src/services/voice/speaker-preset-cache.test.ts +154 -0
- package/src/services/voice/speaker-preset-cache.ts +195 -0
- package/src/services/voice/streaming-asr/streaming-pipeline-adapter.ts +292 -0
- package/src/services/voice/system-audio-sink.d.ts +73 -0
- package/src/services/voice/system-audio-sink.d.ts.map +1 -0
- package/src/services/voice/system-audio-sink.test.ts +29 -0
- package/src/services/voice/system-audio-sink.ts +366 -0
- package/src/services/voice/transcriber.d.ts +244 -0
- package/src/services/voice/transcriber.d.ts.map +1 -0
- package/src/services/voice/transcriber.test.ts +392 -0
- package/src/services/voice/transcriber.ts +704 -0
- package/src/services/voice/transcript-knowledge.d.ts +37 -0
- package/src/services/voice/transcript-knowledge.d.ts.map +1 -0
- package/src/services/voice/transcript-knowledge.test.ts +68 -0
- package/src/services/voice/transcript-knowledge.ts +75 -0
- package/src/services/voice/transcript-service.d.ts +41 -0
- package/src/services/voice/transcript-service.d.ts.map +1 -0
- package/src/services/voice/transcript-service.test.ts +137 -0
- package/src/services/voice/transcript-service.ts +141 -0
- package/src/services/voice/transcript-store.d.ts +53 -0
- package/src/services/voice/transcript-store.d.ts.map +1 -0
- package/src/services/voice/transcript-store.test.ts +153 -0
- package/src/services/voice/transcript-store.ts +132 -0
- package/src/services/voice/turn-controller.d.ts +183 -0
- package/src/services/voice/turn-controller.d.ts.map +1 -0
- package/src/services/voice/turn-controller.test.ts +575 -0
- package/src/services/voice/turn-controller.ts +596 -0
- package/src/services/voice/types.d.ts +643 -0
- package/src/services/voice/types.d.ts.map +1 -0
- package/src/services/voice/types.ts +699 -0
- package/src/services/voice/vad.d.ts +282 -0
- package/src/services/voice/vad.d.ts.map +1 -0
- package/src/services/voice/vad.test.ts +480 -0
- package/src/services/voice/vad.ts +827 -0
- package/src/services/voice/vad.v1-v4.test.ts +222 -0
- package/src/services/voice/voice-budget.d.ts +241 -0
- package/src/services/voice/voice-budget.d.ts.map +1 -0
- package/src/services/voice/voice-budget.test.ts +418 -0
- package/src/services/voice/voice-budget.ts +635 -0
- package/src/services/voice/voice-duet.test.ts +375 -0
- package/src/services/voice/voice-emotion-classifier.d.ts +95 -0
- package/src/services/voice/voice-emotion-classifier.d.ts.map +1 -0
- package/src/services/voice/voice-emotion-classifier.test.ts +210 -0
- package/src/services/voice/voice-emotion-classifier.ts +273 -0
- package/src/services/voice/voice-preset-format.d.ts +158 -0
- package/src/services/voice/voice-preset-format.d.ts.map +1 -0
- package/src/services/voice/voice-preset-format.ts +700 -0
- package/src/services/voice/voice-preset-generator.test.ts +89 -0
- package/src/services/voice/voice-profile-artifact.d.ts +116 -0
- package/src/services/voice/voice-profile-artifact.d.ts.map +1 -0
- package/src/services/voice/voice-profile-artifact.test.ts +138 -0
- package/src/services/voice/voice-profile-artifact.ts +518 -0
- package/src/services/voice/voice-profile-routes.d.ts +83 -0
- package/src/services/voice/voice-profile-routes.d.ts.map +1 -0
- package/src/services/voice/voice-profile-routes.test.ts +429 -0
- package/src/services/voice/voice-profile-routes.ts +425 -0
- package/src/services/voice/voice-scenario.ts +154 -0
- package/src/services/voice/voice-settings.d.ts +82 -0
- package/src/services/voice/voice-settings.d.ts.map +1 -0
- package/src/services/voice/voice-settings.ts +172 -0
- package/src/services/voice/voice-state-machine.d.ts +364 -0
- package/src/services/voice/voice-state-machine.d.ts.map +1 -0
- package/src/services/voice/voice-state-machine.ts +727 -0
- package/src/services/voice/voice-workbench-report.test.ts +168 -0
- package/src/services/voice/voice-workbench-report.ts +326 -0
- package/src/services/voice/voice-workbench.test.ts +158 -0
- package/src/services/voice/voice.test.ts +1070 -0
- package/src/services/voice/wake-word-ggml.d.ts +101 -0
- package/src/services/voice/wake-word-ggml.d.ts.map +1 -0
- package/src/services/voice/wake-word-ggml.ts +320 -0
- package/src/services/voice/wake-word.d.ts +255 -0
- package/src/services/voice/wake-word.d.ts.map +1 -0
- package/src/services/voice/wake-word.test.ts +298 -0
- package/src/services/voice/wake-word.ts +554 -0
- package/src/services/voice/wrap-with-first-line-cache.d.ts +70 -0
- package/src/services/voice/wrap-with-first-line-cache.d.ts.map +1 -0
- package/src/services/voice/wrap-with-first-line-cache.ts +267 -0
- package/src/services/voice-model-updater.d.ts +240 -0
- package/src/services/voice-model-updater.d.ts.map +1 -0
- package/src/services/voice-model-updater.ts +724 -0
- package/src/services/voice-prewarm.d.ts +3 -0
- package/src/services/voice-prewarm.d.ts.map +1 -0
- package/src/services/voice-prewarm.ts +51 -0
- package/dist/index.d.ts +0 -37
- package/dist/index.js +0 -1098
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module plugin-local-inference/actions/identify-speaker
|
|
3
|
+
*
|
|
4
|
+
* `IDENTIFY_SPEAKER` agent action — the explicit, user-driven half of the
|
|
5
|
+
* voice → entity binding (issue #8234, shape #2).
|
|
6
|
+
*
|
|
7
|
+
* When the OWNER names a voice the agent just heard but hasn't identified
|
|
8
|
+
* ("that was Jill", "this is my friend Sam"), this action binds the most
|
|
9
|
+
* recently observed *unidentified* speaker profile to a named person. It
|
|
10
|
+
* does not touch the entity graph directly — it emits `VOICE_TURN_OBSERVED`
|
|
11
|
+
* so the merge engine (plugin-lifeops) creates/merges the Entity, then the
|
|
12
|
+
* round-trip `VOICE_ENTITY_BOUND` handler persists `entityId` back onto the
|
|
13
|
+
* voice profile. If no merge-engine plugin is loaded the action is inert
|
|
14
|
+
* beyond logging intent.
|
|
15
|
+
*
|
|
16
|
+
* Target selection: an explicit `profileId` option wins; otherwise the
|
|
17
|
+
* single most-recently-observed profile whose `entityId` is still `null`
|
|
18
|
+
* (i.e. "the person who just spoke and isn't known yet").
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import {
|
|
22
|
+
type Action,
|
|
23
|
+
type ActionResult,
|
|
24
|
+
type HandlerCallback,
|
|
25
|
+
type IAgentRuntime,
|
|
26
|
+
logger,
|
|
27
|
+
type Memory,
|
|
28
|
+
} from "@elizaos/core";
|
|
29
|
+
import {
|
|
30
|
+
emitVoiceTurnObserved,
|
|
31
|
+
getVoiceProfileStore,
|
|
32
|
+
} from "../runtime/voice-entity-binding.js";
|
|
33
|
+
import type { VoiceProfileRecord } from "../services/voice/profile-store.js";
|
|
34
|
+
|
|
35
|
+
function extractMessageText(message: Memory | null | undefined): string {
|
|
36
|
+
const content = message?.content;
|
|
37
|
+
if (!content) return "";
|
|
38
|
+
const text = (content as { text?: unknown }).text;
|
|
39
|
+
return typeof text === "string" ? text : "";
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Extract a person name the owner is attaching to a heard voice. Mirrors
|
|
44
|
+
* the trigger phrases lifeops' `extractSelfNameClaim` understands so the
|
|
45
|
+
* downstream entity gets the same `preferredName`. Returns the name or
|
|
46
|
+
* `null`.
|
|
47
|
+
*/
|
|
48
|
+
const NAME = "[A-Z][A-Za-z'.-]{1,40}(?:\\s+[A-Z][A-Za-z'.-]{1,40}){0,2}";
|
|
49
|
+
const SPEAKER_NAME_PATTERNS: RegExp[] = [
|
|
50
|
+
new RegExp(`\\bthat\\s+(?:was|is)\\s+(${NAME})\\b`, "i"),
|
|
51
|
+
new RegExp(`\\bthis\\s+is\\s+(?:my\\s+\\w+\\s+)?(${NAME})\\b`, "i"),
|
|
52
|
+
new RegExp(`\\bcall\\s+(?:him|her|them)\\s+(${NAME})\\b`, "i"),
|
|
53
|
+
new RegExp(`\\b(?:his|her|their)\\s+name\\s+is\\s+(${NAME})\\b`, "i"),
|
|
54
|
+
new RegExp(`\\bnamed?\\s+(${NAME})\\b`, "i"),
|
|
55
|
+
new RegExp(`\\bspeaker\\s+(?:was|is)\\s+(${NAME})\\b`, "i"),
|
|
56
|
+
];
|
|
57
|
+
|
|
58
|
+
export function extractSpeakerName(text: string): string | null {
|
|
59
|
+
for (const pattern of SPEAKER_NAME_PATTERNS) {
|
|
60
|
+
const m = pattern.exec(text);
|
|
61
|
+
if (m?.[1]) {
|
|
62
|
+
const cleaned = m[1].replace(/[.,;:!?]+$/, "").trim();
|
|
63
|
+
if (cleaned.length > 0) return cleaned;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function pickTarget(
|
|
70
|
+
records: VoiceProfileRecord[],
|
|
71
|
+
explicitProfileId: string | null,
|
|
72
|
+
): VoiceProfileRecord | null {
|
|
73
|
+
if (explicitProfileId) {
|
|
74
|
+
return records.find((r) => r.profileId === explicitProfileId) ?? null;
|
|
75
|
+
}
|
|
76
|
+
const unbound = records
|
|
77
|
+
.filter((r) => r.entityId === null)
|
|
78
|
+
.sort((a, b) => b.lastObservedAt.localeCompare(a.lastObservedAt));
|
|
79
|
+
return unbound[0] ?? null;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function optionString(options: unknown, key: string): string | null {
|
|
83
|
+
if (!options || typeof options !== "object") return null;
|
|
84
|
+
const value = (options as Record<string, unknown>)[key];
|
|
85
|
+
return typeof value === "string" && value.trim().length > 0
|
|
86
|
+
? value.trim()
|
|
87
|
+
: null;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async function handler(
|
|
91
|
+
runtime: IAgentRuntime,
|
|
92
|
+
message: Memory,
|
|
93
|
+
_state?: unknown,
|
|
94
|
+
options?: unknown,
|
|
95
|
+
callback?: HandlerCallback,
|
|
96
|
+
): Promise<ActionResult> {
|
|
97
|
+
const text = extractMessageText(message);
|
|
98
|
+
const name = optionString(options, "name") ?? extractSpeakerName(text);
|
|
99
|
+
if (!name) {
|
|
100
|
+
const out =
|
|
101
|
+
'Tell me the name to attach to the voice you just heard — e.g. "that was Jill".';
|
|
102
|
+
await callback?.({ text: out });
|
|
103
|
+
return { success: false, text: out };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const store = await getVoiceProfileStore();
|
|
107
|
+
const records = await store.list();
|
|
108
|
+
const target = pickTarget(records, optionString(options, "profileId"));
|
|
109
|
+
if (!target) {
|
|
110
|
+
const out = `I don't have an unidentified recent voice to attach "${name}" to yet.`;
|
|
111
|
+
await callback?.({ text: out });
|
|
112
|
+
return { success: false, text: out };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Drive the merge engine through the event seam. `emitEvent` awaits all
|
|
116
|
+
// handlers, so when this resolves the round-trip binding has run.
|
|
117
|
+
try {
|
|
118
|
+
await emitVoiceTurnObserved(runtime, {
|
|
119
|
+
turnId: typeof message.id === "string" ? message.id : undefined,
|
|
120
|
+
text: `This is ${name}.`,
|
|
121
|
+
imprintClusterId: target.imprintClusterId,
|
|
122
|
+
matchConfidence: 1,
|
|
123
|
+
matchedEntityId: null,
|
|
124
|
+
isOwner: false,
|
|
125
|
+
});
|
|
126
|
+
} catch (err) {
|
|
127
|
+
logger.error(
|
|
128
|
+
{ err, profileId: target.profileId, name },
|
|
129
|
+
"[IDENTIFY_SPEAKER] failed to emit voice-turn observation",
|
|
130
|
+
);
|
|
131
|
+
const out = `I couldn't save ${name}'s voice just now.`;
|
|
132
|
+
await callback?.({ text: out });
|
|
133
|
+
return { success: false, text: out, error: out };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const updated = await store.get(target.profileId);
|
|
137
|
+
const entityId = updated?.entityId ?? null;
|
|
138
|
+
const out = entityId
|
|
139
|
+
? `Got it — I'll remember ${name}'s voice from now on.`
|
|
140
|
+
: `Noted ${name}. I'll bind the voice once identity sync is available.`;
|
|
141
|
+
await callback?.({ text: out });
|
|
142
|
+
return {
|
|
143
|
+
success: true,
|
|
144
|
+
text: out,
|
|
145
|
+
data: {
|
|
146
|
+
profileId: target.profileId,
|
|
147
|
+
imprintClusterId: target.imprintClusterId,
|
|
148
|
+
entityId,
|
|
149
|
+
name,
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
async function validate(
|
|
155
|
+
_runtime: IAgentRuntime,
|
|
156
|
+
message: Memory,
|
|
157
|
+
): Promise<boolean> {
|
|
158
|
+
return extractMessageText(message).trim().length > 0;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export const identifySpeakerAction: Action = {
|
|
162
|
+
name: "IDENTIFY_SPEAKER",
|
|
163
|
+
similes: ["NAME_SPEAKER", "REMEMBER_VOICE", "THIS_IS_SPEAKER", "TAG_VOICE"],
|
|
164
|
+
description:
|
|
165
|
+
'Attach a name to the most recently heard, still-unidentified voice so the agent recognizes that person across sessions. Use when the owner says who a recent speaker is ("that was Jill", "this is my friend Sam").',
|
|
166
|
+
routingHint:
|
|
167
|
+
"owner names a recent unknown speaker -> IDENTIFY_SPEAKER; not for naming the owner themselves or contacts unrelated to a heard voice",
|
|
168
|
+
validate,
|
|
169
|
+
handler,
|
|
170
|
+
examples: [],
|
|
171
|
+
};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* START_TRANSCRIPTION / STOP_TRANSCRIPTION agent actions (#8789).
|
|
3
|
+
*
|
|
4
|
+
* The agent can turn long-form voice transcription on/off on the user's device.
|
|
5
|
+
* Transcription CAPTURE is client-side (the mic lives in the renderer shell), so
|
|
6
|
+
* the action can't toggle it directly — it emits a one-way `voice-control`
|
|
7
|
+
* command on the AgentEventService bus. That stream is forwarded to every
|
|
8
|
+
* connected client as an `agent_event`; the renderer re-dispatches it to the
|
|
9
|
+
* shell's capture toggle (the same server→client pattern as `shell:navigate`).
|
|
10
|
+
* Best-effort + non-blocking: if no client is connected the event is simply
|
|
11
|
+
* dropped, and the action reports intent, not capture success.
|
|
12
|
+
*/
|
|
13
|
+
import { type Action, type IAgentRuntime } from "@elizaos/core";
|
|
14
|
+
/** Bus stream the renderer subscribes to for mic/transcription control. */
|
|
15
|
+
export declare const VOICE_CONTROL_STREAM = "voice-control";
|
|
16
|
+
export type VoiceControlCommand = "start" | "stop";
|
|
17
|
+
/** The payload shape the client matches on. */
|
|
18
|
+
export interface VoiceControlEvent {
|
|
19
|
+
type: "voice-control";
|
|
20
|
+
command: VoiceControlCommand;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Emit a transcription control command to connected clients. Returns false when
|
|
24
|
+
* no event bus is available (e.g. headless) so callers can report honestly.
|
|
25
|
+
*/
|
|
26
|
+
export declare function emitVoiceControl(runtime: IAgentRuntime, command: VoiceControlCommand): boolean;
|
|
27
|
+
export declare const startTranscriptionAction: Action;
|
|
28
|
+
export declare const stopTranscriptionAction: Action;
|
|
29
|
+
//# sourceMappingURL=transcription-control.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transcription-control.d.ts","sourceRoot":"","sources":["transcription-control.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EACN,KAAK,MAAM,EAGX,KAAK,aAAa,EAKlB,MAAM,eAAe,CAAC;AAEvB,2EAA2E;AAC3E,eAAO,MAAM,oBAAoB,kBAAkB,CAAC;AAEpD,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,MAAM,CAAC;AAEnD,+CAA+C;AAC/C,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,eAAe,CAAC;IACtB,OAAO,EAAE,mBAAmB,CAAC;CAC7B;AAYD;;;GAGG;AACH,wBAAgB,gBAAgB,CAC/B,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE,mBAAmB,GAC1B,OAAO,CAWT;AAgCD,eAAO,MAAM,wBAAwB,EAAE,MAetC,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,MAcrC,CAAC"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import type { IAgentRuntime, Memory } from "@elizaos/core";
|
|
2
|
+
import { describe, expect, it, vi } from "vitest";
|
|
3
|
+
import {
|
|
4
|
+
emitVoiceControl,
|
|
5
|
+
startTranscriptionAction,
|
|
6
|
+
stopTranscriptionAction,
|
|
7
|
+
VOICE_CONTROL_STREAM,
|
|
8
|
+
type VoiceControlEvent,
|
|
9
|
+
} from "./transcription-control";
|
|
10
|
+
|
|
11
|
+
function fakeRuntime(opts: { withBus: boolean }): {
|
|
12
|
+
runtime: IAgentRuntime;
|
|
13
|
+
emit: ReturnType<typeof vi.fn>;
|
|
14
|
+
} {
|
|
15
|
+
const emit = vi.fn();
|
|
16
|
+
const bus = { emit };
|
|
17
|
+
const runtime = {
|
|
18
|
+
agentId: "agent-1",
|
|
19
|
+
getService: (name: string) =>
|
|
20
|
+
name === "agent_event" && opts.withBus ? bus : null,
|
|
21
|
+
} as unknown as IAgentRuntime;
|
|
22
|
+
return { runtime, emit };
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
describe("emitVoiceControl", () => {
|
|
26
|
+
it("emits a voice-control command on the agent-event bus", () => {
|
|
27
|
+
const { runtime, emit } = fakeRuntime({ withBus: true });
|
|
28
|
+
expect(emitVoiceControl(runtime, "start")).toBe(true);
|
|
29
|
+
expect(emit).toHaveBeenCalledTimes(1);
|
|
30
|
+
const event = emit.mock.calls[0][0];
|
|
31
|
+
expect(event.stream).toBe(VOICE_CONTROL_STREAM);
|
|
32
|
+
expect(event.agentId).toBe("agent-1");
|
|
33
|
+
expect(event.data as VoiceControlEvent).toEqual({
|
|
34
|
+
type: "voice-control",
|
|
35
|
+
command: "start",
|
|
36
|
+
});
|
|
37
|
+
expect(typeof event.runId).toBe("string");
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it("returns false when no event bus is available", () => {
|
|
41
|
+
const { runtime, emit } = fakeRuntime({ withBus: false });
|
|
42
|
+
expect(emitVoiceControl(runtime, "stop")).toBe(false);
|
|
43
|
+
expect(emit).not.toHaveBeenCalled();
|
|
44
|
+
});
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
describe("START_/STOP_TRANSCRIPTION actions", () => {
|
|
48
|
+
it("validate only when the event bus is present", async () => {
|
|
49
|
+
const withBus = fakeRuntime({ withBus: true });
|
|
50
|
+
const without = fakeRuntime({ withBus: false });
|
|
51
|
+
expect(
|
|
52
|
+
await startTranscriptionAction.validate(withBus.runtime, {} as Memory),
|
|
53
|
+
).toBe(true);
|
|
54
|
+
expect(
|
|
55
|
+
await stopTranscriptionAction.validate(without.runtime, {} as Memory),
|
|
56
|
+
).toBe(false);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it("START emits 'start' and confirms via the callback", async () => {
|
|
60
|
+
const { runtime, emit } = fakeRuntime({ withBus: true });
|
|
61
|
+
const callback = vi.fn();
|
|
62
|
+
const result = await startTranscriptionAction.handler(
|
|
63
|
+
runtime,
|
|
64
|
+
{} as Memory,
|
|
65
|
+
undefined,
|
|
66
|
+
undefined,
|
|
67
|
+
callback,
|
|
68
|
+
);
|
|
69
|
+
expect(emit.mock.calls[0][0].data.command).toBe("start");
|
|
70
|
+
expect(result).toMatchObject({
|
|
71
|
+
success: true,
|
|
72
|
+
text: "Starting transcription.",
|
|
73
|
+
});
|
|
74
|
+
expect(callback).toHaveBeenCalledWith(
|
|
75
|
+
expect.objectContaining({ actions: ["START_TRANSCRIPTION"] }),
|
|
76
|
+
);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it("STOP emits 'stop'; reports failure honestly with no bus", async () => {
|
|
80
|
+
const ok = fakeRuntime({ withBus: true });
|
|
81
|
+
await stopTranscriptionAction.handler(
|
|
82
|
+
ok.runtime,
|
|
83
|
+
{} as Memory,
|
|
84
|
+
undefined,
|
|
85
|
+
undefined,
|
|
86
|
+
vi.fn(),
|
|
87
|
+
);
|
|
88
|
+
expect(ok.emit.mock.calls[0][0].data.command).toBe("stop");
|
|
89
|
+
|
|
90
|
+
const none = fakeRuntime({ withBus: false });
|
|
91
|
+
const result = await stopTranscriptionAction.handler(
|
|
92
|
+
none.runtime,
|
|
93
|
+
{} as Memory,
|
|
94
|
+
undefined,
|
|
95
|
+
undefined,
|
|
96
|
+
vi.fn(),
|
|
97
|
+
);
|
|
98
|
+
expect(result).toMatchObject({ success: false });
|
|
99
|
+
});
|
|
100
|
+
});
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* START_TRANSCRIPTION / STOP_TRANSCRIPTION agent actions (#8789).
|
|
3
|
+
*
|
|
4
|
+
* The agent can turn long-form voice transcription on/off on the user's device.
|
|
5
|
+
* Transcription CAPTURE is client-side (the mic lives in the renderer shell), so
|
|
6
|
+
* the action can't toggle it directly — it emits a one-way `voice-control`
|
|
7
|
+
* command on the AgentEventService bus. That stream is forwarded to every
|
|
8
|
+
* connected client as an `agent_event`; the renderer re-dispatches it to the
|
|
9
|
+
* shell's capture toggle (the same server→client pattern as `shell:navigate`).
|
|
10
|
+
* Best-effort + non-blocking: if no client is connected the event is simply
|
|
11
|
+
* dropped, and the action reports intent, not capture success.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import {
|
|
15
|
+
type Action,
|
|
16
|
+
type ActionResult,
|
|
17
|
+
type HandlerCallback,
|
|
18
|
+
type IAgentRuntime,
|
|
19
|
+
logger,
|
|
20
|
+
type Memory,
|
|
21
|
+
type Service,
|
|
22
|
+
ServiceType,
|
|
23
|
+
} from "@elizaos/core";
|
|
24
|
+
|
|
25
|
+
/** Bus stream the renderer subscribes to for mic/transcription control. */
|
|
26
|
+
export const VOICE_CONTROL_STREAM = "voice-control";
|
|
27
|
+
|
|
28
|
+
export type VoiceControlCommand = "start" | "stop";
|
|
29
|
+
|
|
30
|
+
/** The payload shape the client matches on. */
|
|
31
|
+
export interface VoiceControlEvent {
|
|
32
|
+
type: "voice-control";
|
|
33
|
+
command: VoiceControlCommand;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** The slice of AgentEventService the actions use (real service satisfies it). */
|
|
37
|
+
interface AgentEventBus extends Service {
|
|
38
|
+
emit(event: {
|
|
39
|
+
runId: string;
|
|
40
|
+
stream: string;
|
|
41
|
+
data: unknown;
|
|
42
|
+
agentId?: string;
|
|
43
|
+
}): void;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Emit a transcription control command to connected clients. Returns false when
|
|
48
|
+
* no event bus is available (e.g. headless) so callers can report honestly.
|
|
49
|
+
*/
|
|
50
|
+
export function emitVoiceControl(
|
|
51
|
+
runtime: IAgentRuntime,
|
|
52
|
+
command: VoiceControlCommand,
|
|
53
|
+
): boolean {
|
|
54
|
+
const bus = runtime.getService<AgentEventBus>(ServiceType.AGENT_EVENT);
|
|
55
|
+
if (!bus) return false;
|
|
56
|
+
const data: VoiceControlEvent = { type: "voice-control", command };
|
|
57
|
+
bus.emit({
|
|
58
|
+
runId: crypto.randomUUID(),
|
|
59
|
+
stream: VOICE_CONTROL_STREAM,
|
|
60
|
+
data,
|
|
61
|
+
agentId: runtime.agentId,
|
|
62
|
+
});
|
|
63
|
+
return true;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async function validate(runtime: IAgentRuntime): Promise<boolean> {
|
|
67
|
+
return runtime.getService(ServiceType.AGENT_EVENT) != null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function makeHandler(
|
|
71
|
+
command: VoiceControlCommand,
|
|
72
|
+
okText: string,
|
|
73
|
+
failText: string,
|
|
74
|
+
actionName: string,
|
|
75
|
+
) {
|
|
76
|
+
return async (
|
|
77
|
+
runtime: IAgentRuntime,
|
|
78
|
+
_message: Memory,
|
|
79
|
+
_state?: unknown,
|
|
80
|
+
_options?: unknown,
|
|
81
|
+
callback?: HandlerCallback,
|
|
82
|
+
): Promise<ActionResult> => {
|
|
83
|
+
const delivered = emitVoiceControl(runtime, command);
|
|
84
|
+
const text = delivered ? okText : failText;
|
|
85
|
+
if (!delivered) {
|
|
86
|
+
logger.warn(
|
|
87
|
+
{ command },
|
|
88
|
+
"[transcription-control] no event bus — command not delivered",
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
await callback?.({ text, actions: [actionName] });
|
|
92
|
+
return { success: delivered, text };
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export const startTranscriptionAction: Action = {
|
|
97
|
+
name: "START_TRANSCRIPTION",
|
|
98
|
+
similes: ["BEGIN_TRANSCRIPTION", "START_RECORDING", "RECORD_TRANSCRIPT"],
|
|
99
|
+
description:
|
|
100
|
+
"Start long-form voice transcription (record-only) on the user's device. Use when the user asks to start transcribing/recording a conversation or meeting.",
|
|
101
|
+
routingHint:
|
|
102
|
+
"user asks to start transcribing/recording -> START_TRANSCRIPTION; not for a one-off dictation or a normal voice reply",
|
|
103
|
+
validate,
|
|
104
|
+
handler: makeHandler(
|
|
105
|
+
"start",
|
|
106
|
+
"Starting transcription.",
|
|
107
|
+
"I couldn't reach the voice client to start transcription.",
|
|
108
|
+
"START_TRANSCRIPTION",
|
|
109
|
+
),
|
|
110
|
+
examples: [],
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
export const stopTranscriptionAction: Action = {
|
|
114
|
+
name: "STOP_TRANSCRIPTION",
|
|
115
|
+
similes: ["END_TRANSCRIPTION", "STOP_RECORDING", "FINISH_TRANSCRIPT"],
|
|
116
|
+
description:
|
|
117
|
+
"Stop the long-form voice transcription currently running on the user's device.",
|
|
118
|
+
routingHint: "user asks to stop transcribing/recording -> STOP_TRANSCRIPTION",
|
|
119
|
+
validate,
|
|
120
|
+
handler: makeHandler(
|
|
121
|
+
"stop",
|
|
122
|
+
"Stopping transcription.",
|
|
123
|
+
"I couldn't reach the voice client to stop transcription.",
|
|
124
|
+
"STOP_TRANSCRIPTION",
|
|
125
|
+
),
|
|
126
|
+
examples: [],
|
|
127
|
+
};
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import type { IAgentRuntime } from "@elizaos/core";
|
|
2
|
+
import { ModelType } from "@elizaos/core";
|
|
3
|
+
import { beforeEach, describe, expect, it, vi } from "vitest";
|
|
4
|
+
import type {
|
|
5
|
+
CapacitorLlamaCompletionParams,
|
|
6
|
+
CapacitorLlamaCompletionResult,
|
|
7
|
+
CapacitorLlamaContext,
|
|
8
|
+
CapacitorLlamaTokenData,
|
|
9
|
+
} from "../types";
|
|
10
|
+
|
|
11
|
+
const mocks = vi.hoisted(() => ({
|
|
12
|
+
initCapacitorLlama: vi.fn(),
|
|
13
|
+
}));
|
|
14
|
+
|
|
15
|
+
vi.mock("../..", () => ({
|
|
16
|
+
createLocalInferenceModelHandlers: vi.fn(() => ({})),
|
|
17
|
+
isLocalInferenceUnavailableError: vi.fn(() => false),
|
|
18
|
+
}));
|
|
19
|
+
|
|
20
|
+
vi.mock("../loader", () => ({
|
|
21
|
+
initCapacitorLlama: mocks.initCapacitorLlama,
|
|
22
|
+
}));
|
|
23
|
+
|
|
24
|
+
const { localAiPlugin } = await import("../index");
|
|
25
|
+
|
|
26
|
+
function makeCompletionResult(
|
|
27
|
+
text: string,
|
|
28
|
+
overrides: Partial<CapacitorLlamaCompletionResult> = {},
|
|
29
|
+
): CapacitorLlamaCompletionResult {
|
|
30
|
+
return {
|
|
31
|
+
text,
|
|
32
|
+
content: text,
|
|
33
|
+
reasoning_content: "",
|
|
34
|
+
tool_calls: [],
|
|
35
|
+
chat_format: 0,
|
|
36
|
+
tokens_predicted: 2,
|
|
37
|
+
tokens_evaluated: 3,
|
|
38
|
+
truncated: false,
|
|
39
|
+
stopped_eos: true,
|
|
40
|
+
stopped_word: "",
|
|
41
|
+
stopped_limit: 0,
|
|
42
|
+
stopping_word: "",
|
|
43
|
+
context_full: false,
|
|
44
|
+
interrupted: false,
|
|
45
|
+
tokens_cached: 0,
|
|
46
|
+
timings: {
|
|
47
|
+
prompt_n: 0,
|
|
48
|
+
prompt_ms: 0,
|
|
49
|
+
prompt_per_token_ms: 0,
|
|
50
|
+
prompt_per_second: 0,
|
|
51
|
+
predicted_n: 2,
|
|
52
|
+
predicted_ms: 0,
|
|
53
|
+
predicted_per_token_ms: 0,
|
|
54
|
+
predicted_per_second: 0,
|
|
55
|
+
},
|
|
56
|
+
...overrides,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function makeCtx(
|
|
61
|
+
onCompletion?: (params: CapacitorLlamaCompletionParams) => void,
|
|
62
|
+
): CapacitorLlamaContext {
|
|
63
|
+
return {
|
|
64
|
+
id: 1,
|
|
65
|
+
gpu: false,
|
|
66
|
+
reasonNoGPU: "",
|
|
67
|
+
model: {} as CapacitorLlamaContext["model"],
|
|
68
|
+
async completion(
|
|
69
|
+
params: CapacitorLlamaCompletionParams,
|
|
70
|
+
callback?: (data: CapacitorLlamaTokenData) => void,
|
|
71
|
+
): Promise<CapacitorLlamaCompletionResult> {
|
|
72
|
+
onCompletion?.(params);
|
|
73
|
+
callback?.({ token: "hel" });
|
|
74
|
+
callback?.({ token: "lo" });
|
|
75
|
+
return makeCompletionResult("hello");
|
|
76
|
+
},
|
|
77
|
+
stopCompletion: vi.fn(async () => undefined),
|
|
78
|
+
tokenize: vi.fn(async () => ({
|
|
79
|
+
tokens: [],
|
|
80
|
+
has_images: false,
|
|
81
|
+
bitmap_hashes: [],
|
|
82
|
+
chunk_pos: [],
|
|
83
|
+
chunk_pos_images: [],
|
|
84
|
+
})),
|
|
85
|
+
detokenize: vi.fn(async () => ""),
|
|
86
|
+
embedding: vi.fn(async (text: string) => ({
|
|
87
|
+
embedding: text === "embed me" ? [0.1, 0.2, 0.3] : [1],
|
|
88
|
+
})),
|
|
89
|
+
bench: vi.fn(async () => ({
|
|
90
|
+
modelDesc: "",
|
|
91
|
+
modelSize: 0,
|
|
92
|
+
modelNParams: 0,
|
|
93
|
+
ppAvg: 0,
|
|
94
|
+
ppStd: 0,
|
|
95
|
+
tgAvg: 0,
|
|
96
|
+
tgStd: 0,
|
|
97
|
+
})),
|
|
98
|
+
release: vi.fn(async () => undefined),
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function makeRuntime(): IAgentRuntime {
|
|
103
|
+
return {
|
|
104
|
+
getSetting: vi.fn(() => undefined),
|
|
105
|
+
emitEvent: vi.fn(async () => undefined),
|
|
106
|
+
} as unknown as IAgentRuntime;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
describe("local-ai compat adapter behavior", () => {
|
|
110
|
+
beforeEach(() => {
|
|
111
|
+
vi.clearAllMocks();
|
|
112
|
+
mocks.initCapacitorLlama.mockImplementation(async () => makeCtx());
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it.each([
|
|
116
|
+
null,
|
|
117
|
+
"",
|
|
118
|
+
" ",
|
|
119
|
+
{ text: "" },
|
|
120
|
+
{ text: " " },
|
|
121
|
+
])("rejects empty embedding input %# instead of returning a fake vector", async (params) => {
|
|
122
|
+
await expect(
|
|
123
|
+
localAiPlugin.models?.[ModelType.TEXT_EMBEDDING]?.(
|
|
124
|
+
makeRuntime(),
|
|
125
|
+
params as never,
|
|
126
|
+
),
|
|
127
|
+
).rejects.toThrow("Embedding text must be a non-empty string");
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it("routes non-empty embedding input to the real embedding context", async () => {
|
|
131
|
+
const result = await localAiPlugin.models?.[ModelType.TEXT_EMBEDDING]?.(
|
|
132
|
+
makeRuntime(),
|
|
133
|
+
{ text: "embed me" } as never,
|
|
134
|
+
);
|
|
135
|
+
|
|
136
|
+
expect(result).toEqual([0.1, 0.2, 0.3]);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it("wires onStreamChunk through the compat text adapter", async () => {
|
|
140
|
+
mocks.initCapacitorLlama.mockResolvedValueOnce(makeCtx());
|
|
141
|
+
const onStreamChunk = vi.fn();
|
|
142
|
+
|
|
143
|
+
const result = await localAiPlugin.models?.[ModelType.TEXT_SMALL]?.(
|
|
144
|
+
makeRuntime(),
|
|
145
|
+
{
|
|
146
|
+
prompt: "stream this",
|
|
147
|
+
stream: true,
|
|
148
|
+
onStreamChunk,
|
|
149
|
+
} as never,
|
|
150
|
+
);
|
|
151
|
+
|
|
152
|
+
for await (const _chunk of result.textStream) {
|
|
153
|
+
// drain
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
expect(onStreamChunk).toHaveBeenCalled();
|
|
157
|
+
expect(onStreamChunk.mock.calls.map(([chunk]) => chunk).join("")).toBe(
|
|
158
|
+
"hello",
|
|
159
|
+
);
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
it("sends a desktop-safe prompt and forwards sampler controls", async () => {
|
|
163
|
+
let completionParams: CapacitorLlamaCompletionParams | undefined;
|
|
164
|
+
mocks.initCapacitorLlama.mockResolvedValueOnce(
|
|
165
|
+
makeCtx((params) => {
|
|
166
|
+
completionParams = params;
|
|
167
|
+
}),
|
|
168
|
+
);
|
|
169
|
+
|
|
170
|
+
await localAiPlugin.models?.[ModelType.TEXT_SMALL]?.(makeRuntime(), {
|
|
171
|
+
system: "system prompt",
|
|
172
|
+
prompt: "user prompt",
|
|
173
|
+
maxTokens: 42,
|
|
174
|
+
temperature: 0.2,
|
|
175
|
+
topP: 0.8,
|
|
176
|
+
topK: 17,
|
|
177
|
+
minP: 0.05,
|
|
178
|
+
seed: 1234,
|
|
179
|
+
repetitionPenalty: 1.05,
|
|
180
|
+
frequencyPenalty: 0.3,
|
|
181
|
+
presencePenalty: 0.4,
|
|
182
|
+
stopSequences: ["</s>"],
|
|
183
|
+
} as never);
|
|
184
|
+
|
|
185
|
+
expect(completionParams?.prompt).toContain("system: system prompt");
|
|
186
|
+
expect(completionParams?.prompt).toContain("user: user prompt");
|
|
187
|
+
expect(completionParams?.messages).toBeUndefined();
|
|
188
|
+
expect(completionParams).toMatchObject({
|
|
189
|
+
n_predict: 42,
|
|
190
|
+
temperature: 0.2,
|
|
191
|
+
top_p: 0.8,
|
|
192
|
+
top_k: 17,
|
|
193
|
+
min_p: 0.05,
|
|
194
|
+
seed: 1234,
|
|
195
|
+
penalty_repeat: 1.05,
|
|
196
|
+
penalty_freq: 0.3,
|
|
197
|
+
penalty_present: 0.4,
|
|
198
|
+
stop: ["</s>"],
|
|
199
|
+
});
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
it("preserves function toolChoice objects as a Capacitor tool_choice", async () => {
|
|
203
|
+
let completionParams: CapacitorLlamaCompletionParams | undefined;
|
|
204
|
+
mocks.initCapacitorLlama.mockResolvedValueOnce(
|
|
205
|
+
makeCtx((params) => {
|
|
206
|
+
completionParams = params;
|
|
207
|
+
}),
|
|
208
|
+
);
|
|
209
|
+
|
|
210
|
+
await localAiPlugin.models?.[ModelType.TEXT_SMALL]?.(makeRuntime(), {
|
|
211
|
+
prompt: "use tool",
|
|
212
|
+
tools: [{ name: "lookup" }],
|
|
213
|
+
toolChoice: { type: "function", function: { name: "lookup" } },
|
|
214
|
+
} as never);
|
|
215
|
+
|
|
216
|
+
expect(completionParams?.tool_choice).toBe("lookup");
|
|
217
|
+
});
|
|
218
|
+
});
|