@lovelace_lol/polymer 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +226 -0
- package/dist/index.d.ts +2603 -0
- package/dist/index.js +1031 -0
- package/dist/test.cjs +1519 -0
- package/dist/worker/agency-worker.js +972 -0
- package/dist/worker/manifest.edn +1 -0
- package/package.json +57 -0
- package/shadow-cljs.edn +173 -0
- package/src/polymer/animation/agency.cljs +84 -0
- package/src/polymer/animation/domain.cljs +110 -0
- package/src/polymer/animation/planner.cljs +88 -0
- package/src/polymer/animation/runtime.cljs +137 -0
- package/src/polymer/animation/scheduler.cljs +178 -0
- package/src/polymer/animation/service.cljs +981 -0
- package/src/polymer/animation/state.cljs +112 -0
- package/src/polymer/blink/agency.cljs +131 -0
- package/src/polymer/blink/planner.cljs +115 -0
- package/src/polymer/blink/scheduler.cljs +90 -0
- package/src/polymer/blink/snippet.cljs +50 -0
- package/src/polymer/blink/state.cljs +102 -0
- package/src/polymer/camera_context/agency.cljs +132 -0
- package/src/polymer/camera_context/domain.cljs +186 -0
- package/src/polymer/camera_context/planner.cljs +50 -0
- package/src/polymer/camera_context/scheduler.cljs +113 -0
- package/src/polymer/camera_context/state.cljs +121 -0
- package/src/polymer/character.cljs +471 -0
- package/src/polymer/conversation/agency.cljs +83 -0
- package/src/polymer/conversation/domain.cljs +35 -0
- package/src/polymer/conversation/effector.cljs +130 -0
- package/src/polymer/conversation/planner.cljs +178 -0
- package/src/polymer/conversation/scheduler.cljs +202 -0
- package/src/polymer/conversation/service.cljs +329 -0
- package/src/polymer/conversation/state.cljs +186 -0
- package/src/polymer/core.cljs +172 -0
- package/src/polymer/eye_head/agency.cljs +75 -0
- package/src/polymer/eye_head/domain.cljs +133 -0
- package/src/polymer/eye_head/goap.cljs +106 -0
- package/src/polymer/eye_head/scheduler.cljs +176 -0
- package/src/polymer/eye_head/service.cljs +259 -0
- package/src/polymer/eye_head/snippet.cljs +92 -0
- package/src/polymer/eye_head/state.cljs +167 -0
- package/src/polymer/gaze/agency.cljs +85 -0
- package/src/polymer/gaze/domain.cljs +211 -0
- package/src/polymer/gaze/goap.cljs +208 -0
- package/src/polymer/gaze/scheduler.cljs +184 -0
- package/src/polymer/gaze/state.cljs +202 -0
- package/src/polymer/gesture/agency.cljs +185 -0
- package/src/polymer/gesture/goap.cljs +461 -0
- package/src/polymer/gesture/scheduler.cljs +122 -0
- package/src/polymer/gesture/snippet.cljs +198 -0
- package/src/polymer/gesture/state.cljs +273 -0
- package/src/polymer/lipsync/agency.cljs +593 -0
- package/src/polymer/lipsync/articulation/jaw.cljs +258 -0
- package/src/polymer/lipsync/articulation/snippet.cljs +605 -0
- package/src/polymer/lipsync/articulation/tongue.cljs +418 -0
- package/src/polymer/lipsync/articulation/visemes.cljs +429 -0
- package/src/polymer/lipsync/goap.cljs +132 -0
- package/src/polymer/lipsync/pronunciation/effector.cljs +51 -0
- package/src/polymer/lipsync/scheduler.cljs +302 -0
- package/src/polymer/lipsync/state.cljs +202 -0
- package/src/polymer/nlp/phonemes.cljs +164 -0
- package/src/polymer/prosodic/agency.cljs +196 -0
- package/src/polymer/prosodic/goap.cljs +122 -0
- package/src/polymer/prosodic/scheduler.cljs +99 -0
- package/src/polymer/prosodic/snippet.cljs +106 -0
- package/src/polymer/prosodic/state.cljs +139 -0
- package/src/polymer/runtime/embody.cljs +117 -0
- package/src/polymer/runtime/embody_wasm.cljs +177 -0
- package/src/polymer/runtime/rust_embody_host.cljs +645 -0
- package/src/polymer/society.cljs +239 -0
- package/src/polymer/stream.cljs +130 -0
- package/src/polymer/transcription/agency.cljs +69 -0
- package/src/polymer/transcription/domain.cljs +68 -0
- package/src/polymer/transcription/effector.cljs +611 -0
- package/src/polymer/transcription/planner.cljs +203 -0
- package/src/polymer/transcription/scheduler.cljs +225 -0
- package/src/polymer/transcription/service.cljs +209 -0
- package/src/polymer/transcription/state.cljs +239 -0
- package/src/polymer/tts/agency.cljs +503 -0
- package/src/polymer/tts/azure.cljs +290 -0
- package/src/polymer/tts/planner.cljs +178 -0
- package/src/polymer/tts/runtime.cljs +624 -0
- package/src/polymer/tts/scheduler.cljs +162 -0
- package/src/polymer/tts/state.cljs +191 -0
- package/src/polymer/worker.cljs +70 -0
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
(ns polymer.conversation.service
|
|
2
|
+
(:require [polymer.conversation.agency :as conversation-agency]))
|
|
3
|
+
|
|
4
|
+
;; Compatibility adapter for LoomLarge's old generator-based JavaScript API.
|
|
5
|
+
;; This is not the Conversation agency implementation. It exists so current
|
|
6
|
+
;; callers can keep their historical service shape while the agency owns
|
|
7
|
+
;; conversation state, planning, scheduling, and stream facts.
|
|
8
|
+
;;
|
|
9
|
+
;; Important: the generator (LoomLarge AI Chat) owns LLM replies. The orphan
|
|
10
|
+
;; agency inside this adapter must not auto-respond / wait on a missing
|
|
11
|
+
;; conversation-provider, or turn-taking freezes after the first user utterance.
|
|
12
|
+
;;
|
|
13
|
+
;; Interrupt parity with Latticework:
|
|
14
|
+
;; - Audio barge-in via transcription.onInterruption
|
|
15
|
+
;; - Optional transcript fallback via allowTranscriptInterruptionFallback
|
|
16
|
+
;; - Ignore non-interrupt finals while the agent is still speaking
|
|
17
|
+
;; - Sync TTS playback reference track into transcription for AEC barge-in
|
|
18
|
+
|
|
19
|
+
(defn data-map [value]
|
|
20
|
+
(cond
|
|
21
|
+
(map? value) value
|
|
22
|
+
value (js->clj value :keywordize-keys true)
|
|
23
|
+
:else {}))
|
|
24
|
+
|
|
25
|
+
(defn js-call [target method-name & args]
|
|
26
|
+
(when target
|
|
27
|
+
(when-let [method (aget target method-name)]
|
|
28
|
+
(when (fn? method)
|
|
29
|
+
(.apply method target (to-array args))))))
|
|
30
|
+
|
|
31
|
+
(defn promise-resolve [value]
|
|
32
|
+
(.resolve js/Promise value))
|
|
33
|
+
|
|
34
|
+
(defn promise-reject [value]
|
|
35
|
+
(.reject js/Promise value))
|
|
36
|
+
|
|
37
|
+
(defn now-ms []
|
|
38
|
+
(.now js/Date))
|
|
39
|
+
|
|
40
|
+
(defn service-handles [config]
|
|
41
|
+
;; Keep live JS service handles out of js->clj. Converting them to CLJ maps
|
|
42
|
+
;; breaks later aget/method calls (eye/head setSpeaking became a silent no-op).
|
|
43
|
+
{:eyeHeadTracking (when config (aget config "eyeHeadTracking"))
|
|
44
|
+
:prosodicService (when config (aget config "prosodicService"))})
|
|
45
|
+
|
|
46
|
+
(defn default-config [config]
|
|
47
|
+
(let [handles (service-handles config)
|
|
48
|
+
plain (dissoc (data-map config) :eyeHeadTracking :prosodicService)]
|
|
49
|
+
(merge {:autoListen true
|
|
50
|
+
:detectInterruptions true
|
|
51
|
+
:minSpeakTime 500
|
|
52
|
+
:allowTranscriptInterruptionFallback false
|
|
53
|
+
:eyeHeadTracking (:eyeHeadTracking handles)
|
|
54
|
+
:prosodicService (:prosodicService handles)}
|
|
55
|
+
plain
|
|
56
|
+
handles)))
|
|
57
|
+
|
|
58
|
+
(defn generator-next [flow input has-input?]
|
|
59
|
+
(if has-input?
|
|
60
|
+
(.next flow input)
|
|
61
|
+
(.next flow)))
|
|
62
|
+
|
|
63
|
+
(defn result-value [result]
|
|
64
|
+
(aget result "value"))
|
|
65
|
+
|
|
66
|
+
(defn result-done? [result]
|
|
67
|
+
(boolean (aget result "done")))
|
|
68
|
+
|
|
69
|
+
(defn create-service [tts transcription config callbacks]
|
|
70
|
+
(let [config-atom (atom (default-config config))
|
|
71
|
+
context (atom {:state "idle"
|
|
72
|
+
:lastUserSpeech nil
|
|
73
|
+
:lastAgentSpeech nil
|
|
74
|
+
:isInterrupted false
|
|
75
|
+
:speakStartTime nil})
|
|
76
|
+
;; Generator owns replies; disable agency autoRespond so transcriptFinal
|
|
77
|
+
;; does not emit orphaned conversation.requestResponse with no provider.
|
|
78
|
+
agency (conversation-agency/create-conversation-agency
|
|
79
|
+
#js {:autoRespond false
|
|
80
|
+
:maxHistory (or (:maxHistory @config-atom) 40)})
|
|
81
|
+
flow-atom (atom nil)
|
|
82
|
+
disposed? (atom false)
|
|
83
|
+
unsubscribers (atom [])
|
|
84
|
+
playback-unsub (atom nil)
|
|
85
|
+
agent-speech-active? (atom false)
|
|
86
|
+
pending-interrupted-transcript (atom nil)]
|
|
87
|
+
(letfn [(dispatch! [command]
|
|
88
|
+
(js-call agency "dispatch" (clj->js command)))
|
|
89
|
+
(set-state! [state]
|
|
90
|
+
(swap! context assoc :state state)
|
|
91
|
+
(js-call callbacks "onStateChange" state))
|
|
92
|
+
(eye-head []
|
|
93
|
+
(:eyeHeadTracking @config-atom))
|
|
94
|
+
(prosodic []
|
|
95
|
+
(:prosodicService @config-atom))
|
|
96
|
+
(set-speaking! [value]
|
|
97
|
+
(when-let [service (eye-head)]
|
|
98
|
+
(js-call service "setSpeaking" value)))
|
|
99
|
+
(set-listening! [value]
|
|
100
|
+
(when-let [service (eye-head)]
|
|
101
|
+
(js-call service "setListening" value)))
|
|
102
|
+
(notify-error! [error]
|
|
103
|
+
(js-call callbacks "onError" error))
|
|
104
|
+
(clear-playback-unsub! []
|
|
105
|
+
(when-let [unsubscribe @playback-unsub]
|
|
106
|
+
(unsubscribe)
|
|
107
|
+
(reset! playback-unsub nil)))
|
|
108
|
+
(sync-agent-audio-reference-track! []
|
|
109
|
+
(let [track (or (js-call tts "getPlaybackReferenceTrack") nil)]
|
|
110
|
+
(js-call transcription "setAgentAudioReferenceTrack" track)))
|
|
111
|
+
(start-listening! []
|
|
112
|
+
(when-not @disposed?
|
|
113
|
+
(set-speaking! false)
|
|
114
|
+
(set-listening! true)
|
|
115
|
+
(set-state! "userSpeaking")
|
|
116
|
+
(-> (promise-resolve (js-call transcription "startListening"))
|
|
117
|
+
(.catch (fn [error]
|
|
118
|
+
(when-not @disposed?
|
|
119
|
+
(notify-error! error)))))))
|
|
120
|
+
(stop-listening! []
|
|
121
|
+
(set-listening! false)
|
|
122
|
+
(js-call transcription "stopListening"))
|
|
123
|
+
(arm-interruption-listening! []
|
|
124
|
+
(if (:detectInterruptions @config-atom)
|
|
125
|
+
(-> (promise-resolve (js-call transcription "startListening"))
|
|
126
|
+
(.catch (fn [error]
|
|
127
|
+
(when-not @disposed?
|
|
128
|
+
(notify-error! error)
|
|
129
|
+
nil))))
|
|
130
|
+
(promise-resolve nil)))
|
|
131
|
+
(stop-speaking-behaviors! []
|
|
132
|
+
(set-speaking! false)
|
|
133
|
+
(js-call (prosodic) "stopTalking"))
|
|
134
|
+
(can-interrupt-agent? []
|
|
135
|
+
(and (:detectInterruptions @config-atom)
|
|
136
|
+
(not (:isInterrupted @context))
|
|
137
|
+
(= "agentSpeaking" (:state @context))
|
|
138
|
+
(some? (:speakStartTime @context))
|
|
139
|
+
(>= (- (now-ms) (:speakStartTime @context))
|
|
140
|
+
(:minSpeakTime @config-atom))))
|
|
141
|
+
(handle-interruption! [source event]
|
|
142
|
+
(when (can-interrupt-agent?)
|
|
143
|
+
(js/console.log (str "[ConversationService] Handling interruption from " source))
|
|
144
|
+
(swap! context assoc :isInterrupted true)
|
|
145
|
+
(dispatch! {:type "interrupt"
|
|
146
|
+
:reason source
|
|
147
|
+
:event (data-map event)})
|
|
148
|
+
(js-call tts "stop")
|
|
149
|
+
(stop-speaking-behaviors!)
|
|
150
|
+
(set-state! "interrupted")
|
|
151
|
+
;; Only surface a user-speech callback when we have text (transcript
|
|
152
|
+
;; fallback). Audio barge-in should not emit an empty final utterance.
|
|
153
|
+
(let [text (str (or (:text (data-map event)) ""))]
|
|
154
|
+
(when (pos? (count text))
|
|
155
|
+
(js-call callbacks "onUserSpeech" text true true)))))
|
|
156
|
+
(finish-agent-speech! []
|
|
157
|
+
(clear-playback-unsub!)
|
|
158
|
+
(reset! agent-speech-active? false)
|
|
159
|
+
(stop-speaking-behaviors!)
|
|
160
|
+
(js-call transcription "notifyAgentSpeechEnd")
|
|
161
|
+
(if-let [pending @pending-interrupted-transcript]
|
|
162
|
+
(do
|
|
163
|
+
(reset! pending-interrupted-transcript nil)
|
|
164
|
+
(submit-user-input! pending))
|
|
165
|
+
(cond
|
|
166
|
+
(:isInterrupted @context)
|
|
167
|
+
(start-listening!)
|
|
168
|
+
|
|
169
|
+
(:autoListen @config-atom)
|
|
170
|
+
(start-listening!)
|
|
171
|
+
|
|
172
|
+
:else
|
|
173
|
+
(set-state! "idle"))))
|
|
174
|
+
(speak-agent-text! [text]
|
|
175
|
+
(let [utterance (str text)]
|
|
176
|
+
(reset! agent-speech-active? true)
|
|
177
|
+
(reset! pending-interrupted-transcript nil)
|
|
178
|
+
(swap! context assoc
|
|
179
|
+
:lastAgentSpeech utterance
|
|
180
|
+
:speakStartTime nil
|
|
181
|
+
:isInterrupted false)
|
|
182
|
+
(dispatch! {:type "agentUtterance"
|
|
183
|
+
:text utterance
|
|
184
|
+
:source "conversationService"})
|
|
185
|
+
(js-call callbacks "onAgentUtterance" utterance)
|
|
186
|
+
(sync-agent-audio-reference-track!)
|
|
187
|
+
(js-call transcription "prepareAgentSpeech" utterance)
|
|
188
|
+
(set-listening! false)
|
|
189
|
+
(set-speaking! true)
|
|
190
|
+
(when-let [service (eye-head)]
|
|
191
|
+
(js-call service "setGazeTarget" #js {:x 0 :y 0 :z 0}))
|
|
192
|
+
(js-call (prosodic) "startTalking")
|
|
193
|
+
(set-state! "agentSpeaking")
|
|
194
|
+
(-> (arm-interruption-listening!)
|
|
195
|
+
(.then (fn [_]
|
|
196
|
+
(when-not @disposed?
|
|
197
|
+
(clear-playback-unsub!)
|
|
198
|
+
(when-let [unsubscribe (js-call tts "onPlaybackStart"
|
|
199
|
+
(fn []
|
|
200
|
+
(when-not @disposed?
|
|
201
|
+
(swap! context assoc :speakStartTime (now-ms))
|
|
202
|
+
(sync-agent-audio-reference-track!)
|
|
203
|
+
(js-call transcription "notifyAgentSpeech" utterance))))]
|
|
204
|
+
(reset! playback-unsub unsubscribe))
|
|
205
|
+
(js-call tts "speak" utterance))))
|
|
206
|
+
(.then (fn [_result]
|
|
207
|
+
(when-not @disposed?
|
|
208
|
+
(finish-agent-speech!)))
|
|
209
|
+
(fn [error]
|
|
210
|
+
(when-not @disposed?
|
|
211
|
+
(clear-playback-unsub!)
|
|
212
|
+
(reset! agent-speech-active? false)
|
|
213
|
+
(stop-speaking-behaviors!)
|
|
214
|
+
(js-call transcription "notifyAgentSpeechEnd")
|
|
215
|
+
(set-state! "idle")
|
|
216
|
+
(notify-error! error)))))))
|
|
217
|
+
(handle-yield! [yielded]
|
|
218
|
+
(-> (promise-resolve yielded)
|
|
219
|
+
(.then (fn [value]
|
|
220
|
+
(when-not @disposed?
|
|
221
|
+
(if (and (string? value) (pos? (count value)))
|
|
222
|
+
(speak-agent-text! value)
|
|
223
|
+
(start-listening!))))
|
|
224
|
+
(fn [error]
|
|
225
|
+
(when-not @disposed?
|
|
226
|
+
(set-state! "idle")
|
|
227
|
+
(notify-error! error))))))
|
|
228
|
+
(run-flow! [input has-input?]
|
|
229
|
+
(when-let [flow @flow-atom]
|
|
230
|
+
(try
|
|
231
|
+
(set-state! "processing")
|
|
232
|
+
(let [result (generator-next flow input has-input?)]
|
|
233
|
+
(if (result-done? result)
|
|
234
|
+
(set-state! "idle")
|
|
235
|
+
(handle-yield! (result-value result))))
|
|
236
|
+
(catch :default error
|
|
237
|
+
(set-state! "idle")
|
|
238
|
+
(notify-error! error)))))
|
|
239
|
+
(submit-user-input! [text]
|
|
240
|
+
(when-not @disposed?
|
|
241
|
+
(let [utterance (str text)]
|
|
242
|
+
(swap! context assoc :lastUserSpeech utterance)
|
|
243
|
+
(dispatch! {:type "transcriptFinal"
|
|
244
|
+
:text utterance
|
|
245
|
+
:source "conversationService"})
|
|
246
|
+
(stop-listening!)
|
|
247
|
+
(run-flow! utterance true))))
|
|
248
|
+
(transcript-handler [text is-final]
|
|
249
|
+
(when (and (:allowTranscriptInterruptionFallback @config-atom)
|
|
250
|
+
(can-interrupt-agent?))
|
|
251
|
+
(handle-interruption! "transcript" #js {:source "transcript"
|
|
252
|
+
:timestamp (now-ms)
|
|
253
|
+
:text text}))
|
|
254
|
+
(let [is-interruption (or (:isInterrupted @context)
|
|
255
|
+
(= "interrupted" (:state @context)))]
|
|
256
|
+
(js-call callbacks "onUserSpeech" text is-final is-interruption)
|
|
257
|
+
(when is-final
|
|
258
|
+
(cond
|
|
259
|
+
@agent-speech-active?
|
|
260
|
+
(if is-interruption
|
|
261
|
+
(reset! pending-interrupted-transcript text)
|
|
262
|
+
(js/console.debug "[ConversationService] Ignoring final transcript during uninterrupted agent speech:" text))
|
|
263
|
+
|
|
264
|
+
:else
|
|
265
|
+
(submit-user-input! text)))))
|
|
266
|
+
(interruption-handler [event]
|
|
267
|
+
(handle-interruption! "audio" event))]
|
|
268
|
+
(when-let [unsubscribe (js-call transcription "onTranscript" transcript-handler)]
|
|
269
|
+
(swap! unsubscribers conj unsubscribe))
|
|
270
|
+
(when-let [unsubscribe (js-call transcription "onInterruption" interruption-handler)]
|
|
271
|
+
(swap! unsubscribers conj unsubscribe))
|
|
272
|
+
(when-let [unsubscribe (js-call tts "onPlaybackReferenceTrackChange"
|
|
273
|
+
(fn [track]
|
|
274
|
+
(js-call transcription "setAgentAudioReferenceTrack" track)))]
|
|
275
|
+
(swap! unsubscribers conj unsubscribe))
|
|
276
|
+
#js {:start (fn [flow-generator]
|
|
277
|
+
(when-not @disposed?
|
|
278
|
+
(try
|
|
279
|
+
(js-call tts "stop")
|
|
280
|
+
(js-call transcription "stopListening")
|
|
281
|
+
(reset! flow-atom (flow-generator))
|
|
282
|
+
(dispatch! {:type "start"})
|
|
283
|
+
(run-flow! nil false)
|
|
284
|
+
(catch :default error
|
|
285
|
+
(notify-error! error)))))
|
|
286
|
+
:stop (fn []
|
|
287
|
+
(when-not @disposed?
|
|
288
|
+
(clear-playback-unsub!)
|
|
289
|
+
(reset! agent-speech-active? false)
|
|
290
|
+
(reset! pending-interrupted-transcript nil)
|
|
291
|
+
(js-call tts "stop")
|
|
292
|
+
(stop-listening!)
|
|
293
|
+
(js-call transcription "notifyAgentSpeechEnd")
|
|
294
|
+
(stop-speaking-behaviors!)
|
|
295
|
+
(dispatch! {:type "stop" :reason "manual"})
|
|
296
|
+
(set-state! "idle")))
|
|
297
|
+
:getState (fn [] (clj->js @context))
|
|
298
|
+
:submitUserInput submit-user-input!
|
|
299
|
+
:addSubscriptionCleanup (fn [unsubscribe]
|
|
300
|
+
(swap! unsubscribers conj unsubscribe))
|
|
301
|
+
:receiveTranscript (fn [text is-final]
|
|
302
|
+
(transcript-handler text is-final))
|
|
303
|
+
:receiveAudioInterruption interruption-handler
|
|
304
|
+
:dispose (fn []
|
|
305
|
+
(when-not @disposed?
|
|
306
|
+
(reset! disposed? true)
|
|
307
|
+
(clear-playback-unsub!)
|
|
308
|
+
(doseq [unsubscribe @unsubscribers]
|
|
309
|
+
(unsubscribe))
|
|
310
|
+
(reset! unsubscribers [])
|
|
311
|
+
(reset! agent-speech-active? false)
|
|
312
|
+
(reset! pending-interrupted-transcript nil)
|
|
313
|
+
(js-call tts "stop")
|
|
314
|
+
(js-call transcription "stopListening")
|
|
315
|
+
(js-call transcription "notifyAgentSpeechEnd")
|
|
316
|
+
(set-speaking! false)
|
|
317
|
+
(set-listening! false)
|
|
318
|
+
(js-call (prosodic) "stopTalking")
|
|
319
|
+
(js-call agency "dispose")))})))
|
|
320
|
+
|
|
321
|
+
(defn ConversationService
|
|
322
|
+
([tts transcription] (create-service tts transcription nil nil))
|
|
323
|
+
([tts transcription config] (create-service tts transcription config nil))
|
|
324
|
+
([tts transcription config callbacks] (create-service tts transcription config callbacks)))
|
|
325
|
+
|
|
326
|
+
(defn createConversationService
|
|
327
|
+
([tts transcription] (ConversationService tts transcription nil nil))
|
|
328
|
+
([tts transcription config] (ConversationService tts transcription config nil))
|
|
329
|
+
([tts transcription config callbacks] (ConversationService tts transcription config callbacks)))
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
(ns polymer.conversation.state
|
|
2
|
+
(:require [polymer.conversation.domain :as domain]))
|
|
3
|
+
|
|
4
|
+
;; Conversation state is the pure local transcript and turn-taking ledger. Live
|
|
5
|
+
;; provider handles stay in the agency's private effector; TTS and animation
|
|
6
|
+
;; remain peer agencies reached through plain domain events.
|
|
7
|
+
|
|
8
|
+
(def default-config
|
|
9
|
+
{:autoRespond true
|
|
10
|
+
:maxHistory 40
|
|
11
|
+
:responseSource "conversation"
|
|
12
|
+
:ttsAgency "tts"
|
|
13
|
+
:interruptionMode "cancel-agent"})
|
|
14
|
+
|
|
15
|
+
(defn finite-number?
|
|
16
|
+
[value]
|
|
17
|
+
(and (number? value) (js/isFinite value)))
|
|
18
|
+
|
|
19
|
+
(defn number-or
|
|
20
|
+
[value fallback]
|
|
21
|
+
(if (finite-number? value) value fallback))
|
|
22
|
+
|
|
23
|
+
(defn clamp
|
|
24
|
+
[lo hi value]
|
|
25
|
+
(min hi (max lo value)))
|
|
26
|
+
|
|
27
|
+
(defn normalize-config
|
|
28
|
+
[config]
|
|
29
|
+
(let [input (merge default-config (domain/data-map config))]
|
|
30
|
+
{:autoRespond (boolean (:autoRespond input))
|
|
31
|
+
:maxHistory (int (clamp 1 200 (number-or (:maxHistory input)
|
|
32
|
+
(:maxHistory default-config))))
|
|
33
|
+
:responseSource (domain/text-or (:responseSource input)
|
|
34
|
+
(:responseSource default-config))
|
|
35
|
+
:ttsAgency (domain/text-or (:ttsAgency input)
|
|
36
|
+
(:ttsAgency default-config))
|
|
37
|
+
:interruptionMode (domain/text-or (:interruptionMode input)
|
|
38
|
+
(:interruptionMode default-config))}))
|
|
39
|
+
|
|
40
|
+
(defn default-state
|
|
41
|
+
[config]
|
|
42
|
+
{:agency "conversation"
|
|
43
|
+
:status "idle"
|
|
44
|
+
:started false
|
|
45
|
+
:turnId nil
|
|
46
|
+
:history []
|
|
47
|
+
:context {}
|
|
48
|
+
:pendingResponse nil
|
|
49
|
+
:lastProviderError nil
|
|
50
|
+
:lastUserText nil
|
|
51
|
+
:lastAgentText nil
|
|
52
|
+
:lastEvent nil
|
|
53
|
+
:interrupted false
|
|
54
|
+
:startedCount 0
|
|
55
|
+
:stoppedCount 0
|
|
56
|
+
:userUtteranceCount 0
|
|
57
|
+
:agentUtteranceCount 0
|
|
58
|
+
:responseRequestCount 0
|
|
59
|
+
:ttsRequestCount 0
|
|
60
|
+
:cancelCount 0
|
|
61
|
+
:config (normalize-config config)})
|
|
62
|
+
|
|
63
|
+
(defn config->state
|
|
64
|
+
[config]
|
|
65
|
+
(default-state config))
|
|
66
|
+
|
|
67
|
+
(defn visible-state
|
|
68
|
+
[state]
|
|
69
|
+
(clj->js state))
|
|
70
|
+
|
|
71
|
+
(defn configure
|
|
72
|
+
[state config]
|
|
73
|
+
(assoc state :config (normalize-config (merge (:config state)
|
|
74
|
+
(domain/data-map config)))))
|
|
75
|
+
|
|
76
|
+
(defn reset-state
|
|
77
|
+
[state]
|
|
78
|
+
(default-state (:config state)))
|
|
79
|
+
|
|
80
|
+
(defn append-history
|
|
81
|
+
[state entry]
|
|
82
|
+
(let [max-history (get-in state [:config :maxHistory])
|
|
83
|
+
history (conj (:history state) entry)]
|
|
84
|
+
(assoc state :history (vec (take-last max-history history)))))
|
|
85
|
+
|
|
86
|
+
(defn next-turn-id
|
|
87
|
+
[]
|
|
88
|
+
(str "conversation:" (.now js/Date) ":" (rand-int 1000000)))
|
|
89
|
+
|
|
90
|
+
(defn mark-started
|
|
91
|
+
[state now-ms]
|
|
92
|
+
(-> state
|
|
93
|
+
(assoc :started true
|
|
94
|
+
:status "listening"
|
|
95
|
+
:turnId (or (:turnId state) (next-turn-id))
|
|
96
|
+
:interrupted false
|
|
97
|
+
:lastEvent {:type "conversation.status"
|
|
98
|
+
:status "listening"
|
|
99
|
+
:at now-ms})
|
|
100
|
+
(update :startedCount inc)))
|
|
101
|
+
|
|
102
|
+
(defn mark-stopped
|
|
103
|
+
[state now-ms reason]
|
|
104
|
+
(-> state
|
|
105
|
+
(assoc :started false
|
|
106
|
+
:status "idle"
|
|
107
|
+
:pendingResponse nil
|
|
108
|
+
:lastEvent {:type "conversation.status"
|
|
109
|
+
:status "idle"
|
|
110
|
+
:reason reason
|
|
111
|
+
:at now-ms})
|
|
112
|
+
(update :stoppedCount inc)))
|
|
113
|
+
|
|
114
|
+
(defn record-user-utterance
|
|
115
|
+
[state text now-ms source]
|
|
116
|
+
(let [entry {:role "user"
|
|
117
|
+
:text text
|
|
118
|
+
:source source
|
|
119
|
+
:at now-ms
|
|
120
|
+
:turnId (:turnId state)}]
|
|
121
|
+
(-> state
|
|
122
|
+
(assoc :status "processing"
|
|
123
|
+
:lastUserText text
|
|
124
|
+
:lastEvent {:type "conversation.userUtterance"
|
|
125
|
+
:text text
|
|
126
|
+
:at now-ms})
|
|
127
|
+
(append-history entry)
|
|
128
|
+
(update :userUtteranceCount inc))))
|
|
129
|
+
|
|
130
|
+
(defn record-response-request
|
|
131
|
+
[state request]
|
|
132
|
+
(-> state
|
|
133
|
+
(assoc :pendingResponse request
|
|
134
|
+
:lastProviderError nil
|
|
135
|
+
:lastEvent {:type "conversation.requestResponse"
|
|
136
|
+
:requestId (:requestId request)
|
|
137
|
+
:at (:requestedAt request)})
|
|
138
|
+
(update :responseRequestCount inc)))
|
|
139
|
+
|
|
140
|
+
(defn record-provider-error
|
|
141
|
+
[state request-id message now-ms]
|
|
142
|
+
(-> state
|
|
143
|
+
(assoc :status "listening"
|
|
144
|
+
:pendingResponse nil
|
|
145
|
+
:lastProviderError message
|
|
146
|
+
:lastEvent {:type "conversation.providerError"
|
|
147
|
+
:requestId request-id
|
|
148
|
+
:message message
|
|
149
|
+
:at now-ms})))
|
|
150
|
+
|
|
151
|
+
(defn record-agent-utterance
|
|
152
|
+
[state text now-ms source]
|
|
153
|
+
(let [entry {:role "agent"
|
|
154
|
+
:text text
|
|
155
|
+
:source source
|
|
156
|
+
:at now-ms
|
|
157
|
+
:turnId (:turnId state)}]
|
|
158
|
+
(-> state
|
|
159
|
+
(assoc :status "agentSpeaking"
|
|
160
|
+
:pendingResponse nil
|
|
161
|
+
:lastProviderError nil
|
|
162
|
+
:lastAgentText text
|
|
163
|
+
:lastEvent {:type "conversation.agentUtterance"
|
|
164
|
+
:text text
|
|
165
|
+
:at now-ms})
|
|
166
|
+
(append-history entry)
|
|
167
|
+
(update :agentUtteranceCount inc))))
|
|
168
|
+
|
|
169
|
+
(defn record-tts-request
|
|
170
|
+
[state request]
|
|
171
|
+
(-> state
|
|
172
|
+
(assoc :lastEvent {:type "tts.requestSpeak"
|
|
173
|
+
:requestId (:requestId request)
|
|
174
|
+
:at (:requestedAt request)})
|
|
175
|
+
(update :ttsRequestCount inc)))
|
|
176
|
+
|
|
177
|
+
(defn record-cancel
|
|
178
|
+
[state now-ms reason]
|
|
179
|
+
(-> state
|
|
180
|
+
(assoc :status "interrupted"
|
|
181
|
+
:pendingResponse nil
|
|
182
|
+
:interrupted true
|
|
183
|
+
:lastEvent {:type "conversation.cancelRequested"
|
|
184
|
+
:reason reason
|
|
185
|
+
:at now-ms})
|
|
186
|
+
(update :cancelCount inc)))
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
(ns polymer.core
|
|
2
|
+
(:require [polymer.animation.agency :as animation]
|
|
3
|
+
[polymer.animation.service :as animation-service]
|
|
4
|
+
[polymer.blink.agency :as blink]
|
|
5
|
+
[polymer.camera-context.agency :as camera-context]
|
|
6
|
+
[polymer.character :as character]
|
|
7
|
+
[polymer.conversation.agency :as conversation]
|
|
8
|
+
[polymer.conversation.service :as conversation-service]
|
|
9
|
+
[polymer.eye-head.agency :as eye-head]
|
|
10
|
+
[polymer.eye-head.service :as eye-head-service]
|
|
11
|
+
[polymer.gesture.agency :as gesture]
|
|
12
|
+
[polymer.gaze.agency :as gaze]
|
|
13
|
+
[polymer.tts.agency :as tts]
|
|
14
|
+
[polymer.lipsync.agency :as lipsync]
|
|
15
|
+
[polymer.prosodic.agency :as prosodic]
|
|
16
|
+
[polymer.transcription.agency :as transcription]
|
|
17
|
+
[polymer.transcription.service :as transcription-service]
|
|
18
|
+
[polymer.society :as society]))
|
|
19
|
+
|
|
20
|
+
(def DEFAULT_AGENCY_SOCIETY society/DEFAULT_AGENCY_SOCIETY)
|
|
21
|
+
|
|
22
|
+
;; Public JavaScript entry points.
|
|
23
|
+
;;
|
|
24
|
+
;; Polymer is authored in CLJS, but LoomLarge imports the compiled package from
|
|
25
|
+
;; JavaScript/TypeScript. The exports stay agency-oriented: LoomLarge can create
|
|
26
|
+
;; a character agency network, but Polymer owns the cross-agency routing and
|
|
27
|
+
;; Animation owns the Embody runtime calls.
|
|
28
|
+
|
|
29
|
+
(defn createBlinkAgency
|
|
30
|
+
"Create the Blink agency directly. Use createCharacterAgencies when Blink
|
|
31
|
+
should route animation intent to Polymer Animation."
|
|
32
|
+
([] (blink/create-blink-agency nil))
|
|
33
|
+
([config] (blink/create-blink-agency config)))
|
|
34
|
+
|
|
35
|
+
(defn createAnimationAgency
|
|
36
|
+
"Create the Animation agency directly. Pass an Embody runtime or engine in
|
|
37
|
+
config when the agency should execute snippets."
|
|
38
|
+
([] (animation/create-animation-agency nil))
|
|
39
|
+
([config] (animation/create-animation-agency config)))
|
|
40
|
+
|
|
41
|
+
(defn createAnimationService
|
|
42
|
+
"Create the JS compatibility animation API backed by Polymer Animation.
|
|
43
|
+
LoomLarge uses this while it is still calling the historical animation manager
|
|
44
|
+
shape; snippets still enter Polymer's agency-local planner and scheduler."
|
|
45
|
+
[engine]
|
|
46
|
+
(animation-service/createAnimationService engine))
|
|
47
|
+
|
|
48
|
+
(defn createEyeHeadTrackingService
|
|
49
|
+
"Create the JS compatibility Eye/Head Tracking API backed by Polymer
|
|
50
|
+
Eye/Head. Accepted movement requests route through Polymer Animation."
|
|
51
|
+
([] (eye-head-service/createEyeHeadTrackingService nil nil))
|
|
52
|
+
([config] (eye-head-service/createEyeHeadTrackingService config nil))
|
|
53
|
+
([config callbacks] (eye-head-service/createEyeHeadTrackingService config callbacks)))
|
|
54
|
+
|
|
55
|
+
(defn createTranscriptionService
|
|
56
|
+
"Create the JS compatibility transcription API backed by Polymer
|
|
57
|
+
Transcription. Browser SpeechRecognition is owned inside this package."
|
|
58
|
+
([] (transcription-service/createTranscriptionService nil nil))
|
|
59
|
+
([config] (transcription-service/createTranscriptionService config nil))
|
|
60
|
+
([config callbacks] (transcription-service/createTranscriptionService config callbacks)))
|
|
61
|
+
|
|
62
|
+
(defn createConversationService
|
|
63
|
+
"Create the JS compatibility conversation API backed by Polymer
|
|
64
|
+
Conversation."
|
|
65
|
+
([tts transcription] (conversation-service/createConversationService tts transcription nil nil))
|
|
66
|
+
([tts transcription config] (conversation-service/createConversationService tts transcription config nil))
|
|
67
|
+
([tts transcription config callbacks] (conversation-service/createConversationService tts transcription config callbacks)))
|
|
68
|
+
|
|
69
|
+
(def TranscriptionService transcription-service/TranscriptionService)
|
|
70
|
+
(def ConversationService conversation-service/ConversationService)
|
|
71
|
+
(def DEFAULT_EYE_HEAD_CONFIG eye-head-service/DEFAULT_EYE_HEAD_CONFIG)
|
|
72
|
+
(def DEFAULT_ANIMATION_KEYS eye-head-service/DEFAULT_ANIMATION_KEYS)
|
|
73
|
+
(def EYE_AUS eye-head-service/EYE_AUS)
|
|
74
|
+
(def HEAD_AUS eye-head-service/HEAD_AUS)
|
|
75
|
+
|
|
76
|
+
(def animationEventEmitter animation-service/animationEventEmitter)
|
|
77
|
+
(def snippetList$ animation-service/snippetList$)
|
|
78
|
+
(def globalPlaybackState$ animation-service/globalPlaybackState$)
|
|
79
|
+
(def bakedClipList$ animation-service/bakedClipList$)
|
|
80
|
+
(def playingBakedAnimations$ animation-service/playingBakedAnimations$)
|
|
81
|
+
(def bakedAnimationProgress$ animation-service/bakedAnimationProgress$)
|
|
82
|
+
|
|
83
|
+
(defn snippetState$ [snippet-name]
|
|
84
|
+
(animation-service/snippetState$ snippet-name))
|
|
85
|
+
|
|
86
|
+
(defn snippetTime$ [snippet-name]
|
|
87
|
+
(animation-service/snippetTime$ snippet-name))
|
|
88
|
+
|
|
89
|
+
(defn bakedAnimationState$ [clip-name]
|
|
90
|
+
(animation-service/bakedAnimationState$ clip-name))
|
|
91
|
+
|
|
92
|
+
(defn getBundledSnippetNames [list-key]
|
|
93
|
+
(animation-service/getBundledSnippetNames list-key))
|
|
94
|
+
|
|
95
|
+
(defn getStoredSnippetNames
|
|
96
|
+
([list-key] (animation-service/getStoredSnippetNames list-key))
|
|
97
|
+
([list-key storage] (animation-service/getStoredSnippetNames list-key storage)))
|
|
98
|
+
|
|
99
|
+
(defn getAvailableSnippetNames
|
|
100
|
+
([list-key] (animation-service/getAvailableSnippetNames list-key))
|
|
101
|
+
([list-key storage] (animation-service/getAvailableSnippetNames list-key storage)))
|
|
102
|
+
|
|
103
|
+
(defn resolveSnippetEntry
|
|
104
|
+
([list-key name] (animation-service/resolveSnippetEntry list-key name))
|
|
105
|
+
([list-key name storage] (animation-service/resolveSnippetEntry list-key name storage)))
|
|
106
|
+
|
|
107
|
+
(defn preloadAllSnippets []
|
|
108
|
+
(animation-service/preloadAllSnippets))
|
|
109
|
+
|
|
110
|
+
(defn clearPreloadedSnippets
|
|
111
|
+
([] (animation-service/clearPreloadedSnippets))
|
|
112
|
+
([storage] (animation-service/clearPreloadedSnippets storage)))
|
|
113
|
+
|
|
114
|
+
(defn createGazeAgency
|
|
115
|
+
"Create the Gaze agency directly. Use createCharacterAgencies when Gaze
|
|
116
|
+
should participate in the per-character agency network."
|
|
117
|
+
([] (gaze/create-gaze-agency nil))
|
|
118
|
+
([config] (gaze/create-gaze-agency config)))
|
|
119
|
+
|
|
120
|
+
(defn createEyeHeadTrackingAgency
|
|
121
|
+
"Create the Eye/Head Tracking agency directly."
|
|
122
|
+
([] (eye-head/create-eye-head-tracking-agency nil))
|
|
123
|
+
([config] (eye-head/create-eye-head-tracking-agency config)))
|
|
124
|
+
|
|
125
|
+
(defn createLipSyncAgency
|
|
126
|
+
"Create the LipSync agency directly. Use createCharacterAgencies when
|
|
127
|
+
LipSync should route viseme animation intent to Polymer Animation."
|
|
128
|
+
([] (lipsync/create-lipsync-agency nil))
|
|
129
|
+
([config] (lipsync/create-lipsync-agency config))
|
|
130
|
+
([config runtime] (lipsync/create-lipsync-agency config runtime)))
|
|
131
|
+
|
|
132
|
+
(defn createTTSAgency
|
|
133
|
+
"Create the TTS agency directly. Use createCharacterAgencies when TTS should
|
|
134
|
+
route speech timing facts into Polymer LipSync."
|
|
135
|
+
([] (tts/create-tts-agency nil))
|
|
136
|
+
([config] (tts/create-tts-agency config)))
|
|
137
|
+
|
|
138
|
+
(defn createGestureAgency
|
|
139
|
+
"Create the Gesture agency directly. Use createCharacterAgencies when Gesture
|
|
140
|
+
should route arm/hand animation intent to Polymer Animation."
|
|
141
|
+
([] (gesture/create-gesture-agency nil))
|
|
142
|
+
([config] (gesture/create-gesture-agency config)))
|
|
143
|
+
|
|
144
|
+
(defn createProsodicAgency
|
|
145
|
+
"Create the Prosodic Expression agency directly. Use createCharacterAgencies
|
|
146
|
+
when speech and blink facts should route to prosody inside Polymer."
|
|
147
|
+
([] (prosodic/create-prosodic-agency nil))
|
|
148
|
+
([config] (prosodic/create-prosodic-agency config)))
|
|
149
|
+
|
|
150
|
+
(defn createConversationAgency
|
|
151
|
+
"Create the Conversation agency directly. Use createCharacterAgencies when
|
|
152
|
+
transcript facts should route into Conversation and Conversation speech
|
|
153
|
+
requests should route to TTS inside Polymer."
|
|
154
|
+
([] (conversation/create-conversation-agency nil))
|
|
155
|
+
([config] (conversation/create-conversation-agency config)))
|
|
156
|
+
|
|
157
|
+
(defn createTranscriptionAgency
|
|
158
|
+
"Create the Transcription agency directly. Use createCharacterAgencies when
|
|
159
|
+
final transcript facts should route to Conversation inside Polymer."
|
|
160
|
+
([] (transcription/create-transcription-agency nil))
|
|
161
|
+
([config] (transcription/create-transcription-agency config)))
|
|
162
|
+
|
|
163
|
+
(defn createCameraContextAgency
|
|
164
|
+
"Create the Camera Context agency directly. Use createCharacterAgencies when
|
|
165
|
+
camera-relative facts should route to Gaze inside Polymer."
|
|
166
|
+
([] (camera-context/create-camera-context-agency nil))
|
|
167
|
+
([config] (camera-context/create-camera-context-agency config)))
|
|
168
|
+
|
|
169
|
+
(defn createCharacterAgencies
|
|
170
|
+
"Create the per-character Polymer agency network."
|
|
171
|
+
([] (character/create-character-agencies nil))
|
|
172
|
+
([config] (character/create-character-agencies config)))
|