@lovelace_lol/polymer 0.0.1

package/src/polymer/character.cljs

@@ -0,0 +1,471 @@
+(ns polymer.character
+  (:require [polymer.animation.agency :as animation]
+            [polymer.blink.agency :as blink]
+            [polymer.camera-context.agency :as camera-context]
+            [polymer.conversation.agency :as conversation]
+            [polymer.eye-head.agency :as eye-head]
+            [polymer.gaze.agency :as gaze]
+            [polymer.gesture.agency :as gesture]
+            [polymer.lipsync.agency :as lipsync]
+            [polymer.prosodic.agency :as prosodic]
+            [polymer.society :as society]
+            [polymer.stream :as stream]
+            [polymer.transcription.agency :as transcription]
+            [polymer.tts.agency :as tts]))
+
+;; A character is a network of Polymer agencies.
+;;
+;; LoomLarge may still consume streams from legacy Latticework services during
+;; the migration, but it should not consume Polymer animation events and turn
+;; them into animation calls. Inside Polymer, Blink emits blink intent and fast
+;; blink facts, Camera Context emits gaze facts, Transcription emits transcript
+;; facts, Conversation emits speech requests, TTS emits speech timing facts,
+;; LipSync emits mouth animation intent, Gesture emits arm/hand animation intent,
+;; Prosodic emits speech/blink expression intent, and Animation talks directly
+;; to Embody. Hair appearance and physics are Embody runtime concerns, not a
+;; Polymer behavioral agency.
+
+(defn create-character-agencies [config]
+  (let [input-stream (stream/create-stream)
+        event-stream (stream/create-stream)
+        emit-input (:emit input-stream)
+        emit-event (:emit event-stream)
+        agency-society (society/normalize-agency-society
+                        (when config (aget config "agencySociety")))
+        animation-agency (animation/create-animation-agency
+                          (society/agency-config config agency-society "animation"))
+        blink-agency (blink/create-blink-agency
+                      (society/agency-config config agency-society "blink"))
+        gaze-agency (gaze/create-gaze-agency
+                     (society/agency-config config agency-society "gaze"))
+        eye-head-agency (eye-head/create-eye-head-tracking-agency
+                         (society/agency-config config agency-society "eyeHeadTracking"))
+        gesture-agency (gesture/create-gesture-agency
+                        (society/agency-config config agency-society "gesture"))
+        camera-context-agency (camera-context/create-camera-context-agency
+                               (society/agency-config config agency-society "cameraContext"))
+        transcription-agency (transcription/create-transcription-agency
+                              (society/agency-config config agency-society "transcription"))
+        conversation-config (society/agency-config
+                             config agency-society "conversation")
+        ;; The provider is injected once into Conversation's private effector.
+        ;; The character router never invokes provider functions itself.
+        conversation-provider (when config (aget config "conversationProvider"))
+        conversation-options (if conversation-provider
+                               (let [options (js/Object.assign
+                                              #js {}
+                                              (or conversation-config #js {}))]
+                                 (aset options "provider" conversation-provider)
+                                 options)
+                               conversation-config)
+        conversation-agency (conversation/create-conversation-agency
+                             conversation-options)
+        tts-agency (tts/create-tts-agency
+                    (society/agency-config config agency-society "tts"))
+        lipsync-runtime (when config (aget config "lipSyncRuntime"))
+        lipsync-agency (lipsync/create-lipsync-agency
+                        (society/agency-config config agency-society "lipSync")
+                        lipsync-runtime)
+        prosodic-agency (prosodic/create-prosodic-agency
+                         (society/agency-config config agency-society "prosodic"))
+        unsubscribers (atom [])
+        disposed? (atom false)]
+    (letfn [(track! [unsubscribe]
+              (swap! unsubscribers conj unsubscribe))
+
+            (agency-by-name [name]
+              (case name
+                "animation" animation-agency
+                "blink" blink-agency
+                "gaze" gaze-agency
+                "eyeHeadTracking" eye-head-agency
+                "gesture" gesture-agency
+                "cameraContext" camera-context-agency
+                "transcription" transcription-agency
+                "conversation" conversation-agency
+                "tts" tts-agency
+                "lipSync" lipsync-agency
+                "prosodic" prosodic-agency
+                nil))
+
+            (dispatch-to! [name command]
+              (when-let [target (agency-by-name name)]
+                (when-let [dispatch (aget target "dispatch")]
+                  (.call dispatch target (clj->js command))
+                  true)))
+
+            (route-command! [from event to command]
+              (when-let [route (society/route-context agency-society from event to)]
+                (dispatch-to! to (assoc command :society route))))
+
+            (schedule-animation! [event]
+              (route-command! (:agency event)
+                              event
+                              "animation"
+                              {:type "scheduleSnippet"
+                               :sourceAgency (:agency event)
+                               :snippet (:snippet event)
+                               :options (:options event)}))
+
+            (remove-animation! [event]
+              (route-command! (:agency event)
+                              event
+                              "animation"
+                              {:type "removeSnippet"
+                               :sourceAgency (:agency event)
+                               :name (:name event)}))
+
+            (seek-animation! [event]
+              (route-command! (:agency event)
+                              event
+                              "animation"
+                              {:type "seekSnippet"
+                               :sourceAgency (:agency event)
+                               :name (:name event)
+                               :offsetSec (:offsetSec event)}))
+
+            (dispatch! [message]
+              (when-not @disposed?
+                (let [payload (js->clj message :keywordize-keys true)]
+                  ;; Input is still observable for tests/workers, but the write
+                  ;; path routes to agencies directly inside Polymer.
+                  (emit-input {:type "command"
+                               :agency (:agency payload)
+                               :message payload})
+                  (let [name (:agency payload)]
+                    (cond
+                      (nil? (agency-by-name name))
+                      (emit-event {:type "error"
+                                   :agency (or name "unknown")
+                                   :message "Unknown Polymer agency"})
+
+                      (not (society/agency-enabled? agency-society name))
+                      (emit-event {:type "agencyDisabled"
+                                   :agency name
+                                   :message "Polymer agency is disabled by agencySociety"})
+
+                      :else
+                      (dispatch-to! name (:command payload)))))))
+
+            (snapshot! []
+              (clj->js {:blink (js->clj (.snapshot ^js blink-agency) :keywordize-keys true)
+                        :gaze (js->clj (.snapshot ^js gaze-agency) :keywordize-keys true)
+                        :eyeHeadTracking (js->clj (.snapshot ^js eye-head-agency) :keywordize-keys true)
+                        :gesture (js->clj (.snapshot ^js gesture-agency) :keywordize-keys true)
+                        :cameraContext (js->clj (.snapshot ^js camera-context-agency) :keywordize-keys true)
+                        :transcription (js->clj (.snapshot ^js transcription-agency) :keywordize-keys true)
+                        :conversation (js->clj (.snapshot ^js conversation-agency) :keywordize-keys true)
+                        :tts (js->clj (.snapshot ^js tts-agency) :keywordize-keys true)
+                        :lipSync (js->clj (.snapshot ^js lipsync-agency) :keywordize-keys true)
+                        :prosodic (js->clj (.snapshot ^js prosodic-agency) :keywordize-keys true)
+                        :animation (js->clj (.snapshot ^js animation-agency) :keywordize-keys true)
+                        :agencySociety agency-society}))
+
+            (route-blink-event! [event]
+              (case (:type event)
+                "animation.requestScheduleSnippet"
+                (schedule-animation! event)
+
+                "signal"
+                (when (and (= "blink" (:agency event))
+                           (= "blink-fast" (:signal event)))
+                  (route-command! "blink"
+                                  event
+                                  "prosodic"
+                                  {:type "blinkFast"
+                                   :sourceAgency "blink"
+                                   :plan (:plan event)}))
+
+                nil))
+
+            (route-gaze-event! [event]
+              (case (:type event)
+                ("eyeHeadTracking.requestGaze"
+                 "eyeHeadTracking.requestReset"
+                 "eyeHeadTracking.requestCancel")
+                (route-command! "gaze" event "eyeHeadTracking" event)
+
+                nil))
+
+            (route-eye-head-event! [event]
+              (case (:type event)
+                "animation.requestScheduleSnippet"
+                (schedule-animation! event)
+
+                "animation.requestRemoveSnippet"
+                (remove-animation! event)
+
+                nil))
+
+            (route-gesture-event! [event]
+              (case (:type event)
+                "animation.requestScheduleSnippet"
+                (schedule-animation! event)
+
+                "animation.requestRemoveSnippet"
+                (remove-animation! event)
+
+                nil))
+
+            (route-camera-context-event! [event]
+              (case (:type event)
+                "camera.fact"
+                (route-command! "cameraContext" event "gaze" event)
+
+                "camera.stale"
+                (route-command! "cameraContext" event "gaze" event)
+
+                nil))
+
+            (route-transcription-event! [event]
+              (case (:type event)
+                "transcription.final"
+                (route-command! "transcription"
+                                event
+                                "conversation"
+                                {:type "transcript.final"
+                                 :text (:text event)
+                                 :source "transcription"
+                                 :confidence (:confidence event)
+                                 :sequence (:sequence event)
+                                 :sessionId (:sessionId event)
+                                 :at (:at event)})
+
+                "transcription.interruption"
+                (route-command! "transcription"
+                                event
+                                "conversation"
+                                {:type "interrupt"
+                                 :reason "transcription-interruption"
+                                 :text (:text event)
+                                 :source "transcription"})
+
+                "transcription.audioInterruption"
+                (route-command! "transcription"
+                                event
+                                "conversation"
+                                {:type "interrupt"
+                                 :reason "audio-interruption"
+                                 :event event
+                                 :source "transcription"})
+
+                nil))
+
+            (route-conversation-event! [event]
+              ;; Router only: forward conversation facts/requests. Gaze and
+              ;; Prosodic own local policy through their planners/schedulers.
+              (case (:type event)
+                ("conversation.userUtterance"
+                 "conversation.agentUtterance")
+                (let [fact (assoc event :sourceAgency "conversation")]
+                  (route-command! "conversation" event "prosodic" fact)
+                  (route-command! "conversation" event "gaze" fact))
+
+                "conversation.requestResponse"
+                (route-command! "conversation"
+                                event
+                                "prosodic"
+                                (assoc event :sourceAgency "conversation"))
+
+                "tts.requestSpeak"
+                (route-command! "conversation" event "tts" (:command event))
+
+                "conversation.cancelRequested"
+                (let [cancel (assoc event :sourceAgency "conversation")]
+                  (route-command! "conversation" event "prosodic" cancel)
+                  (route-command! "conversation" event "gaze" cancel)
+                  (route-command! "conversation"
+                                  event
+                                  "tts"
+                                  {:type "stop"
+                                   :reason (:reason event)}))
+
+                nil))
+
+            (route-tts-event! [event]
+              (case (:type event)
+                "lipSync.command"
+                (route-command! "tts" event "lipSync" (:command event))
+
+                "ttsStatusChanged"
+                (let [status (get-in event [:state :status])]
+                  (route-command! "tts"
+                                  event
+                                  "conversation"
+                                  {:type "tts.status"
+                                   :status status})
+                  (route-command! "tts"
+                                  event
+                                  "transcription"
+                                  {:type "tts.status"
+                                   :status status
+                                   :speaking (get-in event [:state :speaking])
+                                   :state (:state event)}))
+
+                "ttsSpeechStarted"
+                (do
+                  (route-command! "tts"
+                                  event
+                                  "prosodic"
+                                  {:type "speechStarted"
+                                   :sourceAgency "tts"
+                                   :name (:name event)
+                                   :engine (:engine event)})
+                  (route-command! "tts" event "transcription" event)
+                  (route-command! "tts"
+                                  event
+                                  "conversation"
+                                  {:type "tts.status"
+                                   :status "speaking"}))
+
+                "ttsWordBoundary"
+                (route-command! "tts"
+                                event
+                                "prosodic"
+                                {:type "wordBoundary"
+                                 :sourceAgency "tts"
+                                 :word (:word event)
+                                 :wordIndex (:wordIndex event)
+                                 :observedElapsedSec (:observedElapsedSec event)
+                                 :hostElapsedSec (:hostElapsedSec event)})
+
+                "ttsSpeechStopped"
+                (do
+                  (route-command! "tts"
+                                  event
+                                  "prosodic"
+                                  {:type "speechStopped"
+                                   :sourceAgency "tts"
+                                   :reason (:reason event)})
+                  (route-command! "tts" event "transcription" event)
+                  (route-command! "tts"
+                                  event
+                                  "conversation"
+                                  {:type "tts.status"
+                                   :status "idle"}))
+
+                "ttsSpeechEnded"
+                (do
+                  (route-command! "tts"
+                                  event
+                                  "prosodic"
+                                  {:type "speechStopped"
+                                   :sourceAgency "tts"
+                                   :reason "completed"})
+                  (route-command! "tts" event "transcription" event)
+                  (route-command! "tts"
+                                  event
+                                  "conversation"
+                                  {:type "tts.status"
+                                   :status "idle"}))
+
+                nil))
+
+            (route-lipsync-event! [event]
+              (case (:type event)
+                "animation.requestScheduleSnippet"
+                (schedule-animation! event)
+
+                "animation.requestRemoveSnippet"
+                (remove-animation! event)
+
+                "animation.requestSeekSnippet"
+                (seek-animation! event)
+
+                nil))
+
+            (route-prosodic-event! [event]
+              (case (:type event)
+                "animation.requestScheduleSnippet"
+                (schedule-animation! event)
+
+                "animation.requestRemoveSnippet"
+                (remove-animation! event)
+
+                nil))]
+      ;; Fan-in agency events to one character-level event stream for tests,
+      ;; workers, and future non-React observers. LoomLarge does not need to
+      ;; consume this stream for Polymer animation playback.
+      (track! (.subscribeEvents ^js animation-agency
+                                #(emit-event (js->clj % :keywordize-keys true))))
+      (track! (.subscribeEvents ^js blink-agency
+                                (fn [event]
+                                  (let [payload (js->clj event :keywordize-keys true)]
+                                    (route-blink-event! payload)
+                                    (emit-event payload)))))
+      (track! (.subscribeEvents ^js gaze-agency
+                                (fn [event]
+                                  (let [payload (js->clj event :keywordize-keys true)]
+                                    (route-gaze-event! payload)
+                                    (emit-event payload)))))
+      (track! (.subscribeEvents ^js eye-head-agency
+                                (fn [event]
+                                  (let [payload (js->clj event :keywordize-keys true)]
+                                    (route-eye-head-event! payload)
+                                    (emit-event payload)))))
+      (track! (.subscribeEvents ^js gesture-agency
+                                (fn [event]
+                                  (let [payload (js->clj event :keywordize-keys true)]
+                                    (route-gesture-event! payload)
+                                    (emit-event payload)))))
+      (track! (.subscribeEvents ^js camera-context-agency
+                                (fn [event]
+                                  (let [payload (js->clj event :keywordize-keys true)]
+                                    (route-camera-context-event! payload)
+                                    (emit-event payload)))))
+      (track! (.subscribeEvents ^js transcription-agency
+                                (fn [event]
+                                  (let [payload (js->clj event :keywordize-keys true)]
+                                    (route-transcription-event! payload)
+                                    (emit-event payload)))))
+      (track! (.subscribeEvents ^js conversation-agency
+                                (fn [event]
+                                  (let [payload (js->clj event :keywordize-keys true)]
+                                    (route-conversation-event! payload)
+                                    (emit-event payload)))))
+      (track! (.subscribeEvents ^js tts-agency
+                                (fn [event]
+                                  (let [payload (js->clj event :keywordize-keys true)]
+                                    (route-tts-event! payload)
+                                    (emit-event payload)))))
+      (track! (.subscribeEvents ^js lipsync-agency
+                                (fn [event]
+                                  (let [payload (js->clj event :keywordize-keys true)]
+                                    (route-lipsync-event! payload)
+                                    (emit-event payload)))))
+      (track! (.subscribeEvents ^js prosodic-agency
+                                (fn [event]
+                                  (let [payload (js->clj event :keywordize-keys true)]
+                                    (route-prosodic-event! payload)
+                                    (emit-event payload)))))
+      #js {:dispatch dispatch!
+           :input (stream/writable-port input-stream dispatch!)
+           :events (stream/readable-port event-stream)
+           :agencySociety (clj->js agency-society)
+           :snapshot snapshot!
+           :agency (fn [name]
+                     (when (society/agency-enabled? agency-society name)
+                       (agency-by-name name)))
+           :subscribeInput (fn [listener] ((:subscribe input-stream) listener))
+           :subscribeEvents (fn [listener] ((:subscribe event-stream) listener))
+           :subscribe (fn [listener] ((:subscribe event-stream) listener))
+           :subscribeStatus (fn [listener] ((:subscribe event-stream) listener))
+           :dispose (fn []
+                      (when-not @disposed?
+                        (reset! disposed? true)
+                        (doseq [unsubscribe @unsubscribers]
+                          (unsubscribe))
+                        (reset! unsubscribers [])
+                        (.dispose ^js blink-agency)
+                        (.dispose ^js gaze-agency)
+                        (.dispose ^js eye-head-agency)
+                        (.dispose ^js gesture-agency)
+                        (.dispose ^js camera-context-agency)
+                        (.dispose ^js transcription-agency)
+                        (.dispose ^js conversation-agency)
+                        (.dispose ^js tts-agency)
+                        (.dispose ^js lipsync-agency)
+                        (.dispose ^js prosodic-agency)
+                        (.dispose ^js animation-agency)
+                        ((:dispose input-stream))
+                        ((:dispose event-stream))))})))

package/src/polymer/conversation/agency.cljs

@@ -0,0 +1,83 @@
+(ns polymer.conversation.agency
+  (:require [polymer.conversation.domain :as domain]
+            [polymer.conversation.effector :as effector]
+            [polymer.conversation.planner :as planner]
+            [polymer.conversation.scheduler :as scheduler]
+            [polymer.conversation.state :as state]
+            [polymer.stream :as stream]))
+
+;; Conversation is a peer agency in the Polymer character society. It accepts
+;; transcript, response, and interruption facts as data; plans local turn-taking
+;; work; schedules those actions; then emits requests/facts that TTS, Gaze,
+;; Prosodic, and other agencies can decide how to handle.
+
+(defn now-ms
+  []
+  (.now js/Date))
+
+(defn js-command
+  [type value]
+  #js {:type type :value value})
+
+(defn create-conversation-agency
+  [config]
+  (let [provider (if (map? config)
+                   (or (:provider config)
+                       (:conversationProvider config))
+                   (when config
+                     (or (aget config "provider")
+                         (aget config "conversationProvider"))))
+        config-map (domain/data-map config)
+        state-config (dissoc config-map :provider :conversationProvider)
+        input-stream (stream/create-stream)
+        event-stream (stream/create-stream)
+        state-atom (atom (state/config->state state-config))
+        disposed? (atom false)
+        emit-input (:emit input-stream)
+        emit-event (:emit event-stream)
+        scheduler-atom (atom nil)]
+    (letfn [(dispatch! [command]
+              (when-not @disposed?
+                (let [payload (domain/data-map command)
+                      plan (planner/plan-command payload @state-atom (now-ms))]
+                  ;; Input is observable for workers/tests and for future
+                  ;; diagnostics. It is not host application state and should
+                  ;; not be mirrored into a UI on every agency message.
+                  (emit-input {:type "command"
+                               :agency "conversation"
+                               :command payload})
+                  (when-let [agency-scheduler @scheduler-atom]
+                    ((:schedule agency-scheduler) payload plan)))))
+            (dispatch-type! [type]
+              (dispatch! #js {:type type}))]
+      (let [agency-scheduler (scheduler/create-scheduler
+                              {:state-atom state-atom
+                               :emit-event emit-event
+                               :dispatch-command dispatch!
+                               :provider-effector
+                               (effector/create-provider-effector provider)})]
+        (reset! scheduler-atom agency-scheduler)
+        #js {:dispatch dispatch!
+             :snapshot (fn [] (state/visible-state @state-atom))
+             :input (stream/writable-port input-stream dispatch!)
+             :events (stream/readable-port event-stream)
+             :subscribeInput (fn [listener] ((:subscribe input-stream) listener))
+             :subscribeEvents (fn [listener] ((:subscribe event-stream) listener))
+             :subscribe (fn [listener] ((:subscribe event-stream) listener))
+             :subscribeStatus (fn [listener] ((:subscribe event-stream) listener))
+             :start (fn [] (dispatch-type! "start"))
+             :stop (fn [] (dispatch-type! "stop"))
+             :reset (fn [] (dispatch-type! "reset"))
+             :interrupt (fn
+                          ([] (dispatch-type! "interrupt"))
+                          ([reason] (dispatch! #js {:type "interrupt" :reason reason})))
+             :setAutoRespond (fn [value]
+                               (dispatch! #js {:type "configure"
+                                               :config #js {:autoRespond value}}))
+             :queue (fn [] (clj->js ((:queue agency-scheduler))))
+             :dispose (fn []
+                        (when-not @disposed?
+                          (reset! disposed? true)
+                          ((:dispose agency-scheduler))
+                          ((:dispose input-stream))
+                          ((:dispose event-stream))))}))))

package/src/polymer/conversation/domain.cljs

@@ -0,0 +1,35 @@
+(ns polymer.conversation.domain
+  (:require [clojure.string :as str]))
+
+;; Pure Conversation transforms live here so command normalization is shared by
+;; state, planner, and the JS-facing agency boundary.
+
+(defn data-map
+  [value]
+  (cond
+    (map? value) value
+    value (js->clj value :keywordize-keys true)
+    :else {}))
+
+(defn clean-text
+  [value]
+  (let [text (str/trim (or value ""))]
+    (when (pos? (count text))
+      text)))
+
+(defn text-or
+  [value fallback]
+  (or (clean-text value) fallback))
+
+(defn user-text
+  [command]
+  (clean-text (or (:text command)
+                  (:transcript command)
+                  (:utterance command))))
+
+(defn agent-text
+  [command]
+  (clean-text (or (:agentText command)
+                  (:responseText command)
+                  (:text command)
+                  (:utterance command))))