@lovelace_lol/polymer 0.0.1

package/src/polymer/transcription/effector.cljs

@@ -0,0 +1,611 @@
+(ns polymer.transcription.effector)
+
+;; This namespace is the only Transcription code allowed to touch browser
+;; recognition, microphone, AudioContext, or animation-frame APIs. The planner
+;; proposes work and the scheduler invokes this private effector.
+
+(defn runtime-value
+  [runtime key]
+  (cond
+    (map? runtime) (get runtime key)
+    runtime (aget runtime (name key))
+    :else nil))
+
+(defn recognition-constructor
+  []
+  (let [win (when (exists? js/window) js/window)]
+    (or (when win (aget win "SpeechRecognition"))
+        (when win (aget win "webkitSpeechRecognition")))))
+
+(defn create-recognition
+  [runtime]
+  (if-let [factory (runtime-value runtime :createRecognition)]
+    (factory)
+    (when-let [ctor (recognition-constructor)]
+      (ctor.))))
+
+(defn create-audio-context
+  [runtime]
+  (if-let [factory (runtime-value runtime :createAudioContext)]
+    (factory)
+    (let [win (when (exists? js/window) js/window)
+          ctor (or (when win (aget win "AudioContext"))
+                   (when win (aget win "webkitAudioContext")))]
+      (when ctor
+        (ctor.)))))
+
+(defn media-devices
+  []
+  (let [navigator (when (exists? js/navigator) js/navigator)]
+    (when navigator (.-mediaDevices navigator))))
+
+(defn apply-config!
+  [recognition config]
+  (when recognition
+    (set! (.-lang recognition) (:lang config))
+    (set! (.-continuous recognition) (boolean (:continuous config)))
+    (set! (.-interimResults recognition) (boolean (:interimResults config)))
+    (set! (.-maxAlternatives recognition) (:maxAlternatives config))))
+
+(defn result-commands
+  [event session-id provider-token interim-results?]
+  (let [results (.-results event)
+        start (or (.-resultIndex event) 0)
+        total (if results (.-length results) 0)]
+    (loop [index start
+           commands []]
+      (if (< index total)
+        (let [result (aget results index)
+              alternative (when result (aget result 0))
+              text (when alternative (.-transcript alternative))
+              confidence (when alternative (.-confidence alternative))
+              final? (boolean (when result (.-isFinal result)))]
+          (recur (inc index)
+                 (cond-> commands
+                   (and text (or final? interim-results?))
+                   (conj {:type (if final? "providerFinal" "providerPartial")
+                          :text text
+                          :confidence (if (number? confidence) confidence 1)
+                          :source "webSpeech"
+                          :sessionId session-id
+                          :providerToken provider-token}))))
+        commands))))
+
+(defn clear-handler!
+  [recognition handler-name]
+  (when recognition
+    (aset recognition handler-name nil)))
+
+(defn detach-handlers!
+  [recognition]
+  (doseq [handler-name ["onstart" "onresult" "onerror" "onend"]]
+    (clear-handler! recognition handler-name)))
+
+(defn disconnect!
+  [node]
+  (when node
+    (try
+      (.disconnect node)
+      (catch :default _ nil))))
+
+(defn stop-media-stream!
+  [stream]
+  (when stream
+    (doseq [track (array-seq (.getTracks stream))]
+      (try
+        (.stop track)
+        (catch :default _ nil)))))
+
+(defn read-level
+  [analyser data]
+  (if (and analyser data)
+    (do
+      (.getByteTimeDomainData analyser data)
+      (let [length (.-length data)]
+        (loop [index 0
+               sum 0]
+          (if (< index length)
+            (let [sample (/ (- (aget data index) 128) 128)]
+              (recur (inc index) (+ sum (* sample sample))))
+            (js/Math.sqrt (/ sum (max 1 length)))))))
+    0))
+
+(defn live-track
+  [stream]
+  (when stream
+    (some #(when (= "live" (.-readyState %)) %)
+          (array-seq (.getAudioTracks stream)))))
+
+(defn create-effector
+  [{:keys [dispatch get-state runtime]}]
+  (let [recognition (atom nil)
+        active-token (atom 0)
+        failed-tokens (atom #{})
+        retry-timer (atom nil)
+        disposed? (atom false)
+        mic-stream (atom nil)
+        mic-request (atom nil)
+        analysis-context (atom nil)
+        mic-analyser (atom nil)
+        mic-analyser-data (atom nil)
+        mic-source-node (atom nil)
+        reference-track (atom nil)
+        reference-analyser (atom nil)
+        reference-analyser-data (atom nil)
+        reference-source-node (atom nil)
+        reference-update-token (atom 0)
+        agent-speaking? (atom false)
+        interruption-frame (atom nil)
+        interruption-candidate-start (atom nil)
+        interruption-latched? (atom false)
+        last-debug-at (atom 0)]
+    (letfn [(current-config []
+              (:config (get-state)))
+
+            (current-token? [token]
+              (and (not @disposed?)
+                   (= token @active-token)))
+
+            (cancel-retry! []
+              (when-let [timer @retry-timer]
+                (js/clearTimeout timer)
+                (reset! retry-timer nil)))
+
+            (invalidate-provider! []
+              (swap! active-token inc)
+              (reset! failed-tokens #{}))
+
+            (retire-current-provider! [instance token]
+              (when (and (= instance @recognition)
+                         (current-token? token))
+                (swap! active-token inc)
+                (reset! recognition nil)
+                (detach-handlers! instance)
+                true))
+
+            (abort-retired-provider! [instance]
+              (try
+                (when-let [abort (aget instance "abort")]
+                  (.call abort instance))
+                (catch :default _ nil)))
+
+            (request-frame! [callback]
+              (if-let [request-frame (runtime-value runtime :requestAnimationFrame)]
+                (request-frame callback)
+                (let [win (when (exists? js/window) js/window)]
+                  (when (and win (.-requestAnimationFrame win))
+                    (.requestAnimationFrame win callback)))))
+
+            (cancel-frame! [frame]
+              (if-let [cancel-frame (runtime-value runtime :cancelAnimationFrame)]
+                (cancel-frame frame)
+                (let [win (when (exists? js/window) js/window)]
+                  (when (and win (.-cancelAnimationFrame win))
+                    (.cancelAnimationFrame win frame)))))
+
+            (clock-ms []
+              (let [performance (when (exists? js/performance) js/performance)]
+                (if performance (.now performance) (.now js/Date))))
+
+            (stop-interruption-monitoring! []
+              (when-let [frame @interruption-frame]
+                (cancel-frame! frame)
+                (reset! interruption-frame nil))
+              (reset! interruption-candidate-start nil))
+
+            (schedule-interruption-frame! [tick]
+              (when-let [frame (request-frame! tick)]
+                (reset! interruption-frame frame)))
+
+            (start-interruption-monitoring! []
+              (stop-interruption-monitoring!)
+              (when (and @agent-speaking?
+                         (:interruptDetectionEnabled (current-config)))
+                (letfn [(tick []
+                          (reset! interruption-frame nil)
+                          (if (or @disposed?
+                                  (not @agent-speaking?)
+                                  (not (:interruptDetectionEnabled
+                                        (current-config))))
+                            (stop-interruption-monitoring!)
+                            (let [config (current-config)
+                                  live-reference? (and @reference-track
+                                                       (= "live"
+                                                          (.-readyState
+                                                           @reference-track))
+                                                       @reference-analyser)]
+                              (if (and (:requireAgentReferenceForInterruption config)
+                                       (not live-reference?))
+                                (do
+                                  (reset! interruption-candidate-start nil)
+                                  (schedule-interruption-frame! tick))
+                                (let [microphone-level
+                                      (read-level @mic-analyser @mic-analyser-data)
+                                      reference-level
+                                      (if live-reference?
+                                        (read-level @reference-analyser
+                                                    @reference-analyser-data)
+                                        0)
+                                      required-level
+                                      (js/Math.max
+                                       (:interruptionVolumeThreshold config)
+                                       (+ (* reference-level
+                                             (:interruptionReferenceScale config))
+                                          (:interruptionReferenceOffset config)))
+                                      interrupt? (>= microphone-level
+                                                     required-level)
+                                      now (clock-ms)]
+                                  (when (and (:interruptionDebugLogging config)
+                                             (>= (- now @last-debug-at) 500))
+                                    (reset! last-debug-at now)
+                                    (js/console.debug
+                                     "[Transcription] Interruption levels"
+                                     #js {:microphoneLevel microphone-level
+                                          :referenceLevel reference-level
+                                          :requiredLevel required-level
+                                          :shouldInterrupt interrupt?
+                                          :hasAgentReference live-reference?}))
+                                  (if (and interrupt?
+                                           (not @interruption-latched?))
+                                    (if-let [candidate-start
+                                             @interruption-candidate-start]
+                                      (when (>= (- now candidate-start)
+                                                (:interruptionHoldMs config))
+                                        (reset! interruption-latched? true)
+                                        (dispatch
+                                         {:type "providerAudioInterruption"
+                                          :timestamp (.now js/Date)
+                                          :microphoneLevel microphone-level
+                                          :referenceLevel reference-level
+                                          :requiredLevel required-level}))
+                                      (reset! interruption-candidate-start now))
+                                    (when-not interrupt?
+                                      (reset! interruption-candidate-start nil)))
+                                  (schedule-interruption-frame! tick))))))]
+                  (schedule-interruption-frame! tick))))
+
+            (ensure-analysis-context! []
+              (if @disposed?
+                (.resolve js/Promise nil)
+                (try
+                  (when (or (nil? @analysis-context)
+                            (= "closed" (.-state @analysis-context)))
+                    (reset! analysis-context (create-audio-context runtime)))
+                  (let [context @analysis-context]
+                    (if (and context (= "suspended" (.-state context)))
+                      (-> (.resume context)
+                          (.then (fn [_] context))
+                          (.catch (fn [_] context)))
+                      (.resolve js/Promise context)))
+                  (catch :default error
+                    (.reject js/Promise error)))))
+
+            (ensure-microphone-analyser! []
+              (-> (ensure-analysis-context!)
+                  (.then
+                   (fn [context]
+                     (when (and (not @disposed?) context @mic-stream)
+                       (let [track (live-track @mic-stream)
+                             old-source @mic-source-node
+                             old-track (when old-source
+                                         (live-track
+                                          (aget old-source "mediaStream")))]
+                         (when (and track (not= track old-track))
+                           (disconnect! old-source)
+                           (disconnect! @mic-analyser)
+                           (let [source (.createMediaStreamSource
+                                         context @mic-stream)
+                                 analyser (.createAnalyser context)]
+                             (set! (.-fftSize analyser) 2048)
+                             (set! (.-smoothingTimeConstant analyser) 0.65)
+                             (.connect source analyser)
+                             (reset! mic-source-node source)
+                             (reset! mic-analyser analyser)
+                             (reset! mic-analyser-data
+                                     (js/Uint8Array.
+                                      (.-fftSize analyser)))))))))))
+
+            (request-microphone! []
+              (if-let [get-user-media (runtime-value runtime :getUserMedia)]
+                (.resolve js/Promise (get-user-media))
+                (if-let [media (media-devices)]
+                  (let [supported (if (.-getSupportedConstraints media)
+                                    (.getSupportedConstraints media)
+                                    #js {})
+                        constraints
+                        (clj->js
+                         (cond-> {:echoCancellation true
+                                  :noiseSuppression true
+                                  :autoGainControl true}
+                           (boolean (aget supported "voiceIsolation"))
+                           (assoc :voiceIsolation true)))]
+                    (-> (.getUserMedia media #js {:audio constraints})
+                        (.catch (fn [_]
+                                  (.getUserMedia media #js {:audio true})))))
+                  ;; Worker and Node tests have no mediaDevices. Recognition can
+                  ;; still start through an injected provider.
+                  (.resolve js/Promise nil))))
+
+            (ensure-microphone-stream! []
+              (cond
+                @disposed?
+                (.resolve js/Promise nil)
+
+                (live-track @mic-stream)
+                (-> (ensure-microphone-analyser!)
+                    (.then (fn [_] @mic-stream)))
+
+                @mic-request
+                (:promise @mic-request)
+
+                :else
+                (let [token (js-obj)
+                      requested
+                      (-> (request-microphone!)
+                          (.then
+                           (fn [stream]
+                             (if @disposed?
+                               (do
+                                 (stop-media-stream! stream)
+                                 nil)
+                               (do
+                                 (reset! mic-stream stream)
+                                 (-> (ensure-microphone-analyser!)
+                                     (.then
+                                      (fn [_]
+                                        (if @disposed?
+                                          (do
+                                            (when (= stream @mic-stream)
+                                              (reset! mic-stream nil))
+                                            (stop-media-stream! stream)
+                                            nil)
+                                          stream)))))))))
+                      settled
+                      (-> requested
+                          (.then
+                           (fn [stream]
+                             (when (identical? token (:token @mic-request))
+                               (reset! mic-request nil))
+                             stream))
+                          (.catch
+                           (fn [error]
+                             (when (identical? token (:token @mic-request))
+                               (reset! mic-request nil))
+                             (throw error))))]
+                  (reset! mic-request {:token token :promise settled})
+                  settled)))
+
+            (replace-reference-analyser! [track]
+              (let [token (swap! reference-update-token inc)]
+                (reset! reference-track track)
+                (disconnect! @reference-source-node)
+                (disconnect! @reference-analyser)
+                (reset! reference-source-node nil)
+                (reset! reference-analyser nil)
+                (reset! reference-analyser-data nil)
+                (when (and track (= "live" (.-readyState track)))
+                  (-> (ensure-analysis-context!)
+                      (.then
+                       (fn [context]
+                         (when (and context
+                                    (not @disposed?)
+                                    (= token @reference-update-token)
+                                    (= track @reference-track)
+                                    (= "live" (.-readyState track)))
+                           (let [stream-constructor
+                                 (or (runtime-value runtime :createMediaStream)
+                                     (when (exists? js/MediaStream)
+                                       (fn [tracks]
+                                         (js/MediaStream. tracks))))
+                                 stream (when stream-constructor
+                                          (stream-constructor #js [track]))]
+                             (when stream
+                               (let [source (.createMediaStreamSource
+                                             context stream)
+                                     analyser (.createAnalyser context)]
+                                 (set! (.-fftSize analyser) 2048)
+                                 (set! (.-smoothingTimeConstant analyser) 0.65)
+                                 (.connect source analyser)
+                                 (if (and (= token @reference-update-token)
+                                          (= track @reference-track)
+                                          (= "live" (.-readyState track)))
+                                   (do
+                                     (reset! reference-source-node source)
+                                     (reset! reference-analyser analyser)
+                                     (reset! reference-analyser-data
+                                             (js/Uint8Array.
+                                              (.-fftSize analyser))))
+                                   (do
+                                     (disconnect! source)
+                                     (disconnect! analyser)))))))))
+                      (.catch (fn [_] nil))))
+                (when @agent-speaking?
+                  (start-interruption-monitoring!))))
+
+            (sync-agent-speaking! [speaking?]
+              (reset! agent-speaking? (boolean speaking?))
+              (reset! interruption-latched? false)
+              (reset! interruption-candidate-start nil)
+              (if @agent-speaking?
+                (-> (ensure-microphone-analyser!)
+                    (.then (fn [_]
+                             (start-interruption-monitoring!)))
+                    (.catch (fn [_] nil)))
+                (stop-interruption-monitoring!)))
+
+            (stop-provider! [method-name]
+              (cancel-retry!)
+              (stop-interruption-monitoring!)
+              (let [instance @recognition]
+                (invalidate-provider!)
+                (reset! recognition nil)
+                (when instance
+                  (detach-handlers! instance)
+                  (try
+                    (when-let [method (aget instance method-name)]
+                      (.call method instance))
+                    (catch :default _ nil)))
+                (.resolve js/Promise nil)))
+
+            (fail-provider! [instance token session-id message]
+              (when (and (current-token? token)
+                         (not (contains? @failed-tokens token)))
+                (swap! failed-tokens conj token)
+                (when (retire-current-provider! instance token)
+                  (abort-retired-provider! instance)
+                  (dispatch {:type "providerError"
+                             :message message
+                             :sessionId session-id
+                             :providerToken token}))))
+
+            (start-recognition-instance! [instance track]
+              (if track
+                (try
+                  (.start instance track)
+                  (catch :default _
+                    (.start instance)))
+                (.start instance)))
+
+            (start-provider! [session-id config]
+              (cancel-retry!)
+              (if @disposed?
+                (.resolve js/Promise nil)
+                (if-let [instance (create-recognition runtime)]
+                  (let [token (swap! active-token inc)
+                        begin! (fn [stream]
+                                 (when (current-token? token)
+                                   (try
+                                     (start-recognition-instance!
+                                      instance (live-track stream))
+                                     (catch :default error
+                                       (fail-provider!
+                                        instance token session-id
+                                        (or (.-message error)
+                                            "speech-recognition-start-error"))))))]
+                    (reset! recognition instance)
+                    (apply-config! instance config)
+                    (set! (.-onstart instance)
+                          (fn []
+                            (when (current-token? token)
+                              (dispatch {:type "providerStarted"
+                                         :sessionId session-id
+                                         :providerToken token})
+                              (when @agent-speaking?
+                                (start-interruption-monitoring!)))))
+                    (set! (.-onresult instance)
+                          (fn [event]
+                            (when (current-token? token)
+                              (doseq [command
+                                      (result-commands
+                                       event
+                                       session-id
+                                       token
+                                       (get-in (get-state)
+                                               [:config :interimResults]))]
+                                (dispatch command)))))
+                    (set! (.-onerror instance)
+                          (fn [event]
+                            (fail-provider!
+                             instance
+                             token
+                             session-id
+                             (or (.-error event)
+                                 "speech-recognition-error"))))
+                    (set! (.-onend instance)
+                          (fn []
+                            (when (retire-current-provider! instance token)
+                              (stop-interruption-monitoring!)
+                              (dispatch {:type "providerEnded"
+                                         :sessionId session-id
+                                         :providerToken token}))))
+                    (if (or (runtime-value runtime :getUserMedia)
+                            (media-devices))
+                      (-> (ensure-microphone-stream!)
+                          (.then begin!)
+                          (.catch
+                           (fn [error]
+                             (fail-provider!
+                              instance token session-id
+                              (or (.-message error)
+                                  "microphone-start-error"))))
+                          (.then (fn [_] nil)))
+                      (do
+                        (begin! nil)
+                        (.resolve js/Promise nil))))
+                  (do
+                    (dispatch {:type "providerError"
+                               :message "Web Speech recognition is not available"
+                               :sessionId session-id})
+                    (.resolve js/Promise nil)))))
+
+            (retry-provider! [session-id delay-ms]
+              (cancel-retry!)
+              (when @recognition
+                (stop-provider! "abort"))
+              (reset! retry-timer
+                      (js/setTimeout
+                       (fn []
+                         (reset! retry-timer nil)
+                         (let [current-state (get-state)]
+                           (when (and (not @disposed?)
+                                      (:active current-state)
+                                      (= session-id (:sessionId current-state)))
+                             (-> (start-provider!
+                                  session-id (:config current-state))
+                                 (.catch (fn [_] nil))))))
+                       delay-ms))
+              nil)
+
+            (dispose-browser-resources! []
+              (stop-interruption-monitoring!)
+              (swap! reference-update-token inc)
+              (reset! reference-track nil)
+              (stop-media-stream! @mic-stream)
+              (reset! mic-stream nil)
+              (disconnect! @mic-source-node)
+              (disconnect! @mic-analyser)
+              (disconnect! @reference-source-node)
+              (disconnect! @reference-analyser)
+              (reset! mic-source-node nil)
+              (reset! mic-analyser nil)
+              (reset! mic-analyser-data nil)
+              (reset! reference-source-node nil)
+              (reset! reference-analyser nil)
+              (reset! reference-analyser-data nil)
+              (when-let [context @analysis-context]
+                (try (.close context) (catch :default _ nil)))
+              (reset! analysis-context nil))]
+      {:start
+       (fn [session-id config]
+         (start-provider! session-id config))
+
+       :stop
+       (fn []
+         (stop-provider! "stop"))
+
+       :abort
+       (fn []
+         (stop-provider! "abort"))
+
+       :retry
+       (fn [session-id delay-ms]
+         (retry-provider! session-id delay-ms))
+
+       :configure
+       (fn [config]
+         (apply-config! @recognition config)
+         (if (and @agent-speaking?
+                  (:interruptDetectionEnabled config))
+           (start-interruption-monitoring!)
+           (stop-interruption-monitoring!))
+         nil)
+
+       :set-reference-track replace-reference-analyser!
+       :sync-agent-speaking sync-agent-speaking!
+
+       :dispose
+       (fn []
+         (when-not @disposed?
+           (reset! disposed? true)
+           (stop-provider! "abort")
+           (dispose-browser-resources!)))})))