@lovelace_lol/polymer 0.0.7 → 0.0.8

package/src/polymer/transcription/effector.cljs

@@ -1,611 +0,0 @@
-(ns polymer.transcription.effector)
-
-;; This namespace is the only Transcription code allowed to touch browser
-;; recognition, microphone, AudioContext, or animation-frame APIs. The planner
-;; proposes work and the scheduler invokes this private effector.
-
-(defn runtime-value
-  [runtime key]
-  (cond
-    (map? runtime) (get runtime key)
-    runtime (aget runtime (name key))
-    :else nil))
-
-(defn recognition-constructor
-  []
-  (let [win (when (exists? js/window) js/window)]
-    (or (when win (aget win "SpeechRecognition"))
-        (when win (aget win "webkitSpeechRecognition")))))
-
-(defn create-recognition
-  [runtime]
-  (if-let [factory (runtime-value runtime :createRecognition)]
-    (factory)
-    (when-let [ctor (recognition-constructor)]
-      (ctor.))))
-
-(defn create-audio-context
-  [runtime]
-  (if-let [factory (runtime-value runtime :createAudioContext)]
-    (factory)
-    (let [win (when (exists? js/window) js/window)
-          ctor (or (when win (aget win "AudioContext"))
-                   (when win (aget win "webkitAudioContext")))]
-      (when ctor
-        (ctor.)))))
-
-(defn media-devices
-  []
-  (let [navigator (when (exists? js/navigator) js/navigator)]
-    (when navigator (.-mediaDevices navigator))))
-
-(defn apply-config!
-  [recognition config]
-  (when recognition
-    (set! (.-lang recognition) (:lang config))
-    (set! (.-continuous recognition) (boolean (:continuous config)))
-    (set! (.-interimResults recognition) (boolean (:interimResults config)))
-    (set! (.-maxAlternatives recognition) (:maxAlternatives config))))
-
-(defn result-commands
-  [event session-id provider-token interim-results?]
-  (let [results (.-results event)
-        start (or (.-resultIndex event) 0)
-        total (if results (.-length results) 0)]
-    (loop [index start
-           commands []]
-      (if (< index total)
-        (let [result (aget results index)
-              alternative (when result (aget result 0))
-              text (when alternative (.-transcript alternative))
-              confidence (when alternative (.-confidence alternative))
-              final? (boolean (when result (.-isFinal result)))]
-          (recur (inc index)
-                 (cond-> commands
-                   (and text (or final? interim-results?))
-                   (conj {:type (if final? "providerFinal" "providerPartial")
-                          :text text
-                          :confidence (if (number? confidence) confidence 1)
-                          :source "webSpeech"
-                          :sessionId session-id
-                          :providerToken provider-token}))))
-        commands))))
-
-(defn clear-handler!
-  [recognition handler-name]
-  (when recognition
-    (aset recognition handler-name nil)))
-
-(defn detach-handlers!
-  [recognition]
-  (doseq [handler-name ["onstart" "onresult" "onerror" "onend"]]
-    (clear-handler! recognition handler-name)))
-
-(defn disconnect!
-  [node]
-  (when node
-    (try
-      (.disconnect node)
-      (catch :default _ nil))))
-
-(defn stop-media-stream!
-  [stream]
-  (when stream
-    (doseq [track (array-seq (.getTracks stream))]
-      (try
-        (.stop track)
-        (catch :default _ nil)))))
-
-(defn read-level
-  [analyser data]
-  (if (and analyser data)
-    (do
-      (.getByteTimeDomainData analyser data)
-      (let [length (.-length data)]
-        (loop [index 0
-               sum 0]
-          (if (< index length)
-            (let [sample (/ (- (aget data index) 128) 128)]
-              (recur (inc index) (+ sum (* sample sample))))
-            (js/Math.sqrt (/ sum (max 1 length)))))))
-    0))
-
-(defn live-track
-  [stream]
-  (when stream
-    (some #(when (= "live" (.-readyState %)) %)
-          (array-seq (.getAudioTracks stream)))))
-
-(defn create-effector
-  [{:keys [dispatch get-state runtime]}]
-  (let [recognition (atom nil)
-        active-token (atom 0)
-        failed-tokens (atom #{})
-        retry-timer (atom nil)
-        disposed? (atom false)
-        mic-stream (atom nil)
-        mic-request (atom nil)
-        analysis-context (atom nil)
-        mic-analyser (atom nil)
-        mic-analyser-data (atom nil)
-        mic-source-node (atom nil)
-        reference-track (atom nil)
-        reference-analyser (atom nil)
-        reference-analyser-data (atom nil)
-        reference-source-node (atom nil)
-        reference-update-token (atom 0)
-        agent-speaking? (atom false)
-        interruption-frame (atom nil)
-        interruption-candidate-start (atom nil)
-        interruption-latched? (atom false)
-        last-debug-at (atom 0)]
-    (letfn [(current-config []
-              (:config (get-state)))
-
-            (current-token? [token]
-              (and (not @disposed?)
-                   (= token @active-token)))
-
-            (cancel-retry! []
-              (when-let [timer @retry-timer]
-                (js/clearTimeout timer)
-                (reset! retry-timer nil)))
-
-            (invalidate-provider! []
-              (swap! active-token inc)
-              (reset! failed-tokens #{}))
-
-            (retire-current-provider! [instance token]
-              (when (and (= instance @recognition)
-                         (current-token? token))
-                (swap! active-token inc)
-                (reset! recognition nil)
-                (detach-handlers! instance)
-                true))
-
-            (abort-retired-provider! [instance]
-              (try
-                (when-let [abort (aget instance "abort")]
-                  (.call abort instance))
-                (catch :default _ nil)))
-
-            (request-frame! [callback]
-              (if-let [request-frame (runtime-value runtime :requestAnimationFrame)]
-                (request-frame callback)
-                (let [win (when (exists? js/window) js/window)]
-                  (when (and win (.-requestAnimationFrame win))
-                    (.requestAnimationFrame win callback)))))
-
-            (cancel-frame! [frame]
-              (if-let [cancel-frame (runtime-value runtime :cancelAnimationFrame)]
-                (cancel-frame frame)
-                (let [win (when (exists? js/window) js/window)]
-                  (when (and win (.-cancelAnimationFrame win))
-                    (.cancelAnimationFrame win frame)))))
-
-            (clock-ms []
-              (let [performance (when (exists? js/performance) js/performance)]
-                (if performance (.now performance) (.now js/Date))))
-
-            (stop-interruption-monitoring! []
-              (when-let [frame @interruption-frame]
-                (cancel-frame! frame)
-                (reset! interruption-frame nil))
-              (reset! interruption-candidate-start nil))
-
-            (schedule-interruption-frame! [tick]
-              (when-let [frame (request-frame! tick)]
-                (reset! interruption-frame frame)))
-
-            (start-interruption-monitoring! []
-              (stop-interruption-monitoring!)
-              (when (and @agent-speaking?
-                         (:interruptDetectionEnabled (current-config)))
-                (letfn [(tick []
-                          (reset! interruption-frame nil)
-                          (if (or @disposed?
-                                  (not @agent-speaking?)
-                                  (not (:interruptDetectionEnabled
-                                        (current-config))))
-                            (stop-interruption-monitoring!)
-                            (let [config (current-config)
-                                  live-reference? (and @reference-track
-                                                       (= "live"
-                                                          (.-readyState
-                                                           @reference-track))
-                                                       @reference-analyser)]
-                              (if (and (:requireAgentReferenceForInterruption config)
-                                       (not live-reference?))
-                                (do
-                                  (reset! interruption-candidate-start nil)
-                                  (schedule-interruption-frame! tick))
-                                (let [microphone-level
-                                      (read-level @mic-analyser @mic-analyser-data)
-                                      reference-level
-                                      (if live-reference?
-                                        (read-level @reference-analyser
-                                                    @reference-analyser-data)
-                                        0)
-                                      required-level
-                                      (js/Math.max
-                                       (:interruptionVolumeThreshold config)
-                                       (+ (* reference-level
-                                             (:interruptionReferenceScale config))
-                                          (:interruptionReferenceOffset config)))
-                                      interrupt? (>= microphone-level
-                                                     required-level)
-                                      now (clock-ms)]
-                                  (when (and (:interruptionDebugLogging config)
-                                             (>= (- now @last-debug-at) 500))
-                                    (reset! last-debug-at now)
-                                    (js/console.debug
-                                     "[Transcription] Interruption levels"
-                                     #js {:microphoneLevel microphone-level
-                                          :referenceLevel reference-level
-                                          :requiredLevel required-level
-                                          :shouldInterrupt interrupt?
-                                          :hasAgentReference live-reference?}))
-                                  (if (and interrupt?
-                                           (not @interruption-latched?))
-                                    (if-let [candidate-start
-                                             @interruption-candidate-start]
-                                      (when (>= (- now candidate-start)
-                                                (:interruptionHoldMs config))
-                                        (reset! interruption-latched? true)
-                                        (dispatch
-                                         {:type "providerAudioInterruption"
-                                          :timestamp (.now js/Date)
-                                          :microphoneLevel microphone-level
-                                          :referenceLevel reference-level
-                                          :requiredLevel required-level}))
-                                      (reset! interruption-candidate-start now))
-                                    (when-not interrupt?
-                                      (reset! interruption-candidate-start nil)))
-                                  (schedule-interruption-frame! tick))))))]
-                  (schedule-interruption-frame! tick))))
-
-            (ensure-analysis-context! []
-              (if @disposed?
-                (.resolve js/Promise nil)
-                (try
-                  (when (or (nil? @analysis-context)
-                            (= "closed" (.-state @analysis-context)))
-                    (reset! analysis-context (create-audio-context runtime)))
-                  (let [context @analysis-context]
-                    (if (and context (= "suspended" (.-state context)))
-                      (-> (.resume context)
-                          (.then (fn [_] context))
-                          (.catch (fn [_] context)))
-                      (.resolve js/Promise context)))
-                  (catch :default error
-                    (.reject js/Promise error)))))
-
-            (ensure-microphone-analyser! []
-              (-> (ensure-analysis-context!)
-                  (.then
-                   (fn [context]
-                     (when (and (not @disposed?) context @mic-stream)
-                       (let [track (live-track @mic-stream)
-                             old-source @mic-source-node
-                             old-track (when old-source
-                                         (live-track
-                                          (aget old-source "mediaStream")))]
-                         (when (and track (not= track old-track))
-                           (disconnect! old-source)
-                           (disconnect! @mic-analyser)
-                           (let [source (.createMediaStreamSource
-                                         context @mic-stream)
-                                 analyser (.createAnalyser context)]
-                             (set! (.-fftSize analyser) 2048)
-                             (set! (.-smoothingTimeConstant analyser) 0.65)
-                             (.connect source analyser)
-                             (reset! mic-source-node source)
-                             (reset! mic-analyser analyser)
-                             (reset! mic-analyser-data
-                                     (js/Uint8Array.
-                                      (.-fftSize analyser)))))))))))
-
-            (request-microphone! []
-              (if-let [get-user-media (runtime-value runtime :getUserMedia)]
-                (.resolve js/Promise (get-user-media))
-                (if-let [media (media-devices)]
-                  (let [supported (if (.-getSupportedConstraints media)
-                                    (.getSupportedConstraints media)
-                                    #js {})
-                        constraints
-                        (clj->js
-                         (cond-> {:echoCancellation true
-                                  :noiseSuppression true
-                                  :autoGainControl true}
-                           (boolean (aget supported "voiceIsolation"))
-                           (assoc :voiceIsolation true)))]
-                    (-> (.getUserMedia media #js {:audio constraints})
-                        (.catch (fn [_]
-                                  (.getUserMedia media #js {:audio true})))))
-                  ;; Worker and Node tests have no mediaDevices. Recognition can
-                  ;; still start through an injected provider.
-                  (.resolve js/Promise nil))))
-
-            (ensure-microphone-stream! []
-              (cond
-                @disposed?
-                (.resolve js/Promise nil)
-
-                (live-track @mic-stream)
-                (-> (ensure-microphone-analyser!)
-                    (.then (fn [_] @mic-stream)))
-
-                @mic-request
-                (:promise @mic-request)
-
-                :else
-                (let [token (js-obj)
-                      requested
-                      (-> (request-microphone!)
-                          (.then
-                           (fn [stream]
-                             (if @disposed?
-                               (do
-                                 (stop-media-stream! stream)
-                                 nil)
-                               (do
-                                 (reset! mic-stream stream)
-                                 (-> (ensure-microphone-analyser!)
-                                     (.then
-                                      (fn [_]
-                                        (if @disposed?
-                                          (do
-                                            (when (= stream @mic-stream)
-                                              (reset! mic-stream nil))
-                                            (stop-media-stream! stream)
-                                            nil)
-                                          stream)))))))))
-                      settled
-                      (-> requested
-                          (.then
-                           (fn [stream]
-                             (when (identical? token (:token @mic-request))
-                               (reset! mic-request nil))
-                             stream))
-                          (.catch
-                           (fn [error]
-                             (when (identical? token (:token @mic-request))
-                               (reset! mic-request nil))
-                             (throw error))))]
-                  (reset! mic-request {:token token :promise settled})
-                  settled)))
-
-            (replace-reference-analyser! [track]
-              (let [token (swap! reference-update-token inc)]
-                (reset! reference-track track)
-                (disconnect! @reference-source-node)
-                (disconnect! @reference-analyser)
-                (reset! reference-source-node nil)
-                (reset! reference-analyser nil)
-                (reset! reference-analyser-data nil)
-                (when (and track (= "live" (.-readyState track)))
-                  (-> (ensure-analysis-context!)
-                      (.then
-                       (fn [context]
-                         (when (and context
-                                    (not @disposed?)
-                                    (= token @reference-update-token)
-                                    (= track @reference-track)
-                                    (= "live" (.-readyState track)))
-                           (let [stream-constructor
-                                 (or (runtime-value runtime :createMediaStream)
-                                     (when (exists? js/MediaStream)
-                                       (fn [tracks]
-                                         (js/MediaStream. tracks))))
-                                 stream (when stream-constructor
-                                          (stream-constructor #js [track]))]
-                             (when stream
-                               (let [source (.createMediaStreamSource
-                                             context stream)
-                                     analyser (.createAnalyser context)]
-                                 (set! (.-fftSize analyser) 2048)
-                                 (set! (.-smoothingTimeConstant analyser) 0.65)
-                                 (.connect source analyser)
-                                 (if (and (= token @reference-update-token)
-                                          (= track @reference-track)
-                                          (= "live" (.-readyState track)))
-                                   (do
-                                     (reset! reference-source-node source)
-                                     (reset! reference-analyser analyser)
-                                     (reset! reference-analyser-data
-                                             (js/Uint8Array.
-                                              (.-fftSize analyser))))
-                                   (do
-                                     (disconnect! source)
-                                     (disconnect! analyser)))))))))
-                      (.catch (fn [_] nil))))
-                (when @agent-speaking?
-                  (start-interruption-monitoring!))))
-
-            (sync-agent-speaking! [speaking?]
-              (reset! agent-speaking? (boolean speaking?))
-              (reset! interruption-latched? false)
-              (reset! interruption-candidate-start nil)
-              (if @agent-speaking?
-                (-> (ensure-microphone-analyser!)
-                    (.then (fn [_]
-                             (start-interruption-monitoring!)))
-                    (.catch (fn [_] nil)))
-                (stop-interruption-monitoring!)))
-
-            (stop-provider! [method-name]
-              (cancel-retry!)
-              (stop-interruption-monitoring!)
-              (let [instance @recognition]
-                (invalidate-provider!)
-                (reset! recognition nil)
-                (when instance
-                  (detach-handlers! instance)
-                  (try
-                    (when-let [method (aget instance method-name)]
-                      (.call method instance))
-                    (catch :default _ nil)))
-                (.resolve js/Promise nil)))
-
-            (fail-provider! [instance token session-id message]
-              (when (and (current-token? token)
-                         (not (contains? @failed-tokens token)))
-                (swap! failed-tokens conj token)
-                (when (retire-current-provider! instance token)
-                  (abort-retired-provider! instance)
-                  (dispatch {:type "providerError"
-                             :message message
-                             :sessionId session-id
-                             :providerToken token}))))
-
-            (start-recognition-instance! [instance track]
-              (if track
-                (try
-                  (.start instance track)
-                  (catch :default _
-                    (.start instance)))
-                (.start instance)))
-
-            (start-provider! [session-id config]
-              (cancel-retry!)
-              (if @disposed?
-                (.resolve js/Promise nil)
-                (if-let [instance (create-recognition runtime)]
-                  (let [token (swap! active-token inc)
-                        begin! (fn [stream]
-                                 (when (current-token? token)
-                                   (try
-                                     (start-recognition-instance!
-                                      instance (live-track stream))
-                                     (catch :default error
-                                       (fail-provider!
-                                        instance token session-id
-                                        (or (.-message error)
-                                            "speech-recognition-start-error"))))))]
-                    (reset! recognition instance)
-                    (apply-config! instance config)
-                    (set! (.-onstart instance)
-                          (fn []
-                            (when (current-token? token)
-                              (dispatch {:type "providerStarted"
-                                         :sessionId session-id
-                                         :providerToken token})
-                              (when @agent-speaking?
-                                (start-interruption-monitoring!)))))
-                    (set! (.-onresult instance)
-                          (fn [event]
-                            (when (current-token? token)
-                              (doseq [command
-                                      (result-commands
-                                       event
-                                       session-id
-                                       token
-                                       (get-in (get-state)
-                                               [:config :interimResults]))]
-                                (dispatch command)))))
-                    (set! (.-onerror instance)
-                          (fn [event]
-                            (fail-provider!
-                             instance
-                             token
-                             session-id
-                             (or (.-error event)
-                                 "speech-recognition-error"))))
-                    (set! (.-onend instance)
-                          (fn []
-                            (when (retire-current-provider! instance token)
-                              (stop-interruption-monitoring!)
-                              (dispatch {:type "providerEnded"
-                                         :sessionId session-id
-                                         :providerToken token}))))
-                    (if (or (runtime-value runtime :getUserMedia)
-                            (media-devices))
-                      (-> (ensure-microphone-stream!)
-                          (.then begin!)
-                          (.catch
-                           (fn [error]
-                             (fail-provider!
-                              instance token session-id
-                              (or (.-message error)
-                                  "microphone-start-error"))))
-                          (.then (fn [_] nil)))
-                      (do
-                        (begin! nil)
-                        (.resolve js/Promise nil))))
-                  (do
-                    (dispatch {:type "providerError"
-                               :message "Web Speech recognition is not available"
-                               :sessionId session-id})
-                    (.resolve js/Promise nil)))))
-
-            (retry-provider! [session-id delay-ms]
-              (cancel-retry!)
-              (when @recognition
-                (stop-provider! "abort"))
-              (reset! retry-timer
-                      (js/setTimeout
-                       (fn []
-                         (reset! retry-timer nil)
-                         (let [current-state (get-state)]
-                           (when (and (not @disposed?)
-                                      (:active current-state)
-                                      (= session-id (:sessionId current-state)))
-                             (-> (start-provider!
-                                  session-id (:config current-state))
-                                 (.catch (fn [_] nil))))))
-                       delay-ms))
-              nil)
-
-            (dispose-browser-resources! []
-              (stop-interruption-monitoring!)
-              (swap! reference-update-token inc)
-              (reset! reference-track nil)
-              (stop-media-stream! @mic-stream)
-              (reset! mic-stream nil)
-              (disconnect! @mic-source-node)
-              (disconnect! @mic-analyser)
-              (disconnect! @reference-source-node)
-              (disconnect! @reference-analyser)
-              (reset! mic-source-node nil)
-              (reset! mic-analyser nil)
-              (reset! mic-analyser-data nil)
-              (reset! reference-source-node nil)
-              (reset! reference-analyser nil)
-              (reset! reference-analyser-data nil)
-              (when-let [context @analysis-context]
-                (try (.close context) (catch :default _ nil)))
-              (reset! analysis-context nil))]
-      {:start
-       (fn [session-id config]
-         (start-provider! session-id config))
-
-       :stop
-       (fn []
-         (stop-provider! "stop"))
-
-       :abort
-       (fn []
-         (stop-provider! "abort"))
-
-       :retry
-       (fn [session-id delay-ms]
-         (retry-provider! session-id delay-ms))
-
-       :configure
-       (fn [config]
-         (apply-config! @recognition config)
-         (if (and @agent-speaking?
-                  (:interruptDetectionEnabled config))
-           (start-interruption-monitoring!)
-           (stop-interruption-monitoring!))
-         nil)
-
-       :set-reference-track replace-reference-analyser!
-       :sync-agent-speaking sync-agent-speaking!
-
-       :dispose
-       (fn []
-         (when-not @disposed?
-           (reset! disposed? true)
-           (stop-provider! "abort")
-           (dispose-browser-resources!)))})))