wco_models 3.1.0.302 → 3.1.0.304

data/app/views/wco/newsproducer/_studio_1.js

@@ -0,0 +1,279 @@
+
+/*
+**/
+const logg = (a, b="", c=null) => {
+  if ('undefined' === typeof window) { return }
+  c = "string" === typeof c ? c : b.replace(/\W/g, "");
+  if (c.length > 0) {
+    window[c] = a;
+  }
+  console.log(`+++ ${b}:`, a); // eslint-disable-line no-console
+};
+
+let avatar_url = 'https://cdn.jsdelivr.net/gh/wasya-co/ishlib3js@0.0.3/public/vendor/models/avatars/brunette.glb'
+
+import * as THREE from 'three';
+
+import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
+import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
+
+import { TalkingHead } from "talkinghead";
+
+const params = new URLSearchParams(window.location.search);
+
+let config = {
+    sampleRate: 8000,
+    mood: 'neutral',
+    gain: 0.5,
+    lipsyncType: "visemes",
+    lipsyncLang: "en",
+    waitForAudioChunks: false,
+    enableMetrics: false,
+  };
+let width = 640;
+let height = 480;
+const FPS = 24;
+
+let frame = 0;
+const api_key    = params.get('api_key')
+const api_secret = params.get('api_secret')
+const wco_origin = params.get('wco_origin')
+const newspartial_id = params.get('newspartial_id')
+let totalFrames
+
+let camera, controls, head, renderer, scene;
+let faceTarget = new THREE.Vector3();
+let chunkedInput = null;
+var capturer = new CCapture( { format: 'webm', framerate: FPS } );
+
+async function init() {
+  if (newspartial_id) {
+    try {
+      const url = wco_origin + '/wco/api/newspartials/' + newspartial_id + '/config.json?api_key=' + api_key + '&api_secret=' + api_secret
+      chunkedInput = await fetch(url).then(r => r.json())
+      // API may return a JSON string payload
+      if (typeof chunkedInput === 'string') {
+        chunkedInput = JSON.parse(chunkedInput)
+      }
+      logg(chunkedInput, 'chunkedInput')
+      const last = chunkedInput.wtimes.length-1
+      const duration_ms = chunkedInput.wtimes[last] + chunkedInput.wdurations[last]
+      logg(duration_ms, 'duration_ms')
+      totalFrames = duration_ms/1000*FPS;
+    } catch (error) {
+      console.log(error);
+      chunkedInput = null;
+      totalFrames = undefined;
+    }
+  }
+
+  scene = new THREE.Scene();
+  scene.background = new THREE.Color(0xd3d3d3);
+
+  /* fov — Camera frustum vertical field of view.
+   * aspect — Camera frustum aspect ratio.
+   * near — Camera frustum near plane.
+   * far — Camera frustum far plane.
+  **/
+  camera = new THREE.PerspectiveCamera( 10, width/height, 0.1, 1000 )
+  camera.position.set( 0, 2, 8 )
+  camera.lookAt( 0, 0, 0 )
+
+  const ambientLight = new THREE.AmbientLight( 0xffffff );
+  scene.add( ambientLight );
+
+  const grid = new THREE.GridHelper(10, 10, 0x888888, 0xbbbbbb);
+  grid.position.y = 0;
+  scene.add(grid);
+
+  const nodeAvatar = document.getElementById('avatar');
+  head = new TalkingHead( nodeAvatar, {
+    avatarOnly: true,
+    avatarOnlyScene: scene,
+    avatarOnlyCamera: camera,
+
+    lipsyncModules: ["en"],
+  });
+  logg(head, 'head')
+
+
+
+  const nodeLoading = document.getElementById('loading');
+  try {
+    nodeLoading.textContent = "Loading...";
+    await head.showAvatar( {
+      url: avatar_url,
+      body: 'F',
+      avatarMood: 'neutral',
+      lipsyncLang: 'en'
+    }, (ev) => {
+      if ( ev.lengthComputable ) {
+        let val = Math.min(100,Math.round(ev.loaded/ev.total * 100 ));
+        nodeLoading.textContent = "Loading " + val + "%";
+      }
+    });
+    nodeLoading.style.display = 'none';
+
+    head.armature.position.set(0, 0, 0);
+    head.armature.rotation.set(0, 0, 0);
+    scene.add(head.armature);
+
+    // Put feet at the origin
+    head.armature.updateMatrixWorld(true);
+    const leftToe = new THREE.Vector3();
+    const rightToe = new THREE.Vector3();
+    head.objectLeftToeBase.getWorldPosition(leftToe);
+    head.objectRightToeBase.getWorldPosition(rightToe);
+    head.armature.position.set(
+      -(leftToe.x + rightToe.x) / 2,
+      -Math.min(leftToe.y, rightToe.y),
+      -(leftToe.z + rightToe.z) / 2
+    );
+    head.armature.updateMatrixWorld(true);
+
+    // Camera at head level, pointed at the face
+    const leftEye = new THREE.Vector3();
+    const rightEye = new THREE.Vector3();
+    head.objectLeftEye.getWorldPosition(leftEye);
+    head.objectRightEye.getWorldPosition(rightEye);
+    faceTarget.copy(leftEye).add(rightEye).multiplyScalar(0.5);
+
+    const camPos = new THREE.Vector3(0, 0, 2);
+    camPos.applyQuaternion(head.armature.quaternion);
+    camPos.add(faceTarget);
+    camera.position.copy(camPos);
+    camera.lookAt(faceTarget);
+
+    await head.streamStart({
+      sampleRate: config.sampleRate,
+      mood: config.mood,
+      gain: config.gain,
+      lipsyncType: config.lipsyncType,
+      lipsyncLang: config.lipsyncLang,
+      waitForAudioChunks: config.waitForAudioChunks,
+      metrics: config.enableMetrics ? { enabled: true, intervalHz: config.metricsInterval } : { enabled: false }
+    })
+
+  } catch (error) {
+    console.log(error);
+    nodeLoading.textContent = error.toString();
+  }
+
+
+  //
+
+  renderer = new THREE.WebGLRenderer( {
+    alpha: false,
+    antialias: true,
+  } );
+  renderer.setClearColor(0xd3d3d3, 1);
+  renderer.setPixelRatio( 1 ); // window.devicePixelRatio );
+  renderer.setSize( width, height );
+  renderer.toneMapping = THREE.ACESFilmicToneMapping;
+  renderer.toneMappingExposure = 1;
+  renderer.outputColorSpace = THREE.SRGBColorSpace;
+  document.getElementById('rotatingC').appendChild( renderer.domElement );
+
+  //
+
+  controls = new OrbitControls( camera, renderer.domElement );
+  controls.enableDamping = true;
+  controls.enableRotate = true;
+  controls.enablePan = true;
+  controls.enableZoom = true;
+  controls.minDistance = 0.5;
+  controls.maxDistance = 100;
+  controls.target.copy(faceTarget);
+  controls.autoRotate = false;
+  controls.update();
+
+  //
+
+  window.addEventListener( 'resize', onWindowResize );
+
+
+  // Keep rendering so OrbitControls drag/damping stay live
+  renderer.setAnimationLoop(animate);
+
+  // Start capture/audio only after the renderer exists
+  if (newspartial_id && chunkedInput) {
+    capturer.start();
+    logg('capturer.start')
+    head.streamAudio(chunkedInput);
+  }
+}
+
+let semafore = false
+document.addEventListener('DOMContentLoaded', async function(e) {
+  if (!semafore) {
+    semafore = true
+    await init()
+  }
+})
+
+function onWindowResize() {
+  camera.aspect = width / height;
+  camera.updateProjectionMatrix();
+  renderer.setSize( width, height );
+  render();
+}
+
+function render() {
+  renderer.render( scene, camera );
+}
+
+function animate() {
+  const t = frame / FPS;
+  if (head) head.animate(1000/FPS);
+  controls.update();
+  renderer.render( scene, camera );
+
+  if (totalFrames) {
+    capturer.capture( renderer.domElement );
+    frame++;
+    if (frame >= totalFrames) {
+      totalFrames = false
+      renderer.setAnimationLoop(null);
+      capturer.stop();
+      logg('capturer.stop')
+
+      if (newspartial_id) {
+        capturer.save((blob) => {
+          logg(blob, 'blob')
+          renderer.domElement.toBlob((thumb) => {
+            logg(thumb, 'thumb')
+            const form = new FormData();
+            form.append('video', blob, 'lips.webm')
+            form.append('name', '<ccapturejs>')
+            form.append('thumb', thumb)
+            form.append('newspartial_id', newspartial_id)
+
+            fetch(wco_origin + '/wco/api/videos/?api_key=' + api_key + '&api_secret=' + api_secret, {
+              method: 'POST',
+              headers: {
+                // 'Content-Type': 'video/webm',
+              },
+              body: form,
+            })
+          })
+        });
+      }
+      // Resume interactive orbit after capture
+      renderer.setAnimationLoop(animate);
+    }
+  }
+}
+
+
+// Speak when clicked
+const nodeSpeak = document.getElementById('speak');
+  nodeSpeak.addEventListener('click', function () {
+    try {
+
+    } catch (error) {
+      console.log(error);
+    }
+  });
+
+
+console.log('+++ loaded wco_models :: newsproducer :: studio_1.js')

data/app/views/wco/newsproducer/canvas.html.erb

@@ -0,0 +1,263 @@
+
+  <style>
+    body {
+      background: #efefef;
+    }
+  </style>
+
+  <script async src="https://unpkg.com/es-module-shims@1.3.6/dist/es-module-shims.js"></script>
+  <script src="https://code.jquery.com/jquery-3.7.1.min.js"></script>
+  <script src="/vendor/javascript/CCapture.all.min.js"></script>
+
+  <script type="importmap">
+  { "imports":
+    {
+      "three": "https://cdn.jsdelivr.net/npm/three@0.180.0/build/three.module.js/+esm",
+      "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.180.0/examples/jsm/",
+      "talkinghead": "https://cdn.jsdelivr.net/gh/met4citizen/TalkingHead@1.7/modules/talkinghead.mjs"
+    }
+  }
+  </script>
+
+
+
+  <h1 class='center text-center'>{{ label }}</h1>
+  <button id='speak'>Speak</button>
+  <div id='avatar'></div>
+  <div id='loading'>Loading...</div>
+  <div id="rotatingC" class="r2" style="width: 640px;
+    height: 480px;
+    border: 1px solid red;
+    text-align: center; " >
+  </div>
+
+  <script type="module">
+    const logg = (a, b="", c=null) => {
+      if ('undefined' === typeof window) { return }
+      c = "string" === typeof c ? c : b.replace(/\W/g, "");
+      if (c.length > 0) {
+        window[c] = a;
+      }
+      console.log(`+++ ${b}:`, a); // eslint-disable-line no-console
+    };
+
+    let avatar_url = 'https://cdn.jsdelivr.net/gh/wasya-co/ishlib3js@0.0.3/public/vendor/models/avatars/brunette.glb'
+
+    import * as THREE from 'three';
+
+    import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
+    import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
+
+    import { TalkingHead } from "talkinghead";
+
+    const params = new URLSearchParams(window.location.search);
+
+    let config = {
+        sampleRate: 8000,
+        mood: 'neutral',
+        gain: 0.5,
+        lipsyncType: "visemes",
+        lipsyncLang: "en",
+        waitForAudioChunks: false,
+        enableMetrics: false,
+      };
+    let width = 640;
+    let height = 480;
+    const FPS = 24;
+
+    let frame = 0;
+    const api_key    = params.get('api_key')
+    const api_secret = params.get('api_secret')
+    const wco_origin = params.get('wco_origin')
+    const newspartial_id = params.get('newspartial_id')
+    let totalFrames
+
+    let camera, controls, head, renderer, scene;
+    var capturer = new CCapture( { format: 'webm', framerate: FPS } );
+
+    async function init() {
+      if (newspartial_id) {
+        const url = wco_origin + '/wco/api/newspartials/' + newspartial_id + '/config.json?api_key=' + api_key + '&api_secret=' + api_secret
+        let chunkedInput = await fetch(url).then(r => r.json())
+        logg(chunkedInput, 'chunkedInput')
+        const last = chunkedInput.wtimes.length-1
+        const duration_ms = chunkedInput.wtimes[last] + chunkedInput.wdurations[last]
+        logg(duration_ms, 'duration_ms')
+        totalFrames = duration_ms/1000*FPS;
+      }
+
+      scene = new THREE.Scene();
+
+      /* fov — Camera frustum vertical field of view.
+       * aspect — Camera frustum aspect ratio.
+       * near — Camera frustum near plane.
+       * far — Camera frustum far plane.
+      **/ //
+      camera = new THREE.PerspectiveCamera( 10, width/height, 0.1, 10 )
+      camera.position.set( 0, 0, 0 )
+      camera.rotation.set( 0, 0, 0 )
+
+      const ambientLight = new THREE.AmbientLight( 0xffffff );
+      scene.add( ambientLight );
+
+      const nodeAvatar = document.getElementById('avatar');
+      head = new TalkingHead( nodeAvatar, {
+        avatarOnly: true,
+        avatarOnlyCamera: camera,
+
+        lipsyncModules: ["en"],
+        // cameraView: "upper",
+        // update: function () {
+        //   capturer.capture( renderer.domElement );
+        // }
+      });
+      logg(head, 'head')
+
+
+
+      const nodeLoading = document.getElementById('loading');
+      try {
+        nodeLoading.textContent = "Loading...";
+        await head.showAvatar( {
+          url: avatar_url,
+          body: 'F',
+          avatarMood: 'neutral',
+          lipsyncLang: 'en'
+        }, (ev) => {
+          if ( ev.lengthComputable ) {
+            let val = Math.min(100,Math.round(ev.loaded/ev.total * 100 ));
+            nodeLoading.textContent = "Loading " + val + "%";
+          }
+        });
+        nodeLoading.style.display = 'none';
+
+        head.armature.position.set(0,-3,0);
+        head.armature.rotation.set(0,1,0);
+        scene.add(head.armature);
+
+        await head.streamStart({
+              sampleRate: config.sampleRate,
+              mood: config.mood,
+              gain: config.gain,
+              lipsyncType: config.lipsyncType,
+              lipsyncLang: config.lipsyncLang,
+              waitForAudioChunks: config.waitForAudioChunks,
+              // Configure metrics: enabled/disabled and reporting rate
+              metrics: config.enableMetrics ? { enabled: true, intervalHz: config.metricsInterval } : { enabled: false }
+            })
+
+        if (newspartial_id) {
+          capturer.start();
+          logg('capturer.start')
+          head.streamAudio(chunkedInput);
+        }
+
+      } catch (error) {
+        console.log(error);
+        nodeLoading.textContent = error.toString();
+      }
+
+
+      //
+
+      renderer = new THREE.WebGLRenderer( {
+        alpha: false,
+        antialias: true,
+      } );
+      renderer.setPixelRatio( window.devicePixelRatio );
+      renderer.setSize( width, height );
+      renderer.toneMapping = THREE.ACESFilmicToneMapping;
+      renderer.toneMappingExposure = 1;
+      renderer.outputEncoding = THREE.sRGBEncoding;
+      document.getElementById('rotatingC').appendChild( renderer.domElement );
+
+      //
+
+      controls = new OrbitControls( camera, renderer.domElement );
+      controls.enableDamping = true;
+      controls.minDistance = 0.5;
+      controls.maxDistance = 100;
+      controls.autoRotate = false;
+      controls.update();
+
+      //
+
+      window.addEventListener( 'resize', onWindowResize );
+
+
+      // renderer.setAnimationLoop(animate);
+      animate()
+    }
+
+    let semafore = false
+    document.addEventListener('DOMContentLoaded', async function(e) {
+      if (!semafore) {
+        semafore = true
+        await init()
+      }
+    })
+
+    function onWindowResize() {
+      camera.aspect = w / h;
+      camera.updateProjectionMatrix();
+      renderer.setSize( w, h );
+      render();
+    }
+
+    function render() {
+      renderer.render( scene, camera );
+    }
+
+    function animate() {
+      const t = frame / FPS;
+      head.animate(1000/FPS);
+      controls.update();
+      renderer.render( scene, camera );
+      capturer.capture( renderer.domElement );
+
+      frame++;
+      if (frame < totalFrames) {
+        requestAnimationFrame(animate)
+      } else {
+        capturer.stop();
+        logg('capturer.stop')
+
+        if (wco_origin) {
+          capturer.save((blob) => {
+            logg(blob, 'blob')
+            renderer.domElement.toBlob((thumb) => {
+              logg(thumb, 'thumb')
+              const form = new FormData();
+              form.append('video', blob, 'lips.webm')
+              form.append('name', '<ccapturejs>')
+              form.append('thumb', thumb)
+              form.append('newspartial_id', newspartial_id)
+
+              fetch(wco_origin + '/wco/api/videos/?api_key=' + api_key + '&api_secret=' + api_secret, {
+                method: 'POST',
+                headers: {
+                  // 'Content-Type': 'video/webm',
+                },
+                body: form,
+              })
+            })
+          });
+        }
+
+
+      }
+    }
+
+
+    // Speak when clicked
+    const nodeSpeak = document.getElementById('speak');
+      nodeSpeak.addEventListener('click', function () {
+        try {
+
+        } catch (error) {
+          console.log(error);
+        }
+      });
+
+
+  </script>

data/app/views/wco/newsproducer/canvas_minimal.html.erb

@@ -0,0 +1,132 @@
+<script src="/vendor/javascript/CCapture.all.min.js"></script>
+  <style>
+    body, html { width:100%; height:100%; max-width: 800px; margin: auto; position: relative; background-color: #202020; color: white; }
+    #avatar { display: block; width:100%; height:100%; border: 1px solid yellow; }
+
+    #controls { display: block; position: absolute; top: 10px; left: 10px; right: 10px; height: 50px; }
+    #text { position: absolute; width: Calc( 100% - 110px ); height: 100%; top: 0; left: 0; bottom: 0; right: 110px; font-family: Arial; font-size: 20px; }
+    #speak { display: block; position: absolute; top: 0; bottom: 0; right: 0; height: 100%; width: 100px; font-family: Arial; font-size: 20px; }
+    #loading { display: block; position: absolute; bottom: 10px; left: 10px; right: 10px; height: 50px; font-family: Arial; font-size: 20px; }
+  </style>
+
+  <script type="importmap">
+  { "imports":
+    {
+      "three": "https://cdn.jsdelivr.net/npm/three@0.180.0/build/three.module.js/+esm",
+      "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.180.0/examples/jsm/",
+      "talkinghead": "https://cdn.jsdelivr.net/gh/met4citizen/TalkingHead@1.7/modules/talkinghead.mjs"
+    }
+  }
+  </script>
+
+  <script type="module">
+    const logg = (a, b="", c=null) => {
+      if ('undefined' === typeof window) { return }
+      c = "string" === typeof c ? c : b.replace(/\W/g, "");
+      if (c.length > 0) {
+        window[c] = a;
+      }
+      console.log(`+++ ${b}:`, a); // eslint-disable-line no-console
+    };
+    let avatar_url = 'https://cdn.jsdelivr.net/gh/wasya-co/ishlib3js@0.0.3/public/vendor/models/avatars/brunette.glb'
+
+
+    import { TalkingHead } from "talkinghead";
+
+    let head;
+    let camera, controls, renderer, scene;
+
+    document.addEventListener('DOMContentLoaded', async function(e) {
+
+      // Instantiate the class
+      // NOTE: Never put your API key in a client-side code unless you know
+      //       that you are the only one to have access to that code!
+      const nodeAvatar = document.getElementById('avatar');
+      head = new TalkingHead( nodeAvatar, {
+        ttsEndpoint: "https://eu-texttospeech.googleapis.com/v1beta1/text:synthesize",
+        ttsApikey: "put-your-own-Google-TTS-API-key-here", // <- Change this
+        lipsyncModules: ["en", "fi"],
+        cameraView: "upper",
+      });
+
+      // Load and show the avatar
+      const nodeLoading = document.getElementById('loading');
+      try {
+        nodeLoading.textContent = "Loading...";
+        await head.showAvatar( {
+          url: avatar_url,
+          body: 'F',
+          avatarMood: 'neutral',
+          ttsLang: "en-GB",
+          ttsVoice: "en-GB-Standard-A",
+          lipsyncLang: 'en'
+        }, (ev) => {
+          if ( ev.lengthComputable ) {
+            let val = Math.min(100,Math.round(ev.loaded/ev.total * 100 ));
+            nodeLoading.textContent = "Loading " + val + "%";
+          }
+        });
+        nodeLoading.style.display = 'none';
+      } catch (error) {
+        console.log(error);
+        nodeLoading.textContent = error.toString();
+      }
+
+      // Speak when clicked
+      const nodeSpeak = document.getElementById('speak');
+      nodeSpeak.addEventListener('click', function () {
+        try {
+          const text = document.getElementById('text').value;
+          if ( text ) {
+            head.speakText( text );
+          }
+        } catch (error) {
+          console.log(error);
+        }
+      });
+
+      // Pause animation when document is not visible
+      document.addEventListener("visibilitychange", async function (ev) {
+        if (document.visibilityState === "visible") {
+          head.start();
+        } else {
+          head.stop();
+        }
+      });
+
+      // herehere
+      const params = new URLSearchParams(window.location.search);
+
+      const w = 1024;
+      const h = 576;
+      let config = {
+          sampleRate: 8000,
+          mood: 'neutral',
+          gain: 0.5,
+          lipsyncType: "visemes",
+          lipsyncLang: "en",
+          waitForAudioChunks: false,
+          enableMetrics: false,
+        };
+      const FPS = 24;
+      let frame = 0;
+      const api_key = params.get('api_key')
+      const api_secret = params.get('api_secret')
+      const wco_origin = params.get('wco_origin')
+      const newspartial_id = params.get('newspartial_id')
+      let totalFrames
+
+
+      var capturer = new CCapture( { format: 'webm', framerate: FPS } );
+
+    });
+
+  </script>
+
+  <div id="avatar"></div>
+  <div id="controls">
+    <input id="text" type="text" value="Hi there. How are you? I'm fine.">
+    <input id="speak" type="button" value="Speak">
+  </div>
+  <div id="loading"></div>
+

data/app/views/wco/newsproducer/studio_1.html.erb

@@ -0,0 +1,26 @@
+
+<script async src="https://unpkg.com/es-module-shims@1.3.6/dist/es-module-shims.js"></script>
+<script src="https://code.jquery.com/jquery-3.7.1.min.js"></script>
+<script src="/vendor/javascript/CCapture.all.min.js"></script>
+
+<script type="importmap">
+{ "imports":
+  {
+    "three": "https://cdn.jsdelivr.net/npm/three@0.180.0/build/three.module.js/+esm",
+    "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.180.0/examples/jsm/",
+    "talkinghead": "https://cdn.jsdelivr.net/gh/met4citizen/TalkingHead@1.7/modules/talkinghead.mjs"
+  }
+}
+</script>
+
+<h1 class='center text-center'>{{ label }}</h1>
+<button id='speak'>Speak</button>
+<div id='avatar'></div>
+<div id='loading'>Loading...</div>
+<div id="rotatingC" class="r2" style="width: 640px;
+  height: 480px;
+  border: 1px solid red;
+  text-align: center; " >
+</div>
+
+<script type="module"><%= render 'studio_1.js' %></script>