@christianriedl/utils 1.0.104 → 1.0.105

package/dist/iOpenAI.d.ts

@@ -6,6 +6,6 @@ export interface ICompleteData {
 }
 export interface IOpenAIService extends ISpeechService {
     complete(prompt: string): Promise<ICompleteData>;
-    completeImage(prompt: string, url: string): Promise<object>;
+    completeImage(prompt: string, url: string): Promise<ICompleteData>;
     completeJson(prompt: string, schema?: object): Promise<object>;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@christianriedl/utils",
-  "version": "1.0.104",
+  "version": "1.0.105",
   "description": "Interfaces, local storage, service worker, configuration, application state",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/components/Camera.vue

@@ -2,11 +2,12 @@
     import { onMounted, onUnmounted, ref } from "vue";
     import { MediaDeviceKind, Resolution } from '@christianriedl/utils';
 
-    const props = withDefaults(defineProps<{
-            resolution?: Resolution; facingMode?: string; autoplay?: boolean, playsinline?: boolean, constraints?: object }>(),
-           { resolution: { width: 1920, height: 1080}, facingMode: "environment", autoplay: true, playsinline: true  });
+    const props = withDefaults(defineProps<{resolution: Resolution; facingMode?: string; autoplay?: boolean, playsinline?: boolean, constraints?: any}>(),
+           { facingMode: "environment", autoplay: true, playsinline: true  });
     const emits = defineEmits(["loading", "started", "stopped", "paused", "resumed", "camera-change", "snapshot", "error" ]);
 
+    const resolution = props.resolution ? props.resolution : { width: 1920, height: 1080};
+
     onMounted(() => {
         if (!navigator.mediaDevices)
             throw new Error("Media devices not available");
@@ -28,8 +29,8 @@
 
     const constraints = props.constraints || {
         video: {
-            width: props.resolution.width,
-            height: props.resolution.height,
+            width: resolution.width,
+            height: resolution.height,
             facingMode: props.facingMode,
             deviceId: {},
         },
@@ -56,9 +57,7 @@
         emits("loading");
 
         try {
-            stream.value = await navigator.mediaDevices.getUserMedia(
-                constraints
-            );
+            stream.value = await navigator.mediaDevices.getUserMedia(constraints);
 
             if (!video.value) throw new Error("Video ref is null");
 
@@ -70,11 +69,11 @@
         }
     };
 
-    function snapshot (resolution: Resolution = props.resolution, type = "image/png", quality?: number): Promise<Blob | null>  {
+    function snapshot (res: Resolution = resolution, type = "image/png", quality?: number): Promise<string | undefined>  {
         if (!video.value) throw new Error("Video ref is null");
         if (!canvas.value) throw new Error("Canvas ref is null");
 
-        const { width, height } = resolution;
+        const { width, height } = res;
 
         canvas.value.width = width;
         canvas.value.height = height;
@@ -82,14 +81,9 @@
         canvas.value.getContext("2d")?.drawImage(video.value, 0, 0, width, height);
 
         return new Promise((resolve) => {
-            canvas.value?.toBlob(
-                (blob: Blob | null) => {
-                    emits("snapshot", blob);
-                    resolve(blob);
-                },
-                type,
-                quality
-            );
+            const data = canvas.value?.toDataURL(type, quality);
+            emits("snapshot", data);
+            resolve(data);
         });
     };
 
@@ -114,6 +108,14 @@
         stream.value?.getTracks().forEach((track) => track.stop());
         emits("stopped");
     };
+    defineExpose({
+        start,
+        stop,
+        pause,
+        resume,
+        snapshot,
+        devices
+    })
 </script>
 
 <template>

package/src/components/OpenAI.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-    import { inject, ref, toRef, computed, onUnmounted, InstanceType } from 'vue';
+    import { inject, ref, toRef, computed, onUnmounted } from 'vue';
     import { appStateSymbol, appConfigSymbol, getOpenAISymbol, IOpenAIService, ICompleteData, ESize } from '@christianriedl/utils';
     import Microphone from '@christianriedl/utils/src/components/Microphone.vue';
     import Camera from '@christianriedl/utils/src/components/Camera.vue';
@@ -22,8 +22,15 @@
     });
 
     async function onComplete() {
+        let answer: ICompleteData;
         replyLines.value = [ "WAITING ..." ];
-        const answer = await openAI.complete(question.value);
+        if (isCamera.value && camera.value) {
+            const dataurl = await camera.value.snapshot();
+            answer = await openAI.completeImage(question.value, dataurl!);
+        }
+        else {
+            answer = await openAI.complete(question.value);
+        }
         if (answer && answer.choices && answer.choices.length > 0) {
             replyLines.value = answer.choices[0].split('\n');
         }
@@ -36,25 +43,14 @@
     }
     function onCamera() {
         isCamera.value = !isCamera.value;
+        if (isCamera.value && !question.value) {
+            question.value = "Was erkennst du auf dem Bild ?";
+        }
         if (!isCamera.value && camera.value) {
             camera.value.stop();
             camera.value = undefined;
         }
     }
-    async function onSnapshot() {
-        if (camera.value) {
-            const blob = await camera.value?.snapshot();
-            const url = URL.createObjectURL(blob);
-            replyLines.value = [ "WAITING ..." ];
-            const answer = await openAI.complete(question.value);
-            if (answer && answer.choices && answer.choices.length > 0) {
-                replyLines.value = answer.choices[0].split('\n');
-            }
-            else {
-                replyLines.value = [ "NO CORRECT ANSWER" ];
-            }
-        }
-    }
 </script>
 
 <template>
@@ -66,16 +62,22 @@
             <v-col :cols="12 - qcols">
                 <v-btn @click="onComplete">AI</v-btn>
                 <microphone lang="de" @result="onResult"></microphone>
-                <v-btn :variant="tonal" :class="bg-office" @click="onCamera">
+                <v-btn variant="tonal" class="bg-office" @click="onCamera">
                     <v-icon size="large" icon="$webcam" color="primary"></v-icon>
                 </v-btn>
             </v-col>
         </v-row>
         <v-row v-if="isCamera" dense align="center">
-            <camera :resolution="{ width: 375, height: 812 }" ref="camera" autoplay></camera>
-            <v-btn @click="onSnapshot">Create snapshot</v-btn>
+            <v-col cols="6">
+                <camera :resolution="{ width: 640, height: 480 }" ref="camera" autoplay></camera>
+            </v-col>
+            <v-col cols="6">
+                <v-row v-for="(line, index) in replyLines" :key=index dense align="center">
+                    <v-col cols="12">{{line}}</v-col>
+                </v-row>
+            </v-col>
         </v-row>
-        <v-row v-for="(line, index) in replyLines" :key=index dense align="center">
+        <v-row v-else v-for="(line, index) in replyLines" :key=index dense align="center">
             <v-col cols="12">{{line}}</v-col>
         </v-row>
     </v-container>