learning_model 1.0.49 → 1.0.51

package/dist/index.bundle.js.LICENSE.txt

@@ -105,6 +105,23 @@
  * =============================================================================
  */
 
+/**
+ * @license
+ * Copyright 2019 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the 'License');
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an 'AS IS' BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
 /**
  * @license
  * Copyright 2020 Google Inc. All Rights Reserved.

package/dist/learning/mobilenet.d.ts

@@ -1,4 +1,5 @@
 import * as tf from '@tensorflow/tfjs';
+import '@tensorflow/tfjs-backend-wasm';
 import { io } from '@tensorflow/tfjs-core';
 import LearningInterface from './base';
 declare class LearningMobilenet implements LearningInterface {
@@ -39,6 +40,8 @@ declare class LearningMobilenet implements LearningInterface {
     private _convertToTfDataset;
     addData(label: string, data: any): Promise<void>;
     init(): Promise<void>;
+    private setupBackend;
+    private checkWasmSupport;
     train(): Promise<tf.History>;
     infer(data: any): Promise<Map<string, number>>;
     saveModel(handlerOrURL: io.IOHandler | string, config?: io.SaveConfig): Promise<void>;

package/dist/learning/mobilenet.js

@@ -38,6 +38,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const tf = __importStar(require("@tensorflow/tfjs"));
+require("@tensorflow/tfjs-backend-wasm"); // WebAssembly 백엔드 추가
 const tfjs_1 = require("@tensorflow/tfjs");
 const tf_1 = require("../utils/tf");
 const canvas_1 = require("../utils/canvas");
@@ -138,8 +139,10 @@ class LearningMobilenet {
         return __awaiter(this, void 0, void 0, function* () {
             try {
                 if (this.mobilenetModule !== null) {
-                    const cap = (0, tf_1.isTensor)(data) ? data : (0, tf_1.capture)(data, false);
-                    const predict = this.mobilenetModule.predict(cap);
+                    const cap = (0, tf_1.isTensor)(data) ? data : yield (0, tf_1.capture)(data, false);
+                    const predict = tf.tidy(() => {
+                        return this.mobilenetModule.predict(cap);
+                    });
                     const activation = yield predict.data();
                     const classIndex = this.registerClassNumber(label);
                     if (!this.imageExamples[classIndex]) {
@@ -149,6 +152,12 @@ class LearningMobilenet {
                     if (this.classNumber.length >= this.limitSize) {
                         this.isReady = true;
                     }
+                    // Dispose of the prediction tensor to free memory
+                    predict.dispose();
+                    // If cap is not a tensor, we don't need to dispose it. Otherwise, we should.
+                    if (!(0, tf_1.isTensor)(data)) {
+                        cap.dispose();
+                    }
                 }
                 else {
                     throw new Error('mobilenetModule is null');
@@ -164,6 +173,8 @@ class LearningMobilenet {
     init() {
         return __awaiter(this, void 0, void 0, function* () {
             try {
+                console.log('init call');
+                yield this.setupBackend();
                 this.mobilenetModule = yield (0, tf_1.loadModel)();
             }
             catch (error) {
@@ -172,6 +183,34 @@ class LearningMobilenet {
             }
         });
     }
+    setupBackend() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const isWasmSupported = yield this.checkWasmSupport();
+            if (isWasmSupported) {
+                yield tf.setBackend('wasm');
+                yield tf.ready();
+                console.log('Backend is set to WebAssembly');
+            }
+            else {
+                yield tf.setBackend('cpu');
+                yield tf.ready();
+                console.log('Backend is set to CPU');
+            }
+        });
+    }
+    checkWasmSupport() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield tf.setBackend('wasm');
+                yield tf.ready();
+                return true;
+            }
+            catch (error) {
+                console.warn('WASM backend is not supported in this environment.', error);
+                return false;
+            }
+        });
+    }
     // 모델 학습 처리
     train() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -246,8 +285,8 @@ class LearningMobilenet {
             try {
                 const classProbabilities = new Map();
                 const croppedImage = (0, canvas_1.cropTo)(data, 224, false);
+                const captured = yield (0, tf_1.capture)(croppedImage, false);
                 const logits = tf.tidy(() => {
-                    const captured = (0, tf_1.capture)(croppedImage, false);
                     return this.model.predict(captured);
                 });
                 const values = yield logits.data();

package/dist/learning/mobilenet.test.js

@@ -56,22 +56,24 @@ function ImagePathToTensor(imagePath) {
 }
 describe('LearningMobilenetImage', () => {
     const learning = new mobilenet_1.default({});
+    const image1Path = path.join(__dirname, '../../public/images/image1.jpeg');
+    const image2Path = path.join(__dirname, '../../public/images/image2.jpeg');
+    console.log('Resolved path for image1:', image1Path);
     beforeAll(() => __awaiter(void 0, void 0, void 0, function* () {
-        const image1Path = path.join(__dirname, '../../public/images/image1.jpeg');
-        const image2Path = path.join(__dirname, '../../public/images/image2.jpeg');
+        learning.init();
         imageTensor1 = yield ImagePathToTensor(image1Path);
         imageTensor2 = yield ImagePathToTensor(image2Path);
     }));
-    test('loads an image and converts it to a tensor', () => {
+    it('loads an image and converts it to a tensor', () => {
         expect(imageTensor1).toBeDefined();
         expect(imageTensor1 instanceof tf.Tensor).toBe(true);
         expect(imageTensor2).toBeDefined();
         expect(imageTensor2 instanceof tf.Tensor).toBe(true);
     });
-    test('mobilenet add data', () => {
-        learning.addData("라벨1", imageTensor1);
-        learning.addData("라벨1", imageTensor1);
-        learning.addData("라벨2", imageTensor2);
-        learning.addData("라벨2", imageTensor2);
-    });
+    // test('mobilenet add data', () => {
+    //   learning.addData("라벨1", imageTensor1);
+    //   learning.addData("라벨1", imageTensor1);
+    //   learning.addData("라벨2", imageTensor2);
+    //   learning.addData("라벨2", imageTensor2);
+    // });
 });

package/dist/lib/learning/mobilenet.d.ts

@@ -1,4 +1,5 @@
 import * as tf from '@tensorflow/tfjs';
+import '@tensorflow/tfjs-backend-wasm';
 import { io } from '@tensorflow/tfjs-core';
 import LearningInterface from './base';
 declare class LearningMobilenet implements LearningInterface {
@@ -39,6 +40,8 @@ declare class LearningMobilenet implements LearningInterface {
     private _convertToTfDataset;
     addData(label: string, data: any): Promise<void>;
     init(): Promise<void>;
+    private setupBackend;
+    private checkWasmSupport;
     train(): Promise<tf.History>;
     infer(data: any): Promise<Map<string, number>>;
     saveModel(handlerOrURL: io.IOHandler | string, config?: io.SaveConfig): Promise<void>;

package/dist/lib/utils/tf.d.ts

@@ -3,5 +3,5 @@ export declare function isTensor(c: any): c is tf.Tensor;
 export declare function loadModel(): Promise<tf.LayersModel>;
 export declare function mobileNetURL(version: number): string;
 export declare function imageToTensor(data: any): tf.Tensor3D;
-export declare function capture(rasterElement: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement, grayscale?: boolean): tf.Tensor<tf.Rank>;
+export declare function capture(rasterElement: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement, grayscale?: boolean): Promise<tf.Tensor<tf.Rank>>;
 export declare function cropTensor(img: tf.Tensor3D, grayscaleModel?: boolean, grayscaleInput?: boolean): tf.Tensor3D;

package/dist/utils/tf.d.ts

@@ -3,5 +3,5 @@ export declare function isTensor(c: any): c is tf.Tensor;
 export declare function loadModel(): Promise<tf.LayersModel>;
 export declare function mobileNetURL(version: number): string;
 export declare function imageToTensor(data: any): tf.Tensor3D;
-export declare function capture(rasterElement: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement, grayscale?: boolean): tf.Tensor<tf.Rank>;
+export declare function capture(rasterElement: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement, grayscale?: boolean): Promise<tf.Tensor<tf.Rank>>;
 export declare function cropTensor(img: tf.Tensor3D, grayscaleModel?: boolean, grayscaleInput?: boolean): tf.Tensor3D;

package/dist/utils/tf.js

@@ -90,15 +90,17 @@ function imageToTensor(data) {
 }
 exports.imageToTensor = imageToTensor;
 function capture(rasterElement, grayscale) {
-    return tf.tidy(() => {
-        const pixels = tf.browser.fromPixels(rasterElement);
-        // crop the image so we're using the center square
-        const cropped = cropTensor(pixels, grayscale);
-        // Expand the outer most dimension so we have a batch size of 1
-        const batchedImage = cropped.expandDims(0);
-        // Normalize the image between -1 and a1. The image comes in between 0-255
-        // so we divide by 127 and subtract 1.
-        return batchedImage.toFloat().div(tf.scalar(127)).sub(tf.scalar(1));
+    return __awaiter(this, void 0, void 0, function* () {
+        return tf.tidy(() => {
+            const pixels = tf.browser.fromPixels(rasterElement);
+            // crop the image so we're using the center square
+            const cropped = cropTensor(pixels, grayscale);
+            // Expand the outer most dimension so we have a batch size of 1
+            const batchedImage = cropped.expandDims(0);
+            // Normalize the image between -1 and a1. The image comes in between 0-255
+            // so we divide by 127 and subtract 1.
+            return batchedImage.toFloat().div(tf.scalar(127)).sub(tf.scalar(1));
+        });
     });
 }
 exports.capture = capture;

package/jest.config.js

@@ -2,7 +2,7 @@ module.exports = {
     preset: 'ts-jest',
     testEnvironment: 'node',
     // 기타 Jest 구성 옵션
-    testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
-    //testMatch: ['/lib/learning/data_model_test.ts'],
+    //testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
+    testMatch: ['**/mobile(*.)+(spec|test).[jt]s?(x)'],
     testPathIgnorePatterns: ['/node_modules/'],
 };

package/lib/learning/mobilenet.test.ts

@@ -20,25 +20,29 @@ async function ImagePathToTensor(imagePath: string): Promise<tf.Tensor3D> {
 
 describe('LearningMobilenetImage', () => {
   const learning = new LearningMobilenetImage({});
-  
+ 
+  const image1Path = path.join(__dirname, '../../public/images/image1.jpeg');
+  const image2Path = path.join(__dirname, '../../public/images/image2.jpeg');
+  console.log('Resolved path for image1:', image1Path);
   beforeAll(async () => {
-    const image1Path = path.join(__dirname, '../../public/images/image1.jpeg');
-    const image2Path = path.join(__dirname, '../../public/images/image2.jpeg');
+    learning.init();
     imageTensor1 = await ImagePathToTensor(image1Path);
     imageTensor2 = await ImagePathToTensor(image2Path);
   });
 
-  test('loads an image and converts it to a tensor', () => {
+
+  it('loads an image and converts it to a tensor', () => {
     expect(imageTensor1).toBeDefined();
     expect(imageTensor1 instanceof tf.Tensor).toBe(true);
     expect(imageTensor2).toBeDefined();
     expect(imageTensor2 instanceof tf.Tensor).toBe(true);
   });
 
-  test('mobilenet add data', () => {
-    learning.addData("라벨1", imageTensor1);
-    learning.addData("라벨1", imageTensor1);
-    learning.addData("라벨2", imageTensor2);
-    learning.addData("라벨2", imageTensor2);
-  });
+  // test('mobilenet add data', () => {
+  //   learning.addData("라벨1", imageTensor1);
+  //   learning.addData("라벨1", imageTensor1);
+  //   learning.addData("라벨2", imageTensor2);
+  //   learning.addData("라벨2", imageTensor2);
+  // });
 });
+

package/lib/learning/mobilenet.ts

@@ -5,6 +5,7 @@
 ///////////////////////////////////////////////////////////////////////////
 
 import * as tf from '@tensorflow/tfjs';
+import '@tensorflow/tfjs-backend-wasm';  // WebAssembly 백엔드 추가
 import { dispose } from '@tensorflow/tfjs';
 import { io } from '@tensorflow/tfjs-core';
 import LearningInterface from './base';
@@ -150,17 +151,32 @@ class LearningMobilenet implements LearningInterface {
   public async addData(label: string, data: any): Promise<void> {
     try {
       if (this.mobilenetModule !== null) {
-        const cap = isTensor(data) ? data : capture(data, false);
-        const predict = this.mobilenetModule.predict(cap) as tf.Tensor;
+        const cap = isTensor(data) ? data : await capture(data, false);
+
+        const predict = tf.tidy(() => {
+          return this.mobilenetModule!.predict(cap) as tf.Tensor;
+        });
+
         const activation = await predict.data() as Float32Array;
+
         const classIndex = this.registerClassNumber(label);
         if (!this.imageExamples[classIndex]) {
           this.imageExamples[classIndex] = [];
         }
         this.imageExamples[classIndex].push(activation);
+
         if(this.classNumber.length >= this.limitSize) {
           this.isReady = true;
         }
+
+        // Dispose of the prediction tensor to free memory
+        predict.dispose();
+
+        // If cap is not a tensor, we don't need to dispose it. Otherwise, we should.
+        if (!isTensor(data)) {
+          cap.dispose();
+        }
+
       } else {
         throw new Error('mobilenetModule is null');
       }
@@ -173,6 +189,8 @@ class LearningMobilenet implements LearningInterface {
 
   public async init() {
     try {
+      console.log('init call')
+      await this.setupBackend();
       this.mobilenetModule = await loadModel();
     } catch(error) {
       console.log('init Error', error);
@@ -180,6 +198,29 @@ class LearningMobilenet implements LearningInterface {
     } 
   } 
 
+  private async setupBackend() {
+    const isWasmSupported = await this.checkWasmSupport();
+    if (isWasmSupported) {
+      await tf.setBackend('wasm');
+      await tf.ready();
+      console.log('Backend is set to WebAssembly');
+    } else {
+      await tf.setBackend('cpu');
+      await tf.ready();
+      console.log('Backend is set to CPU');
+    }
+  }
+
+  private async checkWasmSupport(): Promise<boolean> {
+    try {
+      await tf.setBackend('wasm');
+      await tf.ready();
+      return true;
+    } catch (error) {
+      console.warn('WASM backend is not supported in this environment.', error);
+      return false;
+    }
+  }
 
   // 모델 학습 처리
   public async train(): Promise<tf.History> {
@@ -227,7 +268,9 @@ class LearningMobilenet implements LearningInterface {
       const trainData = datasets.trainDataset.batch(this.batchSize);
       const validationData = datasets.validationDataset.batch(this.batchSize);
       const optimizer = tf.train.adam(this.learningRate); 
+
       const trainModel = await this._createModel(optimizer);
+
       const jointModel = tf.sequential();
       jointModel.add(this.mobilenetModule!);
       jointModel.add(trainModel);
@@ -257,16 +300,19 @@ class LearningMobilenet implements LearningInterface {
     try {
       const classProbabilities = new Map<string, number>();
       const croppedImage = cropTo(data, 224, false);
+      const captured = await capture(croppedImage, false);
 
       const logits = tf.tidy(() => {
-        const captured = capture(croppedImage, false);
         return this.model!.predict(captured);
       });
+
       const values = await (logits as tf.Tensor<tf.Rank>).data();
       const EPSILON = 1e-6; // 매우 작은 값을 표현하기 위한 엡실론
+
       for (let i = 0; i < values.length; i++) {
         let probability = Math.max(0, Math.min(1, values[i])); // 확률 값을 0과 1 사이로 조정
         probability = probability < EPSILON ? 0 : probability; // 매우 작은 확률 값을 0으로 간주
+
         const className = this.classNumber[i]; // 클래스 이름
         const existingProbability = classProbabilities.get(className);
         if (existingProbability !== undefined) {
@@ -275,6 +321,7 @@ class LearningMobilenet implements LearningInterface {
           classProbabilities.set(className, probability);
         }
       }
+
       console.log('classProbabilities', classProbabilities);
       dispose(logits);
       return classProbabilities;

package/lib/utils/tf.ts

@@ -54,7 +54,7 @@ export function imageToTensor(data: any): tf.Tensor3D {
   return tensor;
 }
 
-export function capture(rasterElement: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement, grayscale?: boolean) {
+export async function capture(rasterElement: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement, grayscale?: boolean) {
   return tf.tidy(() => {
     const pixels = tf.browser.fromPixels(rasterElement);
 
@@ -70,6 +70,7 @@ export function capture(rasterElement: HTMLImageElement | HTMLVideoElement | HTM
   });
 }
 
+
 export function cropTensor( img: tf.Tensor3D, grayscaleModel?: boolean, grayscaleInput?: boolean ) : tf.Tensor3D {
   const size = Math.min(img.shape[0], img.shape[1]);
   const centerHeight = img.shape[0] / 2;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "learning_model",
-  "version": "1.0.49",
+  "version": "1.0.51",
   "description": "learning model develop",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -20,6 +20,7 @@
     "@tensorflow-models/mobilenet": "^2.1.0",
     "@tensorflow/tfjs": "^4.6.0",
     "@tensorflow/tfjs-layers": "^4.6.0",
+    "@tensorflow/tfjs-backend-wasm": "^4.20.0",
     "canvas": "^2.11.2",
     "learning_model": "^1.0.0"
   },