learning_model 1.0.28 → 1.0.30

package/dist/learning/util.js

@@ -32,6 +32,7 @@ function ImageToTensor(data) {
         tensor = data;
     }
     else {
+        // MobileNet 모델 로드
         tensor = tf.browser.fromPixels(data);
     }
     return tensor;

package/dist/lib/index.d.ts

@@ -1,3 +1,4 @@
 import LearningImage from './learning/image';
 import LearningMobilenetImage from './learning/mobilenet_image';
-export { LearningImage, LearningMobilenetImage };
+import LearningMobilenet from './learning/mobilenet';
+export { LearningImage, LearningMobilenetImage, LearningMobilenet };

package/dist/lib/learning/base.d.ts

@@ -1,4 +1,5 @@
 import * as tf from '@tensorflow/tfjs';
+import { io } from '@tensorflow/tfjs-core';
 interface LearningInterface {
     model: tf.LayersModel | null;
     labels: string[];
@@ -6,13 +7,13 @@ interface LearningInterface {
     isReady: boolean;
     onProgress(progress: number): void;
     onLoss(loss: number): void;
-    onEvents(logs: any): void;
     onTrainBegin(log: any): void;
     onTrainEnd(log: any): void;
+    onEpochEnd(epoch: number, logs: any): void;
     addData(label: string, data: any): Promise<void>;
     train(): Promise<tf.History>;
     infer(data: any): Promise<any>;
-    saveModel(): Promise<void>;
+    saveModel(handlerOrURL: io.IOHandler | string, config?: io.SaveConfig): Promise<void>;
     running(): boolean;
     ready(): boolean;
 }

package/dist/lib/learning/image.d.ts

@@ -1,4 +1,5 @@
 import * as tf from '@tensorflow/tfjs';
+import { io } from '@tensorflow/tfjs-core';
 import LearningInterface from './base';
 declare class LearningImage implements LearningInterface {
     model: tf.LayersModel | null;
@@ -29,10 +30,11 @@ declare class LearningImage implements LearningInterface {
     onEvents: (logs: any) => void;
     onTrainBegin: (log: any) => void;
     onTrainEnd: (log: any) => void;
+    onEpochEnd: (epoch: number, logs: any) => void;
     addData(label: string, data: any): Promise<void>;
     train(): Promise<tf.History>;
     infer(data: any): Promise<Map<string, number>>;
-    saveModel(): Promise<void>;
+    saveModel(handlerOrURL: io.IOHandler | string, config?: io.SaveConfig): Promise<void>;
     running(): boolean;
     ready(): boolean;
     private _preprocessedTargetData;

package/dist/lib/learning/mobilenet.d.ts

@@ -0,0 +1,49 @@
+import * as tf from '@tensorflow/tfjs';
+import { io } from '@tensorflow/tfjs-core';
+import LearningInterface from './base';
+declare class LearningMobilenet implements LearningInterface {
+    model: tf.LayersModel | null;
+    epochs: number;
+    batchSize: number;
+    learningRate: number;
+    validateRate: number;
+    labels: string[];
+    modelURL: string;
+    isRunning: boolean;
+    isReady: boolean;
+    isTrainedDone: boolean;
+    limitSize: number;
+    mobilenetModule: tf.LayersModel | null;
+    imageTensors: tf.Tensor<tf.Rank>[];
+    readonly MOBILE_NET_INPUT_WIDTH = 224;
+    readonly MOBILE_NET_INPUT_HEIGHT = 224;
+    readonly MOBILE_NET_INPUT_CHANNEL = 3;
+    readonly IMAGE_NORMALIZATION_FACTOR = 255;
+    constructor({ modelURL, // 디폴트 mobilenet 이미지 
+    epochs, batchSize, limitSize, learningRate, validateRate, }?: {
+        modelURL?: string;
+        epochs?: number;
+        batchSize?: number;
+        limitSize?: number;
+        learningRate?: number;
+        validateRate?: number;
+    });
+    onProgress: (progress: number) => void;
+    onLoss: (loss: number) => void;
+    onEvents: (logs: any) => void;
+    onTrainBegin: (log: any) => void;
+    onTrainEnd: (log: any) => void;
+    onEpochEnd: (epoch: number, logs: any) => void;
+    addData(label: string, data: any): Promise<void>;
+    init(): Promise<void>;
+    train(): Promise<tf.History>;
+    infer(data: any): Promise<Map<string, number>>;
+    saveModel(handlerOrURL: io.IOHandler | string, config?: io.SaveConfig): Promise<void>;
+    running(): boolean;
+    ready(): boolean;
+    private _preprocessedTargetData;
+    private _preprocessedInputData;
+    private loadModel;
+    private _createModel;
+}
+export default LearningMobilenet;

package/dist/lib/learning/mobilenet.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/lib/learning/mobilenet_image.d.ts

@@ -1,4 +1,5 @@
 import * as tf from '@tensorflow/tfjs';
+import { io } from '@tensorflow/tfjs-core';
 import LearningInterface from './base';
 declare class LearningMobilenetImage implements LearningInterface {
     model: tf.LayersModel | null;
@@ -31,10 +32,11 @@ declare class LearningMobilenetImage implements LearningInterface {
     onEvents: (logs: any) => void;
     onTrainBegin: (log: any) => void;
     onTrainEnd: (log: any) => void;
+    onEpochEnd: (epoch: number, logs: any) => void;
     addData(label: string, data: any): Promise<void>;
     train(): Promise<tf.History>;
     infer(data: any): Promise<Map<string, number>>;
-    saveModel(): Promise<void>;
+    saveModel(handlerOrURL: io.IOHandler | string, config?: io.SaveConfig): Promise<void>;
     running(): boolean;
     ready(): boolean;
     private _preprocessedTargetData;

package/lib/index.ts

@@ -1,4 +1,5 @@
 import LearningImage from './learning/image';
 import LearningMobilenetImage from './learning/mobilenet_image';    
+import LearningMobilenet from './learning/mobilenet';    
 
-export { LearningImage, LearningMobilenetImage };
+export { LearningImage, LearningMobilenetImage, LearningMobilenet };

package/lib/learning/base.ts

@@ -1,4 +1,5 @@
 import * as tf from '@tensorflow/tfjs';
+import { io } from '@tensorflow/tfjs-core';
 
 // AI Learning 디폴트 인터페이스
 interface LearningInterface {
@@ -17,15 +18,14 @@ interface LearningInterface {
     // loss 정보
     onLoss(loss: number): void;
 
-    // 이벤트 로그 정보
-    onEvents(logs: any): void;
-
     // 학습 시작 이벤트
     onTrainBegin(log: any): void;
 
     // 학습 종료 이벤트
     onTrainEnd(log: any): void;
 
+    onEpochEnd(epoch: number, logs: any): void
+
     // 학습 데이타 추가 data는 다른 클래스에 맞게 any로 설정
     addData(label: string, data: any): Promise<void>;
 
@@ -36,7 +36,7 @@ interface LearningInterface {
     infer(data: any): Promise<any>;
 
     // 모델 저장
-    saveModel(): Promise<void>;
+    saveModel(handlerOrURL: io.IOHandler | string, config?: io.SaveConfig): Promise<void>;
 
     // 학습 진행 여부
     running(): boolean;

package/lib/learning/image.ts

@@ -1,4 +1,5 @@
 import * as tf from '@tensorflow/tfjs';
+import { io } from '@tensorflow/tfjs-core';
 import LearningInterface from './base';
 
 class LearningImage implements LearningInterface {
@@ -48,6 +49,8 @@ class LearningImage implements LearningInterface {
 
   public onTrainEnd: (log: any) => void = () => {};
 
+  public onEpochEnd: (epoch: number, logs: any) => void = () => {};
+
   // 학습 데이타 등록 
   public async addData(label: string, data: any): Promise<void> {
     try {
@@ -129,7 +132,6 @@ class LearningImage implements LearningInterface {
   
   // 추론하기
   public async infer(data: any): Promise<Map<string, number>> {
-    const threshold = 1e-20; // 임계값 설정
     if (this.model === null) {
       throw new Error('Model is null');
     }
@@ -141,16 +143,14 @@ class LearningImage implements LearningInterface {
       const predictions = this.model.predict(reshapedTensor) as tf.Tensor;
       const predictionsData = await predictions.data(); // 예측 텐서의 데이터를 비동기로 가져옴
       const classProbabilities = new Map<string, number>(); // 클래스별 확률 누적값을 저장할 맵
-      console.log('predictionsData', predictionsData);
       for (let i = 0; i < predictionsData.length; i++) {
         const className = this.labels[i]; // 클래스 이름
         const probability = predictionsData[i];
-        const result = Math.abs(probability) < threshold ? 0 : probability;
         const existingProbability = classProbabilities.get(className);
         if (existingProbability !== undefined) {
-          classProbabilities.set(className, existingProbability + result);
+          classProbabilities.set(className, existingProbability + probability);
         } else {
-          classProbabilities.set(className, result);
+          classProbabilities.set(className, probability);
         }
       }
       console.log('Class Probabilities:', classProbabilities);
@@ -162,12 +162,12 @@ class LearningImage implements LearningInterface {
   
 
   // 모델 저장
-  public async saveModel(): Promise<void> {
+  public async saveModel(handlerOrURL: io.IOHandler | string, config?: io.SaveConfig): Promise<void> {
     console.log('saved model');
     if (!this.isTrainedDone) {
       return Promise.reject(new Error('Train is not done status'));
     }
-    await this.model?.save('localstorage://my-model');
+    await this.model?.save(handlerOrURL, config);
   }
 
 
@@ -232,21 +232,34 @@ class LearningImage implements LearningInterface {
     try {
       const inputShape = [this.MOBILE_NET_INPUT_WIDTH, this.MOBILE_NET_INPUT_HEIGHT, this.MOBILE_NET_INPUT_CHANNEL];
       const model = tf.sequential();
-
+      
+      model.add(tf.layers.conv2d({
+        activation: "relu",
+        filters: 32,
+        inputShape: inputShape,
+        kernelSize: 3,
+      }));
       model.add(tf.layers.conv2d({
-        inputShape,
+        activation: "relu",
         filters: 32,
         kernelSize: 3,
-        activation: 'relu'
       }));
-      model.add(tf.layers.maxPooling2d({ poolSize: 2 }));
+      model.add(tf.layers.maxPooling2d({poolSize: [2, 2]}));
+      model.add(tf.layers.conv2d({
+        activation: "relu",
+        filters: 64,
+        kernelSize: 3,
+      }));
       model.add(tf.layers.conv2d({
+        activation: "relu",
         filters: 64,
         kernelSize: 3,
-        activation: 'relu'
       }));
-      model.add(tf.layers.maxPooling2d({ poolSize: 2 }));
+      model.add(tf.layers.maxPooling2d({poolSize: [2, 2]}));
       model.add(tf.layers.flatten());
+      model.add(tf.layers.dropout({rate: 0.25}));
+      model.add(tf.layers.dense({units: 512, activation: "relu"}));
+      model.add(tf.layers.dropout({rate: 0.5}));
       model.add(tf.layers.dense({
         units: numClasses,
         activation: 'softmax'

package/lib/learning/mobilenet.test.ts

@@ -0,0 +1,44 @@
+import * as path from 'path';
+import * as tf from '@tensorflow/tfjs';
+import LearningMobilenetImage from './mobilenet';
+import * as fs from 'fs';
+const { createCanvas, loadImage } = require('canvas');
+
+let imageTensor1: tf.Tensor3D;
+let imageTensor2: tf.Tensor3D;
+
+// 이미지경로를 기준으로
+async function ImagePathToTensor(imagePath: string): Promise<tf.Tensor3D> {
+  const imageBuffer = fs.readFileSync(imagePath);
+  const image = await loadImage(imageBuffer);
+  const canvas = createCanvas(image.width, image.height);
+  const ctx = canvas.getContext('2d');
+  ctx.drawImage(image, 0, 0);
+  const imageData = ctx.getImageData(0, 0, image.width, image.height);
+  return tf.tensor3d(imageData.data, [imageData.height, imageData.width, 4], 'int32');
+}
+
+describe('LearningMobilenetImage', () => {
+  const learning = new LearningMobilenetImage({});
+  
+  beforeAll(async () => {
+    const image1Path = path.join(__dirname, '../../public/images/image1.jpeg');
+    const image2Path = path.join(__dirname, '../../public/images/image2.jpeg');
+    imageTensor1 = await ImagePathToTensor(image1Path);
+    imageTensor2 = await ImagePathToTensor(image2Path);
+  });
+
+  test('loads an image and converts it to a tensor', () => {
+    expect(imageTensor1).toBeDefined();
+    expect(imageTensor1 instanceof tf.Tensor).toBe(true);
+    expect(imageTensor2).toBeDefined();
+    expect(imageTensor2 instanceof tf.Tensor).toBe(true);
+  });
+
+  test('mobilenet add data', () => {
+    learning.addData("라벨1", imageTensor1);
+    learning.addData("라벨1", imageTensor1);
+    learning.addData("라벨2", imageTensor2);
+    learning.addData("라벨2", imageTensor2);
+  });
+});

package/lib/learning/mobilenet.ts

@@ -0,0 +1,322 @@
+///////////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
+// mobilenet 모델을 이용한 전이학습 방법 
+///////////////////////////////////////////////////////////////////////////
+
+import * as tf from '@tensorflow/tfjs';
+import { io } from '@tensorflow/tfjs-core';
+import LearningInterface from './base';
+import { ImageToTensor } from './util';
+
+class LearningMobilenet implements LearningInterface {
+  model: tf.LayersModel | null;
+  epochs: number;
+  batchSize: number;
+  learningRate: number;
+  validateRate: number;
+  labels: string[];
+  modelURL: string;
+  isRunning: boolean;
+  isReady: boolean;
+  isTrainedDone: boolean;
+  limitSize: number;
+  mobilenetModule: tf.LayersModel | null;
+  imageTensors: tf.Tensor<tf.Rank>[] = [];
+  
+  readonly MOBILE_NET_INPUT_WIDTH = 224;
+  readonly MOBILE_NET_INPUT_HEIGHT = 224;
+  readonly MOBILE_NET_INPUT_CHANNEL = 3;
+  readonly IMAGE_NORMALIZATION_FACTOR = 255.0;
+  
+  constructor({ 
+    modelURL = 'https://storage.googleapis.com/tfjs-models/tfjs/mobilenet_v1_0.25_224/model.json', // 디폴트 mobilenet 이미지 
+    epochs = 10,
+    batchSize = 16,
+    limitSize = 2,
+    learningRate = 0.001,
+    validateRate = 0.2,
+  }: { modelURL?: string, epochs?: number, batchSize?: number, limitSize?: number, learningRate?: number, validateRate?: number} = {}) {
+    this.model = null;
+    this.epochs = epochs;
+    this.batchSize = batchSize;
+    this.learningRate = learningRate;
+    this.validateRate = validateRate;
+    this.labels = [];
+    this.modelURL = modelURL;
+    this.isRunning = false;
+    this.isReady = false;
+    this.isTrainedDone = false;
+    this.limitSize = limitSize;
+    this.mobilenetModule = null;
+    this.imageTensors = [];
+    this.init();
+  }
+
+  // 진행 상태를 나타내는 이벤트를 정의합니다.
+  public onProgress: (progress: number) => void = () => {};
+
+  public onLoss: (loss: number) => void = () => {};
+
+  public onEvents: (logs: any) => void = () => {};
+
+  public onTrainBegin: (log: any) => void = () => {};
+
+  public onTrainEnd: (log: any) => void = () => {};
+
+  public onEpochEnd: (epoch: number, logs: any) => void = () => {};
+  
+  // 학습 데이타 등록 
+  public async addData(label: string, data: any): Promise<void> {
+    try {
+      
+     
+      const tensor = ImageToTensor(data)
+
+      const imgTensor = tf.tidy(() => {
+        if (this.mobilenetModule !== null) {
+          console.log('predict before', data);
+          const t = tf.browser.fromPixels(data)
+          // Resize the image to match the model's input shape
+          const resizedTensor = tf.image.resizeBilinear(t, [224, 224]);
+          const normalizedTensor = resizedTensor.div(255.0);
+
+          // Add an extra dimension to the tensor
+          const expandedTensor = normalizedTensor.expandDims(0);
+          
+          console.log('predict extend', expandedTensor);
+          const predict = this.mobilenetModule.predict(expandedTensor);
+          console.log('predict', predict);
+          return predict;
+        } else {
+          throw new Error('mobilenetModule is null');
+        }
+      });
+      console.log('imgTensor', imgTensor);
+      this.imageTensors.push(imgTensor as tf.Tensor<tf.Rank>);
+      this.labels.push(label);
+      if(this.labels.length >= this.limitSize) {
+        this.isReady = true;
+      }
+      return Promise.resolve();
+    } catch (error) {
+      console.error('Model training failed', error);
+      throw error;
+    }
+  }
+
+  async init() {
+    const model = await this.loadModel();
+    // const load_model = await tf.loadLayersModel(this.modelURL);
+    this.mobilenetModule = model;
+  } 
+
+  // 모델 학습 처리
+  public async train(): Promise<tf.History> {
+    if (this.isRunning) {
+      return Promise.reject(new Error('Training is already in progress.'));
+    }
+    
+    // 콜백 정의
+    const customCallback = {
+      onTrainBegin: (log: any) => {
+        this.isTrainedDone = false;
+        this.onTrainBegin(log);
+        console.log('Training has started.');
+      },
+      onTrainEnd: (log: any) => {
+        this.isTrainedDone = true;
+        this.onTrainEnd(log);
+        console.log('Training has ended.');
+        this.isRunning = false;
+      },
+      onBatchBegin: (batch: any, logs: any) => {
+        console.log(`Batch ${batch} is starting.`);
+      },
+      onBatchEnd: (batch: any, logs: any) => {
+        console.log(`Batch ${batch} has ended.`);
+      },
+      onEpochBegin: (epoch: number, logs: any) => {
+        console.log(`Epoch ${epoch+1} is starting.`, logs);
+      },
+      onEpochEnd: (epoch: number, logs: any) => {
+        this.onEpochEnd(epoch, logs);
+        console.log(`Epoch ${epoch+1} has ended.`);
+        console.log('Loss:', logs);
+        this.onLoss(logs.loss);
+        this.onProgress(epoch+1);
+        this.onEvents({
+          epoch: epoch,
+          logs: logs,
+        });
+      }
+    };
+
+    try {
+      this.isRunning = true;
+      if (this.labels.length < this.limitSize) {
+        return Promise.reject(new Error('Please train Data need over 2 data length'));
+      }
+      this.model = await this._createModel(this.labels.length);
+      const inputData = this._preprocessedInputData(this.model);
+
+      console.log('this.imageTensors', this.imageTensors, inputData);
+      const targetData = this._preprocessedTargetData();
+      const history = await this.model.fit(inputData, targetData, {
+        epochs: this.epochs,
+        batchSize: this.batchSize,
+        validationSplit: this.validateRate, // 검증 데이터의 비율 설정
+        callbacks: customCallback
+      });
+      console.log('Model training completed', history);
+      return history;
+    } catch (error) {
+      this.isRunning = false;
+      console.error('Model training failed', error);
+      throw error;
+    }
+  }
+  
+  // 추론하기
+  public async infer(data: any): Promise<Map<string, number>> {
+    if (this.model === null) {
+      return Promise.reject(new Error('Model is Null'));
+    }
+  
+    try {
+      const imgTensor = tf.tidy(() => {
+        if (this.mobilenetModule !== null) {
+          console.log('predict before', data);
+          const t = tf.browser.fromPixels(data)
+          // Resize the image to match the model's input shape
+          const resizedTensor = tf.image.resizeBilinear(t, [224, 224]);
+          const normalizedTensor = resizedTensor.div(255.0);
+
+          // Add an extra dimension to the tensor
+          const expandedTensor = normalizedTensor.expandDims(0);
+          
+          console.log('predict extend', expandedTensor);
+          const predict = this.mobilenetModule.predict(expandedTensor);
+          console.log('predict', predict);
+          return predict;
+        } else {
+          throw new Error('mobilenetModule is null');
+        }
+      });
+      const predictions = this.model.predict(imgTensor) as tf.Tensor;
+      const predictedClass = predictions.as1D().argMax();
+      const classId = (await predictedClass.data())[0];
+      console.log('classId', classId, this.labels[classId]);
+      const predictionsData = await predictions.data(); // 예측 텐서의 데이터를 비동기로 가져옴
+      const classProbabilities = new Map<string, number>(); // 클래스별 확률 누적값을 저장할 맵
+
+      const EPSILON = 1e-6; // 매우 작은 값을 표현하기 위한 엡실론
+      for (let i = 0; i < predictionsData.length; i++) {
+        let probability = Math.max(0, Math.min(1, predictionsData[i])); // 확률 값을 0과 1 사이로 조정
+        probability = probability < EPSILON ? 0 : probability; // 매우 작은 확률 값을 0으로 간주
+        const className = this.labels[i]; // 클래스 이름
+        const existingProbability = classProbabilities.get(className);
+        if (existingProbability !== undefined) {
+          classProbabilities.set(className, existingProbability + probability);
+        } else {
+          classProbabilities.set(className, probability);
+        }
+      }
+      console.log('Class Probabilities:', classProbabilities);
+      predictedClass.dispose();
+      predictions.dispose();
+      return classProbabilities;
+    } catch (error) {
+      throw error;
+    }
+  }
+
+  
+  // 모델 저장
+  public async saveModel(handlerOrURL: io.IOHandler | string, config?: io.SaveConfig): Promise<void> {
+    console.log('saved model');
+    if (!this.isTrainedDone) {
+      return Promise.reject(new Error('Train is not done status'));
+    }
+    await this.model?.save(handlerOrURL, config);
+  }
+
+
+
+  // 진행중 여부
+  public running(): boolean {
+    return this.isRunning;
+  }
+
+
+  public ready(): boolean {
+    return this.isReady;
+  }
+
+
+  // target 라벨 데이타
+  private _preprocessedTargetData(): tf.Tensor<tf.Rank> {
+    // 라벨 unique 처리 & 배열 리턴
+    console.log('uniqueLabels.length', this.labels, this.labels.length);
+    const labelIndices = this.labels.map((label) => this.labels.indexOf(label));
+    console.log('labelIndices', labelIndices);
+    const oneHotEncode = tf.oneHot(tf.tensor1d(labelIndices, 'int32'),this.labels.length);
+    console.log('oneHotEncode', oneHotEncode);
+    return oneHotEncode
+  }
+
+  // 입력 이미지 데이타 
+  private _preprocessedInputData(model: tf.LayersModel): tf.Tensor<tf.Rank> {
+    // 이미지 배열을 배치로 변환 - [null, 224, 224, 3]
+    const inputShape = model.inputs[0].shape;
+    console.log('inputShape', inputShape);
+    // inputShape를 이와 같이 포멧 맞춘다. for reshape to [224, 224, 3]
+    const inputShapeArray = inputShape.slice(1) as number[];
+    console.log('inputShapeArray', inputShapeArray);
+    const inputBatch = tf.stack(this.imageTensors.map((image) => {
+      return tf.reshape(image, inputShapeArray);
+    }));
+    return inputBatch
+  }
+
+
+  private async loadModel(): Promise<tf.LayersModel> {
+    const load_model = await tf.loadLayersModel(this.modelURL);
+    load_model.summary();
+    const layer = load_model.getLayer('conv_pw_13_relu');
+    const truncatedModel = tf.model({
+      inputs: load_model.inputs, 
+      outputs: layer.output
+    })
+    return truncatedModel
+  }
+
+  // 모델 저장
+  private async _createModel(numClasses: number): Promise<tf.Sequential> {
+    try {
+        const truncatedModel = await this.loadModel()
+        console.log('truncatedModel', truncatedModel, truncatedModel.outputs[0].shape.slice(1));
+        // 입력 이미지 크기에 맞게 모델 구조 수정
+        const model = tf.sequential();
+        model.add(tf.layers.flatten({
+          inputShape: truncatedModel.outputs[0].shape.slice(1)
+        }));
+        model.add(tf.layers.dense({ units: 100, activation: 'relu' }));
+        model.add(tf.layers.dense({ units: this.labels.length, activation: 'softmax' }));
+
+        const optimizer = tf.train.adam(this.learningRate); // Optimizer를 생성하고 학습률을 설정합니다.
+        model.compile({
+            loss: (numClasses === 2) ? 'binaryCrossentropy': 'categoricalCrossentropy',
+            optimizer: optimizer,
+            metrics: ['accuracy', 'acc']
+        });
+        model.summary();
+        return model;
+    } catch (error) {
+        console.error('Failed to load model', error);
+        throw error;
+    }
+  }
+}
+
+export default LearningMobilenet;