image-js 1.0.0-alpha.2 → 1.0.0

package/src/featureMatching/descriptors/getBrief.ts

@@ -0,0 +1,42 @@
+import type { Image } from '../../Image.js';
+import { getBestKeypointsInRadius } from '../keypoints/getBestKeypointsInRadius.js';
+import { getOrientedFastKeypoints } from '../keypoints/getOrientedFastKeypoints.js';
+
+import type { Brief } from './getBriefDescriptors.js';
+import { getBriefDescriptors } from './getBriefDescriptors.js';
+
+export interface GetBriefOptions {
+  centroidPatchDiameter?: number;
+  bestKptRadius?: number;
+  minScore?: number;
+}
+
+/**
+ * Get the keypoints and corresponding descriptors for an image.
+ * @param image - Image to process.
+ * @param options - Get brief options.
+ * @returns The Brief (keypoints and corresponding descriptors).
+ */
+export function getBrief(image: Image, options: GetBriefOptions = {}): Brief {
+  const { centroidPatchDiameter = 15, bestKptRadius = 10, minScore } = options;
+  const allSourceKeypoints = getOrientedFastKeypoints(image, {
+    centroidPatchDiameter,
+  });
+  const sourceKeypoints = getBestKeypointsInRadius(
+    allSourceKeypoints,
+    bestKptRadius,
+  );
+
+  const brief = getBriefDescriptors(image, sourceKeypoints);
+  if (minScore) {
+    for (let i = 0; i < brief.keypoints.length; i++) {
+      if (brief.keypoints[i].score < minScore) {
+        return {
+          keypoints: brief.keypoints.slice(0, i - 1),
+          descriptors: brief.descriptors.slice(0, i - 1),
+        };
+      }
+    }
+  }
+  return brief;
+}

package/src/featureMatching/descriptors/getBriefDescriptors.ts

@@ -0,0 +1,108 @@
+import type { Image } from '../../Image.js';
+import type { GaussianBlurSigmaOptions } from '../../filters/index.js';
+import type { GetGaussianPointsOptions } from '../../utils/utils.types.js';
+import checkProcessable from '../../utils/validators/checkProcessable.js';
+import type { OrientedFastKeypoint } from '../keypoints/getOrientedFastKeypoints.js';
+import { compareIntensity } from '../utils/compareIntensity.js';
+import { getGaussianPoints } from '../utils/getGaussianPoints.js';
+
+import { getKeypointPatch } from './utils/getKeypointPatch.js';
+
+export interface GetBriefDescriptorsOptions {
+  /**
+   * Options to smooth the image patch before comparing pairs of points.
+   * Default values are the ones recommended in the original BRIEF article.
+   * DOI: https://doi.org/10.1007/978-3-642-15561-1_56.
+   */
+  smoothingOptions?: GaussianBlurSigmaOptions;
+  /**
+   * Options to modify the gaussian distribution used to generate the points to compare.
+   */
+  pointsDistributionOptions?: Omit<GetGaussianPointsOptions, 'nbPoints'>;
+  /**
+   * Size of the patch around the keypoint used to compute the descriptor.
+   * @default `31`
+   */
+  patchSize?: number;
+  /**
+   * Number of bits of the final descriptor. Typically a power or 2: 128, 256, 512.
+   * @default `256`
+   */
+  descriptorLength?: number;
+}
+
+export type BriefDescriptor = Uint8Array;
+export interface Brief {
+  keypoints: OrientedFastKeypoint[];
+  descriptors: BriefDescriptor[];
+}
+
+/**
+ * Generate the rBRIEF descriptors for the desired keypoints of an image.
+ * The rBRIEF descriptors are presented in these articles:
+ * - ORB article: DOI: 10.1109/ICCV.2011.6126544
+ * - rBRIEF article: DOI: 10.1007/978-3-642-15561-1_56.
+ * @param image - Source image of the keypoints.
+ * @param keypoints - Keypoints for which the descriptors are wanted.
+ * @param options - Get rotated BRIEF descriptors options.
+ * @returns The descriptors for the given keypoints.
+ */
+export function getBriefDescriptors(
+  image: Image,
+  keypoints: OrientedFastKeypoint[],
+  options: GetBriefDescriptorsOptions = {},
+): Brief {
+  const {
+    patchSize = 31,
+    descriptorLength = 256,
+    smoothingOptions = {
+      sigma: Math.sqrt(2),
+      size: Math.min(image.height, image.width, 9),
+    },
+    pointsDistributionOptions,
+  } = options;
+
+  checkProcessable(image, {
+    alpha: false,
+    colorModel: 'GREY',
+  });
+
+  if (!(patchSize % 2)) {
+    throw new TypeError('patchSize must be an odd integer');
+  }
+
+  if (Math.min(image.width, image.height) < patchSize) {
+    throw new RangeError(`image is too small for patchSize = ${patchSize}`);
+  }
+
+  const gaussianPoints = getGaussianPoints(patchSize, patchSize, {
+    ...pointsDistributionOptions,
+    nbPoints: descriptorLength * 2,
+  });
+
+  const smoothed = image.gaussianBlur(smoothingOptions);
+
+  const descriptors: Uint8Array[] = [];
+  const filteredKeypoints: OrientedFastKeypoint[] = [];
+
+  for (const keypoint of keypoints) {
+    let patch: Image;
+    try {
+      patch = getKeypointPatch(smoothed, keypoint, { patchSize });
+    } catch {
+      continue;
+    }
+    if (patch === null) continue;
+
+    const descriptor = new Uint8Array(descriptorLength);
+    for (let i = 0; i < descriptorLength; i++) {
+      const p1 = gaussianPoints[i];
+      const p2 = gaussianPoints[i + descriptorLength];
+      descriptor[i] = Number(compareIntensity(patch, p1, p2));
+    }
+    descriptors.push(descriptor);
+    filteredKeypoints.push(keypoint);
+  }
+
+  return { keypoints: filteredKeypoints, descriptors };
+}

package/src/featureMatching/descriptors/utils/getKeypointPatch.ts

@@ -0,0 +1,59 @@
+import type { Image } from '../../../Image.js';
+import { getRadius } from '../../../utils/getRadius.js';
+import type { OrientedFastKeypoint } from '../../keypoints/getOrientedFastKeypoints.js';
+import { checkBorderDistance } from '../../utils/checkBorderDistance.js';
+import { extractSquareImage } from '../../utils/extractSquareImage.js';
+
+export interface GetKeypointPatchOptions {
+  /**
+   * Size of the patch around the keypoint used to compute the descriptor.
+   * @default `31`
+   */
+  patchSize?: number;
+}
+
+/**
+ * Get the patch around the keypoint with given size.
+ * @param image - Image from which to extract the patch.
+ * @param keypoint - Keypoint around which to extract the patch.
+ * @param options - Get keypoint patch options.
+ * @returns The patch around the keypoint.
+ */
+export function getKeypointPatch(
+  image: Image,
+  keypoint: OrientedFastKeypoint,
+  options: GetKeypointPatchOptions = {},
+): Image {
+  const { patchSize = 31 } = options;
+  // crop smallest square surrounding the tilted patch of the keypoint
+  // we have to handle the fact that this square can have even dimensions
+
+  const radAngle = (keypoint.angle * Math.PI) / 180;
+
+  const rawWidth = Math.floor(
+    patchSize * (Math.abs(Math.cos(radAngle)) + Math.abs(Math.sin(radAngle))),
+  );
+
+  const cropWidth = rawWidth % 2 ? rawWidth : rawWidth + 1;
+
+  // we are not allowing keypoints that are too close to the border of the image
+  const borderDistance = getRadius(cropWidth);
+
+  if (!checkBorderDistance(image, keypoint.origin, borderDistance)) {
+    throw new RangeError(
+      'keypoint is too close to border for given patch size',
+    );
+  }
+
+  const cropped = extractSquareImage(image, keypoint.origin, cropWidth);
+
+  const rotated = cropped.transformRotate(-keypoint.angle, {
+    center: 'center',
+    interpolationType: 'nearest',
+  });
+
+  const cropOrigin = rotated.getCoordinates('center');
+  const result = extractSquareImage(rotated, cropOrigin, patchSize);
+
+  return result;
+}

package/src/featureMatching/descriptors/utils/sliceBrief.ts

@@ -0,0 +1,34 @@
+import type { Brief } from '../getBriefDescriptors.js';
+
+export interface SliceBriefOptions {
+  /**
+   * Start index.
+   * @default `0`
+   */
+  start?: number;
+  /**
+   * End index.
+   * @default `brief.keypoints.length`
+   */
+  end?: number;
+}
+
+/**
+ * Slice a Brief to keep only the desired keypoints and the corresponding descriptors.
+ * @param brief - Brief to process.
+ * @param options - Slice Brief options.
+ * @returns The desired keypoints + descriptors.
+ */
+export function sliceBrief(
+  brief: Brief,
+  options: SliceBriefOptions = {},
+): Brief {
+  const { start = 0, end = brief.keypoints.length } = options;
+  if (start < 0 || end > brief.keypoints.length) {
+    throw new RangeError('start or end are out of range');
+  }
+  return {
+    keypoints: brief.keypoints.slice(start, end),
+    descriptors: brief.descriptors.slice(start, end),
+  };
+}

package/src/featureMatching/featureMatching.types.ts

@@ -0,0 +1,26 @@
+export interface GetColorsOptions {
+  /**
+   * Number of shades to generate.
+   * @default `6`
+   */
+  nbShades?: number;
+  /**
+   * Factor between 0 and 1 by which to multiply the maximal value of the color to obtain the minimum value.
+   * @default `0.2`
+   */
+  minValueFactor?: number;
+}
+
+export interface GetHarrisScoreOptions {
+  /**
+   * Size of the window to compute the Harris score.
+   * Should be an odd number so that the window can be centered on the corner.
+   * @default `7`
+   */
+  windowSize?: number;
+  /**
+   * Constant for the score computation. Should be between 0.04 and 0.06 (empirical values). This consant is commonly called k.
+   * @default `0.04`
+   */
+  harrisConstant?: number;
+}

package/src/featureMatching/index.ts

@@ -0,0 +1,12 @@
+export * from './descriptors/getBriefDescriptors.js';
+export * from './featureMatching.types.js';
+export * from './keypoints/getFastKeypoints.js';
+export * from './keypoints/getIntensityCentroid.js';
+export * from './keypoints/getOrientedFastKeypoints.js';
+export * from './keypoints/getBestKeypointsInRadius.js';
+export * from './matching/bruteForceMatch.js';
+export * from './matching/getCrosscheckMatches.js';
+export * from './visualize/drawKeypoints.js';
+export * from './visualize/drawMatches.js';
+export * from './visualize/Montage.js';
+export * from './visualize/overlapImages.js';

package/src/featureMatching/keypoints/getBestKeypointsInRadius.ts

@@ -0,0 +1,39 @@
+import type { OrientedFastKeypoint } from './getOrientedFastKeypoints.js';
+import { getKeypointsInRadius } from './utils/getKeypointsInRadius.js';
+
+/**
+ * Return the best keypoints within the given radius in pixels.
+ * @param keypoints - Keypoints to process.
+ * @param radius - Minimum distance in pixels between two keypoints.
+ * @returns The filtered keypoints.
+ */
+export function getBestKeypointsInRadius(
+  keypoints: OrientedFastKeypoint[],
+  radius = 5,
+): OrientedFastKeypoint[] {
+  const size = keypoints.length;
+
+  const keypointsInRadius = getKeypointsInRadius(keypoints, radius);
+
+  const toIgnore = new Uint8Array(size).fill(0);
+
+  for (let i = 0; i < size; i++) {
+    const keypoint = keypoints[i];
+    for (const secondKeypointIndex of keypointsInRadius[i]) {
+      const secondKeypoint = keypoints[secondKeypointIndex];
+      if (keypoint.score < secondKeypoint.score) {
+        toIgnore[i] = 1;
+        continue;
+      }
+    }
+  }
+
+  const result: OrientedFastKeypoint[] = [];
+  for (let i = 0; i < size; i++) {
+    if (!toIgnore[i]) {
+      result.push(keypoints[i]);
+    }
+  }
+
+  return result;
+}

package/src/featureMatching/keypoints/getFastKeypoints.ts

@@ -0,0 +1,157 @@
+import { match } from 'ts-pattern';
+
+import type { Image } from '../../Image.js';
+import type { Point } from '../../geometry/index.js';
+import {
+  getCirclePoints,
+  getCompassPoints,
+} from '../../utils/geometry/getCirclePoints.js';
+import { getIndex } from '../../utils/getIndex.js';
+import { surroundingPixels } from '../../utils/surroundingPixels.js';
+import checkProcessable from '../../utils/validators/checkProcessable.js';
+import type { GetHarrisScoreOptions } from '../featureMatching.types.js';
+
+import { getFastScore } from './getFastScore.js';
+import { getHarrisScore } from './getHarrisScore.js';
+import type { IsFastKeypointOptions } from './isFastKeypoint.js';
+import { isFastKeypoint } from './isFastKeypoint.js';
+
+export interface GetFastKeypointsOptions extends IsFastKeypointOptions {
+  /**
+   * Maximum number of features to return.
+   * @default `500`
+   */
+  maxNbFeatures?: number;
+  /**
+   * Whether to apply non-max suppression to the keypoints.
+   * This removes all keypoints which
+   * don't have the highest value within the adjacent keypoints.
+   * @default `true`
+   */
+  nonMaxSuppression?: boolean;
+  /**
+   * Radius of the circle used for the algorithm.
+   * @default `3`
+   */
+  fastRadius?: number;
+  /**
+   * Algorithm to use to compute corners score.
+   * @default `'FAST'`
+   */
+  scoreAlgorithm?: 'HARRIS' | 'FAST';
+  /**
+   * Options for the Harris score computation.
+   */
+  harrisScoreOptions?: GetHarrisScoreOptions;
+}
+
+export interface FastKeypoint {
+  /**
+   * Location of the keypoint in the image.
+   */
+  origin: Point;
+  /**
+   * Score of the keypoint, the bigger it is, the better the feature.
+   * It is the criteria used for the non-maximal suppression.
+   */
+  score: number;
+}
+
+/**
+ * Find the features in a GREY image according to the FAST (Features from Accelerated Segment Test) algorithm.
+ * Based on the paper Machine Learning for High-Speed Corner Detection.
+ * DOI: https://doi.org/10.1007/11744023_34.
+ * @param image - The image to process.
+ * @param options - Get FAST keypoints options.
+ * @returns The FAST keypoints.
+ */
+export function getFastKeypoints(
+  image: Image,
+  options: GetFastKeypointsOptions = {},
+): FastKeypoint[] {
+  const {
+    fastRadius = 3,
+    scoreAlgorithm = 'FAST',
+    harrisScoreOptions,
+  } = options;
+
+  const circlePoints = getCirclePoints(fastRadius);
+  const compassPoints = getCompassPoints(fastRadius);
+
+  const {
+    maxNbFeatures = 500,
+    nbContiguousPixels = (3 / 4) * circlePoints.length,
+    threshold = 20,
+    nonMaxSuppression = true,
+  } = options;
+
+  checkProcessable(image, {
+    channels: [1],
+    alpha: false,
+  });
+
+  const getScore = match(scoreAlgorithm)
+    .with('HARRIS', () => {
+      return (image: Image, corner: Point) =>
+        getHarrisScore(image, corner, harrisScoreOptions);
+    })
+    .with('FAST', () => {
+      return (image: Image, corner: Point) =>
+        getFastScore(image, corner, threshold, circlePoints);
+    })
+    .exhaustive();
+
+  const allKeypoints: FastKeypoint[] = [];
+
+  const scoreArray = new Float64Array(image.size).fill(
+    Number.NEGATIVE_INFINITY,
+  );
+  for (let row = 0; row < image.height; row++) {
+    for (let column = 0; column < image.width; column++) {
+      const corner = { row, column };
+      if (
+        isFastKeypoint(corner, image, circlePoints, compassPoints, {
+          nbContiguousPixels,
+          threshold,
+        })
+      ) {
+        const score = getScore(image, corner);
+        scoreArray[getIndex(corner.column, corner.row, image, 0)] = score;
+        allKeypoints.push({ origin: corner, score });
+      }
+    }
+  }
+
+  let keypoints: FastKeypoint[] = [];
+  if (!nonMaxSuppression) {
+    keypoints = allKeypoints;
+  } else {
+    // Non-Maximal Suppression
+    for (const keypoint of allKeypoints) {
+      const currentScore =
+        scoreArray[
+          getIndex(keypoint.origin.column, keypoint.origin.row, image, 0)
+        ];
+      for (let i = 0; i < surroundingPixels.length; i++) {
+        const neighbour = surroundingPixels[i];
+        const neighbourScore =
+          scoreArray[
+            getIndex(
+              keypoint.origin.column + neighbour.column,
+              keypoint.origin.row + neighbour.row,
+              image,
+              0,
+            )
+          ];
+        if (neighbourScore > currentScore) break;
+        if (i === surroundingPixels.length - 1) {
+          keypoints.push(keypoint);
+        }
+      }
+    }
+  }
+
+  keypoints.sort((a, b) => b.score - a.score);
+
+  return keypoints.slice(0, maxNbFeatures);
+}

package/src/featureMatching/keypoints/getFastScore.ts

@@ -0,0 +1,37 @@
+import type { Image } from '../../Image.js';
+import type { Point } from '../../geometry/index.js';
+
+/**
+ * Compute the score of a keypoint using the function described in the FAST article.
+ * DOI: https://doi.org/10.1007/11744023_34.
+ * @param image - Image to process.
+ * @param origin - Keypoint coordinates.
+ * @param threshold - FAST threshold.
+ * @param circlePoints - Coordinates of the points on the circle.
+ * @returns Score of the corner.
+ */
+export function getFastScore(
+  image: Image,
+  origin: Point,
+  threshold: number,
+  circlePoints: Point[],
+): number {
+  const currentIntensity = image.getValueByPoint(origin, 0);
+
+  let brighterSum = 0;
+  let darkerSum = 0;
+  for (const point of circlePoints) {
+    const pointIntensity = image.getValue(
+      origin.column + point.column,
+      origin.row + point.row,
+      0,
+    );
+    if (pointIntensity >= currentIntensity + threshold) {
+      brighterSum += Math.abs(pointIntensity - currentIntensity) - threshold; // circle point is lighter
+    } else if (pointIntensity <= currentIntensity - threshold) {
+      darkerSum += Math.abs(currentIntensity - pointIntensity) - threshold; // circle point is darker
+    }
+  }
+
+  return Math.max(brighterSum, darkerSum);
+}

package/src/featureMatching/keypoints/getHarrisScore.ts

@@ -0,0 +1,73 @@
+import { EigenvalueDecomposition, Matrix, WrapperMatrix1D } from 'ml-matrix';
+
+import type { Image } from '../../Image.js';
+import type { Point } from '../../geometry/index.js';
+import { SOBEL_X, SOBEL_Y } from '../../utils/constants/kernels.js';
+import type { GetHarrisScoreOptions } from '../featureMatching.types.js';
+
+/**
+ * Get the Harris score of a corner. The idea behind the algorithm is that a
+ * slight shift of a window around a corner along x and y shoud result in
+ * a very different image.
+ *
+ * We distinguish 3 cases:
+ * - the score is highly negative: you have an edge
+ * - the abolute value of the score is small: the region is flat
+ * - the score is highly positive: you have a corner.
+ * @see {@link https://en.wikipedia.org/wiki/Harris_corner_detector}
+ * @param image - Image to which the corner belongs. It must be a greyscale image with only one channel.
+ * @param origin - Center of the window, where the corner should be.
+ * @param options - Get Harris score options.
+ * @returns The Harris score.
+ */
+export function getHarrisScore(
+  image: Image,
+  origin: Point,
+  options: GetHarrisScoreOptions = {},
+): number {
+  const { windowSize = 7, harrisConstant = 0.04 } = options;
+
+  if (!(windowSize % 2)) {
+    throw new TypeError('windowSize must be an odd integer');
+  }
+
+  const cropOrigin = {
+    row: origin.row - (windowSize - 1) / 2,
+    column: origin.column - (windowSize - 1) / 2,
+  };
+  const window = image.crop({
+    origin: cropOrigin,
+    width: windowSize,
+    height: windowSize,
+  });
+  const xDerivative = window.gradientFilter({ kernelX: SOBEL_X });
+  const yDerivative = window.gradientFilter({ kernelY: SOBEL_Y });
+
+  const xMatrix = new WrapperMatrix1D(xDerivative.getRawImage().data, {
+    rows: xDerivative.height,
+  });
+  const yMatrix = new WrapperMatrix1D(yDerivative.getRawImage().data, {
+    rows: yDerivative.height,
+  });
+
+  const xx = xMatrix.mmul(xMatrix);
+  const xy = yMatrix.mmul(xMatrix);
+  const yy = yMatrix.mmul(yMatrix);
+
+  const xxSum = xx.sum();
+  const xySum = xy.sum();
+  const yySum = yy.sum();
+
+  const structureTensor = new Matrix([
+    [xxSum, xySum],
+    [xySum, yySum],
+  ]);
+
+  const eigenValues = new EigenvalueDecomposition(structureTensor)
+    .realEigenvalues;
+
+  return (
+    eigenValues[0] * eigenValues[1] -
+    harrisConstant * (eigenValues[0] + eigenValues[1]) ** 2
+  );
+}

package/src/featureMatching/keypoints/getIntensityCentroid.ts

@@ -0,0 +1,34 @@
+import type { Image } from '../../Image.js';
+import type { Point } from '../../geometry/index.js';
+
+import { getIntensityMoment } from './getIntensityMoment.js';
+
+/**
+ * Compute the intensity centroid of an image for each channel relatively to the center of the image.
+ * Original article: {@link https://doi.org/10.1006/cviu.1998.0719}.
+ * @see {@link https://en.wikipedia.org/wiki/Image_moment}
+ * @param image - Image to process.
+ * @returns The intensity centroid of each channel of the image.
+ */
+export function getIntensityCentroid(image: Image): Point[] {
+  const moment10 = getIntensityMoment(image, 1, 0);
+  const moment01 = getIntensityMoment(image, 0, 1);
+  const moment00 = getIntensityMoment(image, 0, 0);
+  const centroid: Point[] = [];
+
+  for (let channel = 0; channel < image.channels; channel++) {
+    if (moment00[channel] === 0) {
+      centroid.push({
+        column: 0,
+        row: 0,
+      });
+    } else {
+      centroid.push({
+        column: moment10[channel] / moment00[channel],
+        row: moment01[channel] / moment00[channel],
+      });
+    }
+  }
+
+  return centroid;
+}

package/src/featureMatching/keypoints/getIntensityMoment.ts

@@ -0,0 +1,43 @@
+import type { Image } from '../../Image.js';
+import type { Point } from '../../geometry/index.js';
+
+export interface GetIntensityMomentOptions {
+  /**
+   * Origin for the moment computation.
+   * @default `image.getCoordinates('center')`.
+   */
+  origin?: Point;
+}
+
+/**
+ * Compute the pq order intensity moment of the image
+ * relatively to the origin defined in the options.
+ * Original article: {@link https://doi.org/10.1006/cviu.1998.0719}.
+ * @see {@link https://en.wikipedia.org/wiki/Image_moment}
+ * @param image - Image to process. Should have an odd number of rows and columns.
+ * @param p - Order along x.
+ * @param q - Order along y.
+ * @param options - Get intensity moment options.
+ * @returns The intensity moment of order pq.
+ */
+export function getIntensityMoment(
+  image: Image,
+  p: number,
+  q: number,
+  options: GetIntensityMomentOptions = {},
+): number[] {
+  const { origin = image.getCoordinates('center') } = options;
+
+  const moment = new Array<number>(image.channels).fill(0);
+  for (let row = 0; row < image.height; row++) {
+    for (let column = 0; column < image.width; column++) {
+      const xDistance = column - origin.column;
+      const yDistance = row - origin.row;
+      for (let channel = 0; channel < image.channels; channel++) {
+        const intensity = image.getValue(column, row, channel);
+        moment[channel] += xDistance ** p * yDistance ** q * intensity;
+      }
+    }
+  }
+  return moment;
+}