@searpent/react-image-annotate 2.0.0

package/src/ImageCanvas/index.js

@@ -0,0 +1,488 @@
+// @flow weak
+
+import React, {
+  useRef,
+  useState,
+  useLayoutEffect,
+  useEffect,
+  useMemo,
+} from "react"
+import type { Node } from "react"
+import { Matrix } from "transformation-matrix-js"
+import Crosshairs from "../Crosshairs"
+import type {
+  Region,
+  Point,
+  Polygon,
+  Box,
+  Keypoints,
+  KeypointsDefinition,
+} from "./region-tools.js"
+import { makeStyles } from "@mui/styles"
+import { createTheme, ThemeProvider } from "@mui/material/styles"
+import styles from "./styles"
+import PreventScrollToParents from "../PreventScrollToParents"
+import useWindowSize from "../hooks/use-window-size.js"
+import useMouse from "./use-mouse"
+import useProjectRegionBox from "./use-project-box"
+import useExcludePattern from "../hooks/use-exclude-pattern"
+import { useRafState } from "react-use"
+import PointDistances from "../PointDistances"
+import RegionTags from "../RegionTags"
+import RegionLabel from "../RegionLabel"
+import ImageMask from "../ImageMask"
+import RegionSelectAndTransformBoxes from "../RegionSelectAndTransformBoxes"
+import VideoOrImageCanvasBackground from "../VideoOrImageCanvasBackground"
+import useEventCallback from "use-event-callback"
+import RegionShapes from "../RegionShapes"
+import useWasdMode from "./use-wasd-mode"
+
+const theme = createTheme()
+const useStyles = makeStyles((theme) => styles)
+
+type Props = {
+  regions: Array<Region>,
+  imageSrc?: string,
+  videoSrc?: string,
+  videoTime?: number,
+  keypointDefinitions?: KeypointDefinitions,
+  onMouseMove?: ({ x: number, y: number }) => any,
+  onMouseDown?: ({ x: number, y: number }) => any,
+  onMouseUp?: ({ x: number, y: number }) => any,
+  dragWithPrimary?: boolean,
+  zoomWithPrimary?: boolean,
+  createWithPrimary?: boolean,
+  showTags?: boolean,
+  realSize?: { width: number, height: number, unitName: string },
+  showCrosshairs?: boolean,
+  showMask?: boolean,
+  showHighlightBox?: boolean,
+  showPointDistances?: boolean,
+  pointDistancePrecision?: number,
+  regionClsList?: Array<string>,
+  regionTagList?: Array<string>,
+  allowedArea?: { x: number, y: number, w: number, h: number },
+  RegionEditLabel?: Node,
+  videoPlaying?: boolean,
+  zoomOnAllowedArea?: boolean,
+  fullImageSegmentationMode?: boolean,
+  autoSegmentationOptions?: Object,
+  modifyingAllowedArea?: boolean,
+  allowComments?: Boolean,
+  onChangeRegion: (Region) => any,
+  onBeginRegionEdit: (Region) => any,
+  onCloseRegionEdit: (Region) => any,
+  onDeleteRegion: (Region) => any,
+  onBeginBoxTransform: (Box, [number, number]) => any,
+  onBeginMovePolygonPoint: (Polygon, index: number) => any,
+  onBeginMoveKeypoint: (Keypoints, index: number) => any,
+  onAddPolygonPoint: (Polygon, point: [number, number], index: number) => any,
+  onSelectRegion: (Region) => any,
+  onBeginMovePoint: (Point) => any,
+  onImageOrVideoLoaded: ({
+    naturalWidth: number,
+    naturalHeight: number,
+    duration?: number,
+  }) => any,
+  onChangeVideoTime: (number) => any,
+  onRegionClassAdded: () => {},
+  onChangeVideoPlaying?: Function,
+}
+
+const getDefaultMat = (allowedArea = null, { iw, ih } = {}) => {
+  let mat = Matrix.from(1, 0, 0, 1, -10, -10)
+  if (allowedArea && iw) {
+    mat = mat
+      .translate(allowedArea.x * iw, allowedArea.y * ih)
+      .scaleU(allowedArea.w + 0.05)
+  }
+  return mat
+}
+
+export const ImageCanvas = ({
+  regions,
+  imageSrc,
+  videoSrc,
+  videoTime,
+  realSize,
+  showTags,
+  onMouseMove = (p) => null,
+  onMouseDown = (p) => null,
+  onMouseUp = (p) => null,
+  dragWithPrimary = false,
+  zoomWithPrimary = false,
+  createWithPrimary = false,
+  pointDistancePrecision = 0,
+  regionClsList,
+  regionTagList,
+  showCrosshairs,
+  showHighlightBox = true,
+  showPointDistances,
+  allowedArea,
+  RegionEditLabel = null,
+  videoPlaying = false,
+  showMask = true,
+  fullImageSegmentationMode,
+  autoSegmentationOptions,
+  onImageOrVideoLoaded,
+  onChangeRegion,
+  onBeginRegionEdit,
+  onCloseRegionEdit,
+  onBeginBoxTransform,
+  onBeginMovePolygonPoint,
+  onAddPolygonPoint,
+  onBeginMoveKeypoint,
+  onSelectRegion,
+  onBeginMovePoint,
+  onDeleteRegion,
+  onChangeVideoTime,
+  onChangeVideoPlaying,
+  onRegionClassAdded,
+  zoomOnAllowedArea = true,
+  modifyingAllowedArea = false,
+  keypointDefinitions,
+  allowComments,
+}: Props) => {
+  const classes = useStyles()
+
+  const canvasEl = useRef(null)
+  const layoutParams = useRef({})
+  const [dragging, changeDragging] = useRafState(false)
+  const [maskImagesLoaded, changeMaskImagesLoaded] = useRafState(0)
+  const [zoomStart, changeZoomStart] = useRafState(null)
+  const [zoomEnd, changeZoomEnd] = useRafState(null)
+  const [mat, changeMat] = useRafState(getDefaultMat())
+  const maskImages = useRef({})
+  const windowSize = useWindowSize()
+
+  const getLatestMat = useEventCallback(() => mat)
+  useWasdMode({ getLatestMat, changeMat })
+
+  const { mouseEvents, mousePosition } = useMouse({
+    canvasEl,
+    dragging,
+    mat,
+    layoutParams,
+    changeMat,
+    zoomStart,
+    zoomEnd,
+    changeZoomStart,
+    changeZoomEnd,
+    changeDragging,
+    zoomWithPrimary,
+    dragWithPrimary,
+    onMouseMove,
+    onMouseDown,
+    onMouseUp,
+  })
+
+  useLayoutEffect(() => changeMat(mat.clone()), [windowSize])
+
+  const innerMousePos = mat.applyToPoint(
+    mousePosition.current.x,
+    mousePosition.current.y
+  )
+
+  const projectRegionBox = useProjectRegionBox({ layoutParams, mat })
+
+  const [imageDimensions, changeImageDimensions] = useState()
+  const imageLoaded = Boolean(imageDimensions && imageDimensions.naturalWidth)
+
+  const onVideoOrImageLoaded = useEventCallback(
+    ({ naturalWidth, naturalHeight, duration }) => {
+      const dims = { naturalWidth, naturalHeight, duration }
+      if (onImageOrVideoLoaded) onImageOrVideoLoaded(dims)
+      changeImageDimensions(dims)
+      // Redundant update to fix rerendering issues
+      setTimeout(() => changeImageDimensions(dims), 10)
+    }
+  )
+
+  const excludePattern = useExcludePattern()
+
+  const canvas = canvasEl.current
+  if (canvas && imageLoaded) {
+    const { clientWidth, clientHeight } = canvas
+
+    const fitScale = Math.max(
+      imageDimensions.naturalWidth / (clientWidth - 20),
+      imageDimensions.naturalHeight / (clientHeight - 20)
+    )
+
+    const [iw, ih] = [
+      imageDimensions.naturalWidth / fitScale,
+      imageDimensions.naturalHeight / fitScale,
+    ]
+
+    layoutParams.current = {
+      iw,
+      ih,
+      fitScale,
+      canvasWidth: clientWidth,
+      canvasHeight: clientHeight,
+    }
+  }
+
+  useEffect(() => {
+    if (!imageLoaded) return
+    changeMat(
+      getDefaultMat(
+        zoomOnAllowedArea ? allowedArea : null,
+        layoutParams.current
+      )
+    )
+    // eslint-disable-next-line
+  }, [imageLoaded])
+
+  useLayoutEffect(() => {
+    if (!imageDimensions) return
+    const { clientWidth, clientHeight } = canvas
+    canvas.width = clientWidth
+    canvas.height = clientHeight
+    const context = canvas.getContext("2d")
+
+    context.save()
+    context.transform(...mat.clone().inverse().toArray())
+
+    const { iw, ih } = layoutParams.current
+
+    if (allowedArea) {
+      // Pattern to indicate the NOT allowed areas
+      const { x, y, w, h } = allowedArea
+      context.save()
+      context.globalAlpha = 1
+      const outer = [
+        [0, 0],
+        [iw, 0],
+        [iw, ih],
+        [0, ih],
+      ]
+      const inner = [
+        [x * iw, y * ih],
+        [x * iw + w * iw, y * ih],
+        [x * iw + w * iw, y * ih + h * ih],
+        [x * iw, y * ih + h * ih],
+      ]
+      context.moveTo(...outer[0])
+      outer.forEach((p) => context.lineTo(...p))
+      context.lineTo(...outer[0])
+      context.closePath()
+
+      inner.reverse()
+      context.moveTo(...inner[0])
+      inner.forEach((p) => context.lineTo(...p))
+      context.lineTo(...inner[0])
+
+      context.fillStyle = excludePattern || "#f00"
+      context.fill()
+
+      context.restore()
+    }
+
+    context.restore()
+  })
+
+  const { iw, ih } = layoutParams.current
+
+  let zoomBox =
+    !zoomStart || !zoomEnd
+      ? null
+      : {
+          ...mat.clone().inverse().applyToPoint(zoomStart.x, zoomStart.y),
+          w: (zoomEnd.x - zoomStart.x) / mat.a,
+          h: (zoomEnd.y - zoomStart.y) / mat.d,
+        }
+  if (zoomBox) {
+    if (zoomBox.w < 0) {
+      zoomBox.x += zoomBox.w
+      zoomBox.w *= -1
+    }
+    if (zoomBox.h < 0) {
+      zoomBox.y += zoomBox.h
+      zoomBox.h *= -1
+    }
+  }
+
+  const imagePosition = {
+    topLeft: mat.clone().inverse().applyToPoint(0, 0),
+    bottomRight: mat.clone().inverse().applyToPoint(iw, ih),
+  }
+
+  const highlightedRegion = useMemo(() => {
+    const highlightedRegions = regions.filter((r) => r.highlighted)
+    if (highlightedRegions.length !== 1) return null
+    return highlightedRegions[0]
+  }, [regions])
+
+  return (
+    <ThemeProvider theme={theme}>
+      <div
+        style={{
+          width: "100%",
+          height: "100%",
+          maxHeight: "calc(100vh - 68px)",
+          position: "relative",
+          overflow: "hidden",
+          cursor: createWithPrimary
+            ? "crosshair"
+            : dragging
+            ? "grabbing"
+            : dragWithPrimary
+            ? "grab"
+            : zoomWithPrimary
+            ? mat.a < 1
+              ? "zoom-out"
+              : "zoom-in"
+            : undefined,
+        }}
+      >
+        {showCrosshairs && (
+          <Crosshairs key="crossHairs" mousePosition={mousePosition} />
+        )}
+        {imageLoaded && !dragging && (
+          <RegionSelectAndTransformBoxes
+            key="regionSelectAndTransformBoxes"
+            regions={
+              !modifyingAllowedArea || !allowedArea
+                ? regions
+                : [
+                    {
+                      type: "box",
+                      id: "$$allowed_area",
+                      cls: "allowed_area",
+                      highlighted: true,
+                      x: allowedArea.x,
+                      y: allowedArea.y,
+                      w: allowedArea.w,
+                      h: allowedArea.h,
+                      visible: true,
+                      color: "#ff0",
+                    },
+                  ]
+            }
+            mouseEvents={mouseEvents}
+            projectRegionBox={projectRegionBox}
+            dragWithPrimary={dragWithPrimary}
+            createWithPrimary={createWithPrimary}
+            zoomWithPrimary={zoomWithPrimary}
+            onBeginMovePoint={onBeginMovePoint}
+            onSelectRegion={onSelectRegion}
+            layoutParams={layoutParams}
+            mat={mat}
+            onBeginBoxTransform={onBeginBoxTransform}
+            onBeginMovePolygonPoint={onBeginMovePolygonPoint}
+            onBeginMoveKeypoint={onBeginMoveKeypoint}
+            onAddPolygonPoint={onAddPolygonPoint}
+            showHighlightBox={showHighlightBox}
+          />
+        )}
+        {imageLoaded && showTags && !dragging && (
+          <PreventScrollToParents key="regionTags">
+            <RegionTags
+              regions={regions}
+              projectRegionBox={projectRegionBox}
+              mouseEvents={mouseEvents}
+              regionClsList={regionClsList}
+              regionTagList={regionTagList}
+              onBeginRegionEdit={onBeginRegionEdit}
+              onChangeRegion={onChangeRegion}
+              onCloseRegionEdit={onCloseRegionEdit}
+              onDeleteRegion={onDeleteRegion}
+              layoutParams={layoutParams}
+              imageSrc={imageSrc}
+              RegionEditLabel={RegionEditLabel}
+              onRegionClassAdded={onRegionClassAdded}
+              allowComments={allowComments}
+            />
+          </PreventScrollToParents>
+        )}
+        {!showTags && highlightedRegion && (
+          <div key="topLeftTag" className={classes.fixedRegionLabel}>
+            <RegionLabel
+              disableClose
+              allowedClasses={regionClsList}
+              allowedTags={regionTagList}
+              onChange={onChangeRegion}
+              onDelete={onDeleteRegion}
+              editing
+              region={highlightedRegion}
+              imageSrc={imageSrc}
+              allowComments={allowComments}
+            />
+          </div>
+        )}
+
+        {zoomWithPrimary && zoomBox !== null && (
+          <div
+            key="zoomBox"
+            style={{
+              position: "absolute",
+              zIndex: 1,
+              border: "1px solid #fff",
+              pointerEvents: "none",
+              left: zoomBox.x,
+              top: zoomBox.y,
+              width: zoomBox.w,
+              height: zoomBox.h,
+            }}
+          />
+        )}
+        {showPointDistances && (
+          <PointDistances
+            key="pointDistances"
+            regions={regions}
+            realSize={realSize}
+            projectRegionBox={projectRegionBox}
+            pointDistancePrecision={pointDistancePrecision}
+          />
+        )}
+        <PreventScrollToParents
+          style={{ width: "100%", height: "100%" }}
+          {...mouseEvents}
+        >
+          <>
+            {fullImageSegmentationMode && (
+              <ImageMask
+                hide={!showMask}
+                autoSegmentationOptions={autoSegmentationOptions}
+                imagePosition={imagePosition}
+                regionClsList={regionClsList}
+                imageSrc={imageSrc}
+                regions={regions}
+              />
+            )}
+            <canvas
+              style={{ opacity: 0.25 }}
+              className={classes.canvas}
+              ref={canvasEl}
+            />
+            <RegionShapes
+              mat={mat}
+              keypointDefinitions={keypointDefinitions}
+              imagePosition={imagePosition}
+              regions={regions}
+              fullSegmentationMode={fullImageSegmentationMode}
+            />
+            <VideoOrImageCanvasBackground
+              videoPlaying={videoPlaying}
+              imagePosition={imagePosition}
+              mouseEvents={mouseEvents}
+              onLoad={onVideoOrImageLoaded}
+              videoTime={videoTime}
+              videoSrc={videoSrc}
+              imageSrc={imageSrc}
+              useCrossOrigin={fullImageSegmentationMode}
+              onChangeVideoTime={onChangeVideoTime}
+              onChangeVideoPlaying={onChangeVideoPlaying}
+            />
+          </>
+        </PreventScrollToParents>
+        <div className={classes.zoomIndicator}>
+          {((1 / mat.a) * 100).toFixed(0)}%
+        </div>
+      </div>
+    </ThemeProvider>
+  )
+}
+
+export default ImageCanvas

package/src/ImageCanvas/index.story.js

@@ -0,0 +1,214 @@
+import React from "react"
+
+import { storiesOf } from "@storybook/react"
+import { action } from "@storybook/addon-actions"
+
+import ImageCanvas from "./"
+import exampleMask from "./mouse_mask.story.png"
+import exampleImage from "./seves_desk.story.jpg"
+import dancingManImage from "./dancing-man.story.jpg"
+
+export const testRegions = [
+  {
+    type: "point",
+    id: "point1",
+    cls: "Paper",
+    highlighted: true,
+    x: 0.8,
+    y: 0.5,
+    visible: true,
+    color: "#f00",
+  },
+  {
+    type: "point",
+    id: "point2",
+    cls: "Dude's Head",
+    tags: ["human", "head", "male"],
+    x: 0.1,
+    y: 0.15,
+    visible: true,
+    color: "#0F0",
+  },
+  {
+    type: "box",
+    id: "box1",
+    cls: "Business Card",
+    highlighted: true,
+    x: 0.315,
+    y: 0.63,
+    w: 0.067,
+    h: 0.045,
+    visible: true,
+    color: "#ff0",
+  },
+  {
+    type: "polygon",
+    id: "polygon1",
+    cls: "Laptop",
+    tags: ["Electronic Device"],
+    editingLabels: true,
+    highlighted: true,
+    points: [
+      [0.4019, 0.5065],
+      [0.407, 0.5895],
+      [0.4157, 0.6801],
+      [0.6579, 0.656],
+      [0.6115, 0.5674],
+      [0.5792, 0.4895],
+    ],
+    visible: true,
+    color: "#f0f",
+  },
+  {
+    type: "polygon",
+    id: "polygon2",
+    open: true,
+    points: [
+      [0.1201, 0.5987],
+      [0.0674, 0.7063],
+      [0.0726, 0.7477],
+      [0.2132, 0.7311],
+    ],
+    visible: true,
+    color: "#00f",
+  },
+  {
+    type: "pixel",
+    id: "pixel1",
+    cls: "Mouse",
+    tags: ["Electronic Device"],
+    sx: 0.7433,
+    sy: 0.5847,
+    w: 0.83 - 0.7433,
+    h: 0.67 - 0.5847,
+    src: exampleMask,
+    visible: true,
+    color: "#00f",
+  },
+]
+
+const events = {
+  // Ignore common mouse movements, they fill the action log
+  onMouseMove: () => null, //action("onMouseMove"),
+  onMouseDown: () => null, //action("onMouseDown"),
+  onMouseUp: () => null, //action("onMouseUp"),
+
+  onChangeRegion: action("onChangeRegion"),
+  onBeginRegionEdit: action("onBeginRegionEdit"),
+  onCloseRegionEdit: action("onCloseRegionEdit"),
+
+  onSelectRegion: action("onSelectRegion"),
+
+  onBeginBoxTransform: action("onBeginBoxTransform"),
+
+  onBeginMovePolygonPoint: action("onBeginMovePolygonPoint"),
+  onAddPolygonPoint: action("onAddPolygonPoint"),
+  onClosePolygon: action("onClosePolygon"),
+
+  onBeginMoveKeypoint: action("onBeginMoveKeypoint"),
+
+  onBeginMovePoint: action("onBeginMovePoint"),
+  onDeleteRegion: action("onDeleteRegion"),
+}
+
+storiesOf("ImageCanvas", module)
+  .add("Basic", () => (
+    <ImageCanvas
+      regions={testRegions}
+      imageSrc={exampleImage}
+      showTags
+      {...events}
+    />
+  ))
+  .add("Zoom Tool Selected", () => (
+    <ImageCanvas
+      showTags
+      regions={testRegions}
+      imageSrc={exampleImage}
+      zoomWithPrimary
+      {...events}
+    />
+  ))
+  .add("Allowed Area", () => (
+    <ImageCanvas
+      showTags
+      regions={[]}
+      imageSrc={exampleImage}
+      zoomWithPrimary
+      allowedArea={{ x: 0.25, y: 0.25, w: 0.5, h: 0.5 }}
+      {...events}
+    />
+  ))
+  .add("Allowed Area (2)", () => (
+    <ImageCanvas
+      showTags
+      regions={[]}
+      imageSrc={exampleImage}
+      zoomWithPrimary
+      allowedArea={{ x: 0.6, y: 0.6, w: 0.2, h: 0.2 }}
+      {...events}
+    />
+  ))
+  .add("Modify Allowed Area", () => (
+    <ImageCanvas
+      showTags
+      regions={[]}
+      imageSrc={exampleImage}
+      modifyingAllowedArea
+      allowedArea={{ x: 0.6, y: 0.6, w: 0.2, h: 0.2 }}
+      {...events}
+    />
+  ))
+  .add("Poses", () => (
+    <ImageCanvas
+      keypointDefinitions={{
+        human: {
+          connections: [
+            ["head", "sternum"],
+            ["sternum", "leftElbow"],
+            ["sternum", "rightElbow"],
+          ],
+          landmarks: {
+            head: {
+              label: "Head",
+              color: "#f00",
+              defaultPosition: [0, -0.05],
+            },
+            sternum: {
+              label: "Torso",
+              color: "#0f0",
+              defaultPosition: [0, 0],
+            },
+            leftElbow: {
+              label: "Left Elbow",
+              color: "#00f",
+              defaultPosition: [-0.05, 0],
+            },
+            rightElbow: {
+              label: "Right Elbow",
+              color: "#00f",
+              defaultPosition: [0.05, 0],
+            },
+          },
+        },
+      }}
+      regions={[
+        {
+          type: "keypoints",
+          id: "keypoints1",
+          keypointsDefinitionId: "human",
+          highlighted: true,
+          points: {
+            head: { x: 0.54, y: 0.2 },
+            sternum: { x: 0.57, y: 0.3 },
+            leftElbow: { x: 0.4, y: 0.39 },
+            rightElbow: { x: 0.7, y: 0.32 },
+          },
+          visible: true,
+        },
+      ]}
+      imageSrc={dancingManImage}
+      showTags
+      {...events}
+    />
+  ))