lost-sia 2.0.0-alpha9 → 2.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lost-sia",
-  "version": "2.0.0-alpha9",
+  "version": "2.0.0",
   "description": "Single Image Annotation Tool",
   "license": "MIT",
   "repository": "l3p-cv/lost-sia",
@@ -42,6 +42,7 @@
     "@fortawesome/free-regular-svg-icons": "^6.5.2",
     "@fortawesome/free-solid-svg-icons": "^6.5.2",
     "@fortawesome/react-fontawesome": "^0.2.0",
+    "lodash-es": "^4.17.21",
     "react": "^18.0.0",
     "react-dom": "^18.0.0",
     "react-draggable": "^4.4.6",
@@ -90,4 +91,4 @@
       "plugin:storybook/recommended"
     ]
   }
-}
+}

package/src/Annotation/Annotation.jsx

@@ -281,6 +281,7 @@ class Annotation extends Component {
     //     this.props.allowedActions,
     //     this.state.anno
     // )
+    if (this.state.anno){console.log('ANNOTATION: ', this.state.anno.mode, this.state.anno)}
     switch (type) {
       case "point":
         return (

package/src/Annotation/Polygon.jsx

@@ -135,6 +135,11 @@ class Polygon extends Component {
           );
         }
         break;
+      case modes.ADD:
+        if (e.button === 0) {
+          this.removeNode(e, idx)
+        }
+        break;
       default:
         break;
     }
@@ -228,6 +233,17 @@ class Polygon extends Component {
     this.performedAction(newAnno, canvasActions.ANNO_ADDED_NODE);
   }
 
+  removeNode(e, idx) {
+    let newAnnoData = [...this.state.anno.data]
+    newAnnoData.splice(idx, 1)
+    const newAnno = {
+      ...this.state.anno,
+      data: newAnnoData,
+    };
+    this.setState({ anno: newAnno });
+    this.performedAction(newAnno, canvasActions.ANNO_REMOVED_SPECIFIC_NODE);
+  }
+  
   removeLastNode() {
     const newAnno = {
       ...this.state.anno,

package/src/Canvas.jsx

@@ -1,37 +1,29 @@
+import _ from "lodash";
 import React, { Component } from "react";
-import _, { transform } from "lodash";
+import { uniqueId } from "lodash-es";
 import Annotation from "./Annotation/Annotation";
 import AnnoLabelInput from "./AnnoLabelInput";
+import AnnoToolBar from "./AnnoToolBar";
 import ImgBar from "./ImgBar";
-import Prompt from "./Prompt";
 import LabelInput from "./LabelInput";
-import AnnoToolBar from "./AnnoToolBar";
+import Prompt from "./Prompt";
 
-import * as annoConversion from "./utils/annoConversion";
-import * as keyActions from "./utils/keyActions";
-import KeyMapper from "./utils/keyActions";
-import * as TOOLS from "./types/tools";
-import * as modes from "./types/modes";
-import UndoRedo from "./utils/hist";
-import * as transformAnnos from "./utils/transform";
+import { Button, Dimmer, Header, Icon, Loader } from "semantic-ui-react";
 import * as annoStatus from "./types/annoStatus";
 import * as canvasActions from "./types/canvasActions";
-import {
-  Loader,
-  Dimmer,
-  Icon,
-  Header,
-  Button,
-  Form,
-  TextArea,
-} from "semantic-ui-react";
-import * as mouse from "./utils/mouse";
-import * as colorlut from "./utils/colorlut";
+import * as modes from "./types/modes";
 import * as notificationType from "./types/notificationType";
+import * as TOOLS from "./types/tools";
+import * as annoConversion from "./utils/annoConversion";
+import * as colorlut from "./utils/colorlut";
+import UndoRedo from "./utils/hist";
+import KeyMapper, * as keyActions from "./utils/keyActions";
+import * as mouse from "./utils/mouse";
+import * as transformAnnos from "./utils/transform";
 import * as wv from "./utils/windowViewport";
 
-import "./SIA.scss";
 import InfoBoxes from "./InfoBoxes/InfoBoxArea";
+import "./SIA.scss";
 
 /**
  * SIA Canvas element that handles annotations within an image
@@ -192,6 +184,8 @@ class Canvas extends Component {
       possibleLabels: undefined,
       annoCommentInputTrigger: 0,
       imgActions: [],
+      isDrawingSamBBox: false, // Flag to indicate if a SAM bbox is currently being drawn,
+      samBBoxStartPoint: null,
     };
     this.img = React.createRef();
     this.svg = React.createRef();
@@ -386,6 +380,13 @@ class Canvas extends Component {
     } else if (e.button === 1) {
       this.setMode(modes.CAMERA_MOVE);
     } else if (e.button === 2) {
+      if (
+        this.props.selectedTool === TOOLS.POSITIVE_POINT ||
+        this.props.selectedTool === TOOLS.NEGATIVE_POINT ||
+        this.props.selectedTool === TOOLS.SAM_BBOX
+      ) {
+        return; // do nothing for sam tools
+      }
       //Create annotation on right click
       this.createNewAnnotation(e);
     }
@@ -396,6 +397,13 @@ class Canvas extends Component {
       // this.collectAnnos()
       this.setMode(modes.CAMERA_MOVE);
     } else if (e.button === 2) {
+      if (
+        this.props.selectedTool === TOOLS.POSITIVE_POINT ||
+        this.props.selectedTool === TOOLS.NEGATIVE_POINT ||
+        this.props.selectedTool === TOOLS.SAM_BBOX
+      ) {
+        return; // Note: do nothing for sam tools
+      }
       //Create annotation on right click
       this.createNewAnnotation(e);
     } else if (e.button === 0) {
@@ -427,6 +435,7 @@ class Canvas extends Component {
   handleKeyAction(action) {
     const anno = this.findAnno(this.state.selectedAnnoId);
     const camKeyStepSize = 20 * this.state.svg.scale;
+    console.log('handleKeyAction', action, anno)
 
     switch (action) {
       case keyActions.EDIT_LABEL:
@@ -499,7 +508,7 @@ class Canvas extends Component {
           this.recreateAnnotation(this.state.selectedAnnoId);
         break;
       default:
-      // console.warn("Unknown key action", action);
+        // console.warn("Unknown key action", action);
     }
   }
 
@@ -585,6 +594,7 @@ class Canvas extends Component {
     // console.log("handleAnnoEvent", pAction, anno);
     let newAnnos = undefined;
     let actionHistoryStore = undefined;
+    console.log('canvasActions: ', anno, pAction)
 
     switch (pAction) {
       case canvasActions.ANNO_ENTER_CREATE_MODE:
@@ -644,7 +654,7 @@ class Canvas extends Component {
         break;
       case canvasActions.ANNO_ADDED_NODE:
         actionHistoryStore = [...this.state.imgActions, pAction];
-        newAnnos = this.updateSelectedAnno(anno, modes.VIEW);
+        newAnnos = this.updateSelectedAnno(anno, modes.ADD);
         this.pushHist(newAnnos, anno.id, pAction, this.state.showSingleAnno);
         this.handleAnnoSaveEvent(pAction, anno, {
           imgActions: actionHistoryStore,
@@ -665,6 +675,21 @@ class Canvas extends Component {
           });
         }
         break;
+      case canvasActions.ANNO_REMOVED_SPECIFIC_NODE:
+        actionHistoryStore = [...this.state.imgActions, pAction];
+        if (!this.checkAnnoLength(anno)) {
+          newAnnos = this.updateSelectedAnno(anno, modes.DELETED);
+          this.showSingleAnno(undefined);
+        } else {
+          newAnnos = this.updateSelectedAnno(anno, modes.ADD)
+        }
+        this.pushHist(newAnnos, anno.id, pAction, this.state.showSingleAnno);
+        if (anno.status !== annoStatus.NEW) {
+          this.handleAnnoSaveEvent(pAction, anno, {
+            imgActions: actionHistoryStore,
+          });
+        }
+        break;
       case canvasActions.ANNO_EDITED:
         actionHistoryStore = [...this.state.imgActions, pAction];
         anno = this.stopAnnotimeMeasure(anno);
@@ -860,6 +885,53 @@ class Canvas extends Component {
     }
   }
 
+  handleContextMenu(e) {
+    e.preventDefault();
+    if (
+      this.props.selectedTool === TOOLS.POSITIVE_POINT ||
+      this.props.selectedTool === TOOLS.NEGATIVE_POINT
+    ) {
+      const { x, y } = this.getMousePosition(e);
+
+      const imgWidth = this.state.svg.width - 2 * this.state.imageOffset.x;
+      const imgHeight = this.state.svg.height - 2 * this.state.imageOffset.y;
+
+      const normX = (x - this.state.imageOffset.x) / imgWidth;
+      const normY = (y - this.state.imageOffset.y) / imgHeight;
+
+      const type =
+        this.props.selectedTool === TOOLS.POSITIVE_POINT
+          ? "positive"
+          : "negative";
+
+      this.props.onSamPointClick(x, y, normX, normY, type);
+    }
+  }
+
+  handleMouseDown(e) {
+    if (this.props.selectedTool === TOOLS.SAM_BBOX) {
+      if (e.button !== 2) {
+        return;
+      }
+
+      e.preventDefault();
+      const { x, y } = this.getMousePosition(e);
+
+      this.setState({
+        isDrawingSamBBox: true,
+        samBBoxStartPoint: { x, y },
+      });
+
+      // start a new rectangle
+      this.props.onUpdateSamBBox({
+        x,
+        y,
+        width: 0,
+        height: 0,
+      });
+    }
+  }
+
   handleImgBarMouseEnter(e) {
     this.setState({ imgBarVisible: false });
   }
@@ -870,6 +942,64 @@ class Canvas extends Component {
 
   handleSvgMouseMove(e) {
     this.mousePosAbs = mouse.getMousePositionAbs(e, this.state.svg);
+
+    if (this.props.selectedTool === TOOLS.SAM_BBOX) {
+      if (!this.state.isDrawingSamBBox || !this.state.samBBoxStartPoint) {
+        // If not drawing a SAM bbox, do nothing
+        return;
+      }
+
+      const { x, y } = this.getMousePosition(e);
+      const startPoint = this.state.samBBoxStartPoint;
+
+      const imgWidth = this.state.svg.width - 2 * this.state.imageOffset.x;
+      const imgHeight = this.state.svg.height - 2 * this.state.imageOffset.y;
+
+      const xMin = Math.min(startPoint.x, x);
+      const yMin = Math.min(startPoint.y, y);
+      const width = Math.abs(x - startPoint.x);
+      const height = Math.abs(y - startPoint.y);
+
+      // normalized coordinates
+      const xMinNorm = (xMin - this.state.imageOffset.x) / imgWidth;
+      const yMinNorm = (yMin - this.state.imageOffset.y) / imgHeight;
+      const xMaxNorm = (xMin + width - this.state.imageOffset.x) / imgWidth;
+      const yMaxNorm = (yMin + height - this.state.imageOffset.y) / imgHeight;
+
+      // update the rectangle with the new width and height
+      this.props.onUpdateSamBBox({
+        x: xMin,
+        y: yMin,
+        width,
+        height,
+        xMinNorm,
+        yMinNorm,
+        xMaxNorm,
+        yMaxNorm,
+      });
+    }
+  }
+
+  handleMouseUp(e) {
+    if (this.props.selectedTool === TOOLS.SAM_BBOX) {
+      if (e.button !== 2) {
+        return;
+      }
+
+      e.preventDefault();
+      this.setState({
+        isDrawingSamBBox: false,
+      });
+    }
+  }
+
+  handleMouseLeave(e) {
+    if (this.props.selectedTool === TOOLS.SAM_BBOX) {
+      // If mouse leaves the canvas while drawing a SAM bbox, reset the state
+      this.setState({
+        isDrawingSamBBox: false,
+      });
+    }
   }
 
   handleNotification(messageObj) {
@@ -929,7 +1059,7 @@ class Canvas extends Component {
     // const corrected = transform.correctAnnotation(anno.data, this.props.svg, this.props.imageOffset)
     if (this.clipboard) {
       // let annos = [...this.state.annos]
-      const uid = _.uniqueId("new");
+      const uid = uniqueId("new");
       // this.handleAnnoEvent()
       const newData = this.clipboard.data.map((e) => {
         return { x: e.x + offset, y: e.y + offset };
@@ -1333,6 +1463,7 @@ class Canvas extends Component {
     //Do not create new Annotation if controlKey was pressed!
     let allowed = false;
     if (this.keyMapper.controlDown) return;
+    if (this.keyMapper.shiftDown) return;
     if (this.props.selectedTool) {
       const maxAnnos = this.props.canvasConfig.annos.maxAnnos;
       if (maxAnnos) {
@@ -1364,7 +1495,7 @@ class Canvas extends Component {
       const mousePos = this.getMousePosition(e);
       // const selAnno = this.findAnno(this.state.selectedAnnoId)
       let newAnno = {
-        id: this.props.nextAnnoId ? this.props.nextAnnoId : _.uniqueId("new"),
+        id: this.props.nextAnnoId ? this.props.nextAnnoId : uniqueId("new"),
         type: this.props.selectedTool,
         data: [
           {
@@ -1587,6 +1718,7 @@ class Canvas extends Component {
    *      the new annos list that was set in state
    */
   updateSelectedAnno(anno, mode = undefined, returnNewAnno = false) {
+    // console.log('updateSelectedAnno: ', mode, anno)
     if (!anno) return;
     const { newAnnos, newAnno } = this.mergeSelectedAnno(anno, mode);
     this.setState({
@@ -1878,6 +2010,34 @@ class Canvas extends Component {
     );
   }
 
+  renderSamPoints() {
+    return this.props.samPoints.map((point, index) => (
+      <circle
+        key={index}
+        cx={point.x}
+        cy={point.y}
+        r={5}
+        fill={point.type === "positive" ? "lightgreen" : "lightcoral"}
+      />
+    ));
+  }
+
+  renderSamBBox() {
+    return (
+      this.props.samBBox && (
+        <rect
+          x={this.props.samBBox.x}
+          y={this.props.samBBox.y}
+          width={this.props.samBBox.width}
+          height={this.props.samBBox.height}
+          fill="rgba(0, 0, 255, 0.1)"
+          stroke="blue"
+          strokeWidth="2"
+        />
+      )
+    );
+  }
+
   render() {
     const selectedAnno = this.findAnno(this.state.selectedAnnoId);
     return (
@@ -1970,6 +2130,10 @@ class Canvas extends Component {
             onKeyUp={(e) => this.onKeyUp(e)}
             onClick={(e) => this.handleCanvasClick(e)}
             onMouseMove={(e) => this.handleSvgMouseMove(e)}
+            onContextMenu={(e) => this.handleContextMenu(e)}
+            onMouseDown={(e) => this.handleMouseDown(e)}
+            onMouseUp={(e) => this.handleMouseUp(e)}
+            onMouseLeave={(e) => this.handleMouseLeave(e)}
             tabIndex="0"
           >
             <g
@@ -2005,6 +2169,8 @@ class Canvas extends Component {
                 }
               />
               {this.renderAnnotations()}
+              {this.renderSamPoints()}
+              {this.renderSamBBox()}
             </g>
           </svg>
           <img

package/src/Sia.jsx

@@ -1,10 +1,10 @@
-import React, { useRef, useEffect, useState } from "react";
+import React, { useEffect, useRef, useState } from "react";
 
-import ToolBar from "./ToolBar";
 import Canvas from "./Canvas";
-import * as tbe from "./types/toolbarEvents";
-import * as annoActions from "./types/canvasActions";
 import { noAnnos } from "./siaDummyData";
+import ToolBar from "./ToolBar";
+import * as annoActions from "./types/canvasActions";
+import * as tbe from "./types/toolbarEvents";
 
 /**
  * SIA element that handles annotations within an image
@@ -446,6 +446,10 @@ const Sia = (props) => {
         isImageChanging={props.isImageChanging}
         isStaticPosition={props.isStaticPosition}
         fixedImageSize={props.fixedImageSize}
+        samPoints={props.samPoints || []}
+        onSamPointClick={props.onSamPointClick}
+        samBBox={props.samBBox || null}
+        onUpdateSamBBox={props.onUpdateSamBBox}
       />
       <ToolBar
         onToolBarEvent={(e, data) => handleToolBarEvent(e, data)}

package/src/ToolBar.jsx

@@ -1,11 +1,16 @@
-import React, { useState, useEffect, useRef } from "react";
-import { Button, Card } from "semantic-ui-react";
+import {
+  faPaperPlane,
+  faSquare,
+  faTrashCan,
+} from "@fortawesome/free-regular-svg-icons";
 import {
   faArrowLeft,
   faArrowRight,
   faBan,
   faCheck,
   faMaximize,
+  faMinusCircle,
+  faPlusCircle,
   faQuestion,
   faSave,
   faSearch,
@@ -13,18 +18,19 @@ import {
   faTrash,
   faVectorSquare,
 } from "@fortawesome/free-solid-svg-icons";
-import { faPaperPlane } from "@fortawesome/free-regular-svg-icons";
-import SIASettingButton from "./SIASettingButton";
-import SIAFilterButton from "./SIAFilterButton";
+import React, { useEffect, useRef, useState } from "react";
+import { Button, Card } from "semantic-ui-react";
 import Prompt from "./Prompt";
+import SIAFilterButton from "./SIAFilterButton";
+import SIASettingButton from "./SIASettingButton";
 import "./Toolbar.css";
 
-import * as TOOLS from "./types/tools";
-import * as siaIcons from "./utils/siaIcons";
-import * as tbe from "./types/toolbarEvents";
 import { CSidebar, CSidebarNav } from "@coreui/react";
-import ToolbarItem from "./ToolbarItem";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import ToolbarItem from "./ToolbarItem";
+import * as tbe from "./types/toolbarEvents";
+import * as TOOLS from "./types/tools";
+import * as siaIcons from "./utils/siaIcons";
 
 const ToolBar = (props) => {
   const [position, setPosition] = useState({
@@ -125,6 +131,44 @@ const ToolBar = (props) => {
     if (!props.enabled.toolSelection) return null;
     if (!props.canvasConfig.annos.actions.draw) return null;
     let btns = [];
+
+    if (props.canvasConfig.tools.sam) {
+      btns.push(
+        <ToolbarItem
+          color="#0081FB"
+          active={props.active.selectedTool === TOOLS.POSITIVE_POINT}
+          onClick={(e) => onClick(e, TOOLS.POSITIVE_POINT)}
+          faIcon={faPlusCircle}
+        />,
+      );
+
+      btns.push(
+        <ToolbarItem
+          color="#0081FB"
+          active={props.active.selectedTool === TOOLS.NEGATIVE_POINT}
+          onClick={(e) => onClick(e, TOOLS.NEGATIVE_POINT)}
+          faIcon={faMinusCircle}
+        />,
+      );
+
+      btns.push(
+        <ToolbarItem
+          color="#0081FB"
+          active={props.active.selectedTool === TOOLS.SAM_BBOX}
+          onClick={(e) => onClick(e, TOOLS.SAM_BBOX)}
+          faIcon={faSquare}
+        />,
+      );
+
+      btns.push(
+        <ToolbarItem
+          color="#0081FB"
+          onClick={(e) => triggerToolBarEvent(tbe.CLEAR_SAM_HELPER_ANNOS)}
+          faIcon={faTrashCan}
+        />,
+      );
+    }
+
     if (props.canvasConfig.tools.point) {
       btns.push(
         <ToolbarItem
@@ -279,8 +323,9 @@ const ToolBar = (props) => {
                   <Card.Content description="Redo: Hit STRG + R" />
                 </Card>
                 <Card>
-                  <Card.Content header="Add a node to Line/Polygon" />
-                  <Card.Content description="Hit STRG + Click left on the line" />
+                  <Card.Content header="Add/ Remove a node to Line/Polygon" />
+                  <Card.Content description="Add: Hit STRG + Click left on the line" />
+                  <Card.Content description="Remove: Hit STRG + Click left on the node to delete" />
                 </Card>
                 <Card>
                   <Card.Content header="Remove a node from Line/Polygon in create mode" />

package/src/ToolbarItem.jsx

@@ -8,8 +8,8 @@ const ToolbarItem = ({
   faIcon,
   siaIcon,
   onClick,
+  color = "white",
 }) => {
-  let color = "white";
   if (active) color = "red";
   if (disabled) color = "grey";
 

package/src/types/canvasActions.js

@@ -8,6 +8,7 @@ export const ANNO_EDITED = "annoEdited";
 export const ANNO_ENTER_EDIT_MODE = "annoEnterEditMode";
 export const ANNO_ADDED_NODE = "annoAddedNode";
 export const ANNO_REMOVED_NODE = "annoRemovedNode";
+export const ANNO_REMOVED_SPECIFIC_NODE = "annoRemovedSpecificNode";
 export const ANNO_LABEL_UPDATE = "annoLabelUpdate";
 export const ANNO_CREATED_NODE = "annoCreatedNode";
 export const ANNO_CREATED_FINAL_NODE = "annoCreatedFinalNode";

package/src/types/modes.js

@@ -3,6 +3,7 @@ export const EDIT = "edit";
 export const CREATE = "create";
 export const MOVE = "move";
 export const ADD = "add";
+export const REMOVE_NODE = "removeNode";
 export const CAMERA_MOVE = "cameraMove";
 export const EDIT_LABEL = "editLabel";
 export const DELETED = "deleted";

package/src/types/toolbarEvents.js

@@ -13,6 +13,7 @@ export const EDIT_STROKE_WIDTH = "editStrokeWidth";
 export const EDIT_NODE_RADIUS = "editNodeRadius";
 export const APPLY_FILTER = "applyFilter";
 export const SAVE = "save";
+export const CLEAR_SAM_HELPER_ANNOS = "clearSamHelperAnnos"; // for clearing SAM helper annotations
 
 export default {
   TOOL_SELECTED,
@@ -30,4 +31,5 @@ export default {
   EDIT_NODE_RADIUS,
   APPLY_FILTER,
   SAVE,
+  CLEAR_POINTS: CLEAR_SAM_HELPER_ANNOS,
 };

package/src/types/tools.js

@@ -2,10 +2,16 @@ export const POINT = "point";
 export const BBOX = "bBox";
 export const LINE = "line";
 export const POLYGON = "polygon";
+export const POSITIVE_POINT = "positivePoint"; // for positive point SAM
+export const NEGATIVE_POINT = "negativePoint"; // for negative point SAM
+export const SAM_BBOX = "samBBox"; // bbox for guiding SAM
 
 export default {
   POINT,
   BBOX,
   LINE,
   POLYGON,
+  POSITIVE_POINT,
+  NEGATIVE_POINT,
+  SAM_BBOX,
 };

package/src/utils/annoConversion.js

@@ -1,12 +1,12 @@
 import * as transform from "./transform";
 import * as annoStatus from "../types/annoStatus";
 import * as modes from "../types/modes";
-import _ from "lodash";
+import { uniqueId } from "lodash-es";
 
 function _fixBackendAnnoElement(element) {
   return {
     ...element,
-    id: element.id ? element.id : _.uniqueId("new"),
+    id: element.id ? element.id : uniqueId("new"),
     annoTime: element.annoTime ? element.annoTime : 0.0,
     mode: element.mode ? element.mode : modes.VIEW,
     status: element.status ? element.status : annoStatus.DATABASE,

package/src/utils/keyActions.js

@@ -19,6 +19,7 @@ export const TOGGLE_ANNO_COMMENT_INPUT = "toggleAnnoCommentInput";
 class KeyMapper {
   constructor(keyActionHandler = undefined) {
     this.controlDown = false;
+    this.shiftDown = false;
     this.keyActionHandler = keyActionHandler;
   }
 
@@ -37,6 +38,9 @@ class KeyMapper {
         this.controlDown = true;
         this.triggerKeyAction(ENTER_ANNO_ADD_MODE);
         break;
+      case "Shift":
+        this.shiftDown = true;
+        break;
       case "z":
         if (this.controlDown) {
           this.triggerKeyAction(UNDO);
@@ -91,6 +95,9 @@ class KeyMapper {
         this.controlDown = false;
         this.triggerKeyAction(LEAVE_ANNO_ADD_MODE);
         break;
+      case "Shift":
+        this.shiftDown = false;
+        break;
       default:
         break;
     }