@midscene/shared 1.10.2-beta-20260706032158.0 → 1.10.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/env/parse-model-config.mjs +1 -1
- package/dist/es/img/box-select.mjs +122 -47
- package/dist/lib/env/parse-model-config.js +1 -1
- package/dist/lib/img/box-select.js +122 -47
- package/dist/types/img/box-select.d.ts +2 -0
- package/package.json +1 -1
- package/src/agent-tools/base-tools.ts +1 -1
- package/src/img/box-select.ts +165 -35
|
@@ -5,7 +5,7 @@ import { assert } from "../utils.mjs";
|
|
|
5
5
|
import { maskConfig, parseJson } from "./helper.mjs";
|
|
6
6
|
import { initDebugConfig } from "./init-debug.mjs";
|
|
7
7
|
const MODEL_CONFIG_DOC_URL = 'https://midscenejs.com/model-common-config.html';
|
|
8
|
-
const getCurrentVersion = ()=>"1.10.
|
|
8
|
+
const getCurrentVersion = ()=>"1.10.3";
|
|
9
9
|
const getInvalidModelFamilyMessage = (modelFamily)=>`Invalid MIDSCENE_MODEL_FAMILY value: ${modelFamily}. Current version v${getCurrentVersion()} accepts the following model families: ${MODEL_FAMILY_VALUES.join(', ')}. You can also visit ${MODEL_CONFIG_DOC_URL} for the latest configuration information.`;
|
|
10
10
|
const KEYS_MAP = {
|
|
11
11
|
insight: INSIGHT_MODEL_CONFIG_KEYS,
|
|
@@ -597,6 +597,27 @@ function drawCircle(pixels, width, height, center, radius, color, options = {})
|
|
|
597
597
|
pixels[idx + 3] = color.a;
|
|
598
598
|
}
|
|
599
599
|
}
|
|
600
|
+
function drawEllipse(pixels, width, height, center, radiusX, radiusY, color, thickness = 2) {
|
|
601
|
+
const cx = Math.round(center.x);
|
|
602
|
+
const cy = Math.round(center.y);
|
|
603
|
+
const outerRadiusX = Math.max(Math.round(radiusX), 1);
|
|
604
|
+
const outerRadiusY = Math.max(Math.round(radiusY), 1);
|
|
605
|
+
const innerRadiusX = Math.max(outerRadiusX - Math.round(thickness), 0);
|
|
606
|
+
const innerRadiusY = Math.max(outerRadiusY - Math.round(thickness), 0);
|
|
607
|
+
for(let py = cy - outerRadiusY; py <= cy + outerRadiusY; py++)for(let px = cx - outerRadiusX; px <= cx + outerRadiusX; px++){
|
|
608
|
+
if (px < 0 || py < 0 || px >= width || py >= height) continue;
|
|
609
|
+
const dx = px - cx;
|
|
610
|
+
const dy = py - cy;
|
|
611
|
+
const outer = dx * dx / (outerRadiusX * outerRadiusX) + dy * dy / (outerRadiusY * outerRadiusY);
|
|
612
|
+
const inner = 0 === innerRadiusX || 0 === innerRadiusY ? 0 : dx * dx / (innerRadiusX * innerRadiusX) + dy * dy / (innerRadiusY * innerRadiusY);
|
|
613
|
+
if (outer > 1 || inner < 1) continue;
|
|
614
|
+
const idx = (py * width + px) * 4;
|
|
615
|
+
pixels[idx + 0] = color.r;
|
|
616
|
+
pixels[idx + 1] = color.g;
|
|
617
|
+
pixels[idx + 2] = color.b;
|
|
618
|
+
pixels[idx + 3] = color.a;
|
|
619
|
+
}
|
|
620
|
+
}
|
|
600
621
|
function drawLine(pixels, width, height, from, to, color, thickness = 2) {
|
|
601
622
|
const x0 = Math.round(from.x);
|
|
602
623
|
const y0 = Math.round(from.y);
|
|
@@ -619,59 +640,107 @@ function drawLine(pixels, width, height, from, to, color, thickness = 2) {
|
|
|
619
640
|
}
|
|
620
641
|
}
|
|
621
642
|
}
|
|
622
|
-
function
|
|
623
|
-
const
|
|
643
|
+
function drawCallout(pixels, width, height, targetPoint, markerRadius, style) {
|
|
644
|
+
const cx = Math.round(targetPoint.x);
|
|
645
|
+
const cy = Math.round(targetPoint.y);
|
|
646
|
+
const gap = Math.max(markerRadius + 18, 30);
|
|
647
|
+
const candidates = [
|
|
648
|
+
{
|
|
649
|
+
x: cx + gap,
|
|
650
|
+
y: cy - gap
|
|
651
|
+
},
|
|
652
|
+
{
|
|
653
|
+
x: cx + gap,
|
|
654
|
+
y: cy + gap
|
|
655
|
+
},
|
|
656
|
+
{
|
|
657
|
+
x: cx - gap,
|
|
658
|
+
y: cy - gap
|
|
659
|
+
},
|
|
660
|
+
{
|
|
661
|
+
x: cx - gap,
|
|
662
|
+
y: cy + gap
|
|
663
|
+
},
|
|
664
|
+
{
|
|
665
|
+
x: cx,
|
|
666
|
+
y: cy - gap
|
|
667
|
+
},
|
|
668
|
+
{
|
|
669
|
+
x: cx,
|
|
670
|
+
y: cy + gap
|
|
671
|
+
},
|
|
672
|
+
{
|
|
673
|
+
x: cx + gap,
|
|
674
|
+
y: cy
|
|
675
|
+
},
|
|
676
|
+
{
|
|
677
|
+
x: cx - gap,
|
|
678
|
+
y: cy
|
|
679
|
+
}
|
|
680
|
+
];
|
|
681
|
+
const calloutCenter = candidates.find((candidate)=>candidate.x - markerRadius >= 0 && candidate.x + markerRadius < width && candidate.y - markerRadius >= 0 && candidate.y + markerRadius < height) || {
|
|
682
|
+
x: Math.min(Math.max(cx + gap, markerRadius), width - markerRadius - 1),
|
|
683
|
+
y: Math.min(Math.max(cy - gap, markerRadius), height - markerRadius - 1)
|
|
684
|
+
};
|
|
685
|
+
const lineAngle = Math.atan2(cy - calloutCenter.y, cx - calloutCenter.x);
|
|
686
|
+
const lineStart = {
|
|
687
|
+
x: calloutCenter.x + Math.cos(lineAngle) * (markerRadius + 1),
|
|
688
|
+
y: calloutCenter.y + Math.sin(lineAngle) * (markerRadius + 1)
|
|
689
|
+
};
|
|
690
|
+
const targetRadiusX = 30;
|
|
691
|
+
const targetRadiusY = 15;
|
|
692
|
+
const ellipseBoundaryDistance = 1 / Math.sqrt(Math.cos(lineAngle) ** 2 / targetRadiusX ** 2 + Math.sin(lineAngle) ** 2 / targetRadiusY ** 2);
|
|
693
|
+
const lineEnd = {
|
|
694
|
+
x: cx - Math.cos(lineAngle) * ellipseBoundaryDistance,
|
|
695
|
+
y: cy - Math.sin(lineAngle) * ellipseBoundaryDistance
|
|
696
|
+
};
|
|
697
|
+
drawLine(pixels, width, height, lineStart, lineEnd, style.callout, 2);
|
|
698
|
+
drawCircle(pixels, width, height, calloutCenter, markerRadius + 2, style.calloutBorder, {
|
|
699
|
+
thickness: 3
|
|
700
|
+
});
|
|
701
|
+
drawCircle(pixels, width, height, calloutCenter, markerRadius, style.callout, {
|
|
702
|
+
fill: true
|
|
703
|
+
});
|
|
704
|
+
drawEllipse(pixels, width, height, targetPoint, targetRadiusX, targetRadiusY, style.callout, 2);
|
|
705
|
+
const textWidth = getNumberWidth(style.indexId);
|
|
706
|
+
const textHeight = FONT_HEIGHT * FONT_SCALE;
|
|
707
|
+
drawNumber(pixels, width, height, style.indexId, Math.round(calloutCenter.x - textWidth / 2), Math.round(calloutCenter.y - textHeight / 2), style.calloutText);
|
|
708
|
+
}
|
|
709
|
+
function drawPointMarker(pixels, width, height, point, radius, indexId = 1) {
|
|
710
|
+
const markerColors = [
|
|
711
|
+
{
|
|
712
|
+
r: 0xc6,
|
|
713
|
+
g: 0x23,
|
|
714
|
+
b: 0x00,
|
|
715
|
+
a: 0xee
|
|
716
|
+
},
|
|
717
|
+
{
|
|
718
|
+
r: 0x00,
|
|
719
|
+
g: 0x00,
|
|
720
|
+
b: 0xff,
|
|
721
|
+
a: 0xee
|
|
722
|
+
}
|
|
723
|
+
];
|
|
724
|
+
const callout = markerColors[(indexId - 1) % markerColors.length];
|
|
725
|
+
const calloutBorder = {
|
|
624
726
|
r: 0xff,
|
|
625
|
-
g:
|
|
626
|
-
b:
|
|
727
|
+
g: 0xff,
|
|
728
|
+
b: 0xff,
|
|
627
729
|
a: 0xff
|
|
628
730
|
};
|
|
629
|
-
const
|
|
731
|
+
const calloutText = {
|
|
630
732
|
r: 0xff,
|
|
631
733
|
g: 0xff,
|
|
632
734
|
b: 0xff,
|
|
633
735
|
a: 0xff
|
|
634
736
|
};
|
|
635
|
-
const
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
drawCircle(pixels, width, height, point, markerRadius, red, {
|
|
642
|
-
thickness: 3
|
|
737
|
+
const markerRadius = Math.max(Math.round(radius), 10);
|
|
738
|
+
drawCallout(pixels, width, height, point, markerRadius, {
|
|
739
|
+
callout,
|
|
740
|
+
calloutBorder,
|
|
741
|
+
calloutText,
|
|
742
|
+
indexId
|
|
643
743
|
});
|
|
644
|
-
drawCircle(pixels, width, height, point, 3, red, {
|
|
645
|
-
fill: true
|
|
646
|
-
});
|
|
647
|
-
drawLine(pixels, width, height, {
|
|
648
|
-
x: cx - markerRadius - 4,
|
|
649
|
-
y: cy
|
|
650
|
-
}, {
|
|
651
|
-
x: cx - markerRadius + 2,
|
|
652
|
-
y: cy
|
|
653
|
-
}, red, 2);
|
|
654
|
-
drawLine(pixels, width, height, {
|
|
655
|
-
x: cx + markerRadius - 2,
|
|
656
|
-
y: cy
|
|
657
|
-
}, {
|
|
658
|
-
x: cx + markerRadius + 4,
|
|
659
|
-
y: cy
|
|
660
|
-
}, red, 2);
|
|
661
|
-
drawLine(pixels, width, height, {
|
|
662
|
-
x: cx,
|
|
663
|
-
y: cy - markerRadius - 4
|
|
664
|
-
}, {
|
|
665
|
-
x: cx,
|
|
666
|
-
y: cy - markerRadius + 2
|
|
667
|
-
}, red, 2);
|
|
668
|
-
drawLine(pixels, width, height, {
|
|
669
|
-
x: cx,
|
|
670
|
-
y: cy + markerRadius - 2
|
|
671
|
-
}, {
|
|
672
|
-
x: cx,
|
|
673
|
-
y: cy + markerRadius + 4
|
|
674
|
-
}, red, 2);
|
|
675
744
|
}
|
|
676
745
|
function blendPixels(basePixels, overlayPixels, width, height) {
|
|
677
746
|
const result = new Uint8Array(basePixels.length);
|
|
@@ -700,7 +769,7 @@ function blendPixels(basePixels, overlayPixels, width, height) {
|
|
|
700
769
|
}
|
|
701
770
|
return result;
|
|
702
771
|
}
|
|
703
|
-
const createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5, borderThickness = 2, prompt)=>{
|
|
772
|
+
const createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5, borderThickness = 2, prompt, centerPoint = false)=>{
|
|
704
773
|
const overlayPixels = new Uint8Array(imageWidth * imageHeight * 4);
|
|
705
774
|
const colors = [
|
|
706
775
|
{
|
|
@@ -804,6 +873,12 @@ const createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding =
|
|
|
804
873
|
h: paddedHeight
|
|
805
874
|
};
|
|
806
875
|
drawRect(overlayPixels, imageWidth, imageHeight, paddedRect, color.rect, borderThickness);
|
|
876
|
+
if (centerPoint) drawCircle(overlayPixels, imageWidth, imageHeight, {
|
|
877
|
+
x: element.rect.left + element.rect.width / 2,
|
|
878
|
+
y: element.rect.top + element.rect.height / 2
|
|
879
|
+
}, 4, color.rect, {
|
|
880
|
+
fill: true
|
|
881
|
+
});
|
|
807
882
|
const indexId = element.indexId;
|
|
808
883
|
if ('number' != typeof indexId) continue;
|
|
809
884
|
const textWidth = getNumberWidth(indexId);
|
|
@@ -868,7 +943,7 @@ const compositeElementInfoImg = async (options)=>{
|
|
|
868
943
|
const { elementsPositionInfo, prompt } = options;
|
|
869
944
|
try {
|
|
870
945
|
const basePixels = photonImage.get_raw_pixels();
|
|
871
|
-
const overlayPixels = await createSvgOverlay(elementsPositionInfo, width, height, options.annotationPadding, options.borderThickness, prompt);
|
|
946
|
+
const overlayPixels = await createSvgOverlay(elementsPositionInfo, width, height, options.annotationPadding, options.borderThickness, prompt, options.centerPoint);
|
|
872
947
|
const blendedPixels = blendPixels(basePixels, overlayPixels, width, height);
|
|
873
948
|
const resultImage = new PhotonImage(blendedPixels, width, height);
|
|
874
949
|
const base64 = await photonToBase64(resultImage, 90);
|
|
@@ -907,7 +982,7 @@ const compositePointMarkerImg = async (options)=>{
|
|
|
907
982
|
try {
|
|
908
983
|
const basePixels = photonImage.get_raw_pixels();
|
|
909
984
|
const overlayPixels = new Uint8Array(width * height * 4);
|
|
910
|
-
drawPointMarker(overlayPixels, width, height, options.point, options.radius ?? 14);
|
|
985
|
+
drawPointMarker(overlayPixels, width, height, options.point, options.radius ?? 14, options.indexId ?? 1);
|
|
911
986
|
const blendedPixels = blendPixels(basePixels, overlayPixels, width, height);
|
|
912
987
|
const resultImage = new PhotonImage(blendedPixels, width, height);
|
|
913
988
|
const base64 = await photonToBase64(resultImage, 90);
|
|
@@ -37,7 +37,7 @@ const external_utils_js_namespaceObject = require("../utils.js");
|
|
|
37
37
|
const external_helper_js_namespaceObject = require("./helper.js");
|
|
38
38
|
const external_init_debug_js_namespaceObject = require("./init-debug.js");
|
|
39
39
|
const MODEL_CONFIG_DOC_URL = 'https://midscenejs.com/model-common-config.html';
|
|
40
|
-
const getCurrentVersion = ()=>"1.10.
|
|
40
|
+
const getCurrentVersion = ()=>"1.10.3";
|
|
41
41
|
const getInvalidModelFamilyMessage = (modelFamily)=>`Invalid MIDSCENE_MODEL_FAMILY value: ${modelFamily}. Current version v${getCurrentVersion()} accepts the following model families: ${external_types_js_namespaceObject.MODEL_FAMILY_VALUES.join(', ')}. You can also visit ${MODEL_CONFIG_DOC_URL} for the latest configuration information.`;
|
|
42
42
|
const KEYS_MAP = {
|
|
43
43
|
insight: external_constants_js_namespaceObject.INSIGHT_MODEL_CONFIG_KEYS,
|
|
@@ -639,6 +639,27 @@ function drawCircle(pixels, width, height, center, radius, color, options = {})
|
|
|
639
639
|
pixels[idx + 3] = color.a;
|
|
640
640
|
}
|
|
641
641
|
}
|
|
642
|
+
function drawEllipse(pixels, width, height, center, radiusX, radiusY, color, thickness = 2) {
|
|
643
|
+
const cx = Math.round(center.x);
|
|
644
|
+
const cy = Math.round(center.y);
|
|
645
|
+
const outerRadiusX = Math.max(Math.round(radiusX), 1);
|
|
646
|
+
const outerRadiusY = Math.max(Math.round(radiusY), 1);
|
|
647
|
+
const innerRadiusX = Math.max(outerRadiusX - Math.round(thickness), 0);
|
|
648
|
+
const innerRadiusY = Math.max(outerRadiusY - Math.round(thickness), 0);
|
|
649
|
+
for(let py = cy - outerRadiusY; py <= cy + outerRadiusY; py++)for(let px = cx - outerRadiusX; px <= cx + outerRadiusX; px++){
|
|
650
|
+
if (px < 0 || py < 0 || px >= width || py >= height) continue;
|
|
651
|
+
const dx = px - cx;
|
|
652
|
+
const dy = py - cy;
|
|
653
|
+
const outer = dx * dx / (outerRadiusX * outerRadiusX) + dy * dy / (outerRadiusY * outerRadiusY);
|
|
654
|
+
const inner = 0 === innerRadiusX || 0 === innerRadiusY ? 0 : dx * dx / (innerRadiusX * innerRadiusX) + dy * dy / (innerRadiusY * innerRadiusY);
|
|
655
|
+
if (outer > 1 || inner < 1) continue;
|
|
656
|
+
const idx = (py * width + px) * 4;
|
|
657
|
+
pixels[idx + 0] = color.r;
|
|
658
|
+
pixels[idx + 1] = color.g;
|
|
659
|
+
pixels[idx + 2] = color.b;
|
|
660
|
+
pixels[idx + 3] = color.a;
|
|
661
|
+
}
|
|
662
|
+
}
|
|
642
663
|
function drawLine(pixels, width, height, from, to, color, thickness = 2) {
|
|
643
664
|
const x0 = Math.round(from.x);
|
|
644
665
|
const y0 = Math.round(from.y);
|
|
@@ -661,59 +682,107 @@ function drawLine(pixels, width, height, from, to, color, thickness = 2) {
|
|
|
661
682
|
}
|
|
662
683
|
}
|
|
663
684
|
}
|
|
664
|
-
function
|
|
665
|
-
const
|
|
685
|
+
function drawCallout(pixels, width, height, targetPoint, markerRadius, style) {
|
|
686
|
+
const cx = Math.round(targetPoint.x);
|
|
687
|
+
const cy = Math.round(targetPoint.y);
|
|
688
|
+
const gap = Math.max(markerRadius + 18, 30);
|
|
689
|
+
const candidates = [
|
|
690
|
+
{
|
|
691
|
+
x: cx + gap,
|
|
692
|
+
y: cy - gap
|
|
693
|
+
},
|
|
694
|
+
{
|
|
695
|
+
x: cx + gap,
|
|
696
|
+
y: cy + gap
|
|
697
|
+
},
|
|
698
|
+
{
|
|
699
|
+
x: cx - gap,
|
|
700
|
+
y: cy - gap
|
|
701
|
+
},
|
|
702
|
+
{
|
|
703
|
+
x: cx - gap,
|
|
704
|
+
y: cy + gap
|
|
705
|
+
},
|
|
706
|
+
{
|
|
707
|
+
x: cx,
|
|
708
|
+
y: cy - gap
|
|
709
|
+
},
|
|
710
|
+
{
|
|
711
|
+
x: cx,
|
|
712
|
+
y: cy + gap
|
|
713
|
+
},
|
|
714
|
+
{
|
|
715
|
+
x: cx + gap,
|
|
716
|
+
y: cy
|
|
717
|
+
},
|
|
718
|
+
{
|
|
719
|
+
x: cx - gap,
|
|
720
|
+
y: cy
|
|
721
|
+
}
|
|
722
|
+
];
|
|
723
|
+
const calloutCenter = candidates.find((candidate)=>candidate.x - markerRadius >= 0 && candidate.x + markerRadius < width && candidate.y - markerRadius >= 0 && candidate.y + markerRadius < height) || {
|
|
724
|
+
x: Math.min(Math.max(cx + gap, markerRadius), width - markerRadius - 1),
|
|
725
|
+
y: Math.min(Math.max(cy - gap, markerRadius), height - markerRadius - 1)
|
|
726
|
+
};
|
|
727
|
+
const lineAngle = Math.atan2(cy - calloutCenter.y, cx - calloutCenter.x);
|
|
728
|
+
const lineStart = {
|
|
729
|
+
x: calloutCenter.x + Math.cos(lineAngle) * (markerRadius + 1),
|
|
730
|
+
y: calloutCenter.y + Math.sin(lineAngle) * (markerRadius + 1)
|
|
731
|
+
};
|
|
732
|
+
const targetRadiusX = 30;
|
|
733
|
+
const targetRadiusY = 15;
|
|
734
|
+
const ellipseBoundaryDistance = 1 / Math.sqrt(Math.cos(lineAngle) ** 2 / targetRadiusX ** 2 + Math.sin(lineAngle) ** 2 / targetRadiusY ** 2);
|
|
735
|
+
const lineEnd = {
|
|
736
|
+
x: cx - Math.cos(lineAngle) * ellipseBoundaryDistance,
|
|
737
|
+
y: cy - Math.sin(lineAngle) * ellipseBoundaryDistance
|
|
738
|
+
};
|
|
739
|
+
drawLine(pixels, width, height, lineStart, lineEnd, style.callout, 2);
|
|
740
|
+
drawCircle(pixels, width, height, calloutCenter, markerRadius + 2, style.calloutBorder, {
|
|
741
|
+
thickness: 3
|
|
742
|
+
});
|
|
743
|
+
drawCircle(pixels, width, height, calloutCenter, markerRadius, style.callout, {
|
|
744
|
+
fill: true
|
|
745
|
+
});
|
|
746
|
+
drawEllipse(pixels, width, height, targetPoint, targetRadiusX, targetRadiusY, style.callout, 2);
|
|
747
|
+
const textWidth = getNumberWidth(style.indexId);
|
|
748
|
+
const textHeight = FONT_HEIGHT * FONT_SCALE;
|
|
749
|
+
drawNumber(pixels, width, height, style.indexId, Math.round(calloutCenter.x - textWidth / 2), Math.round(calloutCenter.y - textHeight / 2), style.calloutText);
|
|
750
|
+
}
|
|
751
|
+
function drawPointMarker(pixels, width, height, point, radius, indexId = 1) {
|
|
752
|
+
const markerColors = [
|
|
753
|
+
{
|
|
754
|
+
r: 0xc6,
|
|
755
|
+
g: 0x23,
|
|
756
|
+
b: 0x00,
|
|
757
|
+
a: 0xee
|
|
758
|
+
},
|
|
759
|
+
{
|
|
760
|
+
r: 0x00,
|
|
761
|
+
g: 0x00,
|
|
762
|
+
b: 0xff,
|
|
763
|
+
a: 0xee
|
|
764
|
+
}
|
|
765
|
+
];
|
|
766
|
+
const callout = markerColors[(indexId - 1) % markerColors.length];
|
|
767
|
+
const calloutBorder = {
|
|
666
768
|
r: 0xff,
|
|
667
|
-
g:
|
|
668
|
-
b:
|
|
769
|
+
g: 0xff,
|
|
770
|
+
b: 0xff,
|
|
669
771
|
a: 0xff
|
|
670
772
|
};
|
|
671
|
-
const
|
|
773
|
+
const calloutText = {
|
|
672
774
|
r: 0xff,
|
|
673
775
|
g: 0xff,
|
|
674
776
|
b: 0xff,
|
|
675
777
|
a: 0xff
|
|
676
778
|
};
|
|
677
|
-
const
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
drawCircle(pixels, width, height, point, markerRadius, red, {
|
|
684
|
-
thickness: 3
|
|
779
|
+
const markerRadius = Math.max(Math.round(radius), 10);
|
|
780
|
+
drawCallout(pixels, width, height, point, markerRadius, {
|
|
781
|
+
callout,
|
|
782
|
+
calloutBorder,
|
|
783
|
+
calloutText,
|
|
784
|
+
indexId
|
|
685
785
|
});
|
|
686
|
-
drawCircle(pixels, width, height, point, 3, red, {
|
|
687
|
-
fill: true
|
|
688
|
-
});
|
|
689
|
-
drawLine(pixels, width, height, {
|
|
690
|
-
x: cx - markerRadius - 4,
|
|
691
|
-
y: cy
|
|
692
|
-
}, {
|
|
693
|
-
x: cx - markerRadius + 2,
|
|
694
|
-
y: cy
|
|
695
|
-
}, red, 2);
|
|
696
|
-
drawLine(pixels, width, height, {
|
|
697
|
-
x: cx + markerRadius - 2,
|
|
698
|
-
y: cy
|
|
699
|
-
}, {
|
|
700
|
-
x: cx + markerRadius + 4,
|
|
701
|
-
y: cy
|
|
702
|
-
}, red, 2);
|
|
703
|
-
drawLine(pixels, width, height, {
|
|
704
|
-
x: cx,
|
|
705
|
-
y: cy - markerRadius - 4
|
|
706
|
-
}, {
|
|
707
|
-
x: cx,
|
|
708
|
-
y: cy - markerRadius + 2
|
|
709
|
-
}, red, 2);
|
|
710
|
-
drawLine(pixels, width, height, {
|
|
711
|
-
x: cx,
|
|
712
|
-
y: cy + markerRadius - 2
|
|
713
|
-
}, {
|
|
714
|
-
x: cx,
|
|
715
|
-
y: cy + markerRadius + 4
|
|
716
|
-
}, red, 2);
|
|
717
786
|
}
|
|
718
787
|
function blendPixels(basePixels, overlayPixels, width, height) {
|
|
719
788
|
const result = new Uint8Array(basePixels.length);
|
|
@@ -742,7 +811,7 @@ function blendPixels(basePixels, overlayPixels, width, height) {
|
|
|
742
811
|
}
|
|
743
812
|
return result;
|
|
744
813
|
}
|
|
745
|
-
const createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5, borderThickness = 2, prompt)=>{
|
|
814
|
+
const createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5, borderThickness = 2, prompt, centerPoint = false)=>{
|
|
746
815
|
const overlayPixels = new Uint8Array(imageWidth * imageHeight * 4);
|
|
747
816
|
const colors = [
|
|
748
817
|
{
|
|
@@ -846,6 +915,12 @@ const createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding =
|
|
|
846
915
|
h: paddedHeight
|
|
847
916
|
};
|
|
848
917
|
drawRect(overlayPixels, imageWidth, imageHeight, paddedRect, color.rect, borderThickness);
|
|
918
|
+
if (centerPoint) drawCircle(overlayPixels, imageWidth, imageHeight, {
|
|
919
|
+
x: element.rect.left + element.rect.width / 2,
|
|
920
|
+
y: element.rect.top + element.rect.height / 2
|
|
921
|
+
}, 4, color.rect, {
|
|
922
|
+
fill: true
|
|
923
|
+
});
|
|
849
924
|
const indexId = element.indexId;
|
|
850
925
|
if ('number' != typeof indexId) continue;
|
|
851
926
|
const textWidth = getNumberWidth(indexId);
|
|
@@ -910,7 +985,7 @@ const compositeElementInfoImg = async (options)=>{
|
|
|
910
985
|
const { elementsPositionInfo, prompt } = options;
|
|
911
986
|
try {
|
|
912
987
|
const basePixels = photonImage.get_raw_pixels();
|
|
913
|
-
const overlayPixels = await createSvgOverlay(elementsPositionInfo, width, height, options.annotationPadding, options.borderThickness, prompt);
|
|
988
|
+
const overlayPixels = await createSvgOverlay(elementsPositionInfo, width, height, options.annotationPadding, options.borderThickness, prompt, options.centerPoint);
|
|
914
989
|
const blendedPixels = blendPixels(basePixels, overlayPixels, width, height);
|
|
915
990
|
const resultImage = new PhotonImage(blendedPixels, width, height);
|
|
916
991
|
const base64 = await (0, external_transform_js_namespaceObject.photonToBase64)(resultImage, 90);
|
|
@@ -949,7 +1024,7 @@ const compositePointMarkerImg = async (options)=>{
|
|
|
949
1024
|
try {
|
|
950
1025
|
const basePixels = photonImage.get_raw_pixels();
|
|
951
1026
|
const overlayPixels = new Uint8Array(width * height * 4);
|
|
952
|
-
drawPointMarker(overlayPixels, width, height, options.point, options.radius ?? 14);
|
|
1027
|
+
drawPointMarker(overlayPixels, width, height, options.point, options.radius ?? 14, options.indexId ?? 1);
|
|
953
1028
|
const blendedPixels = blendPixels(basePixels, overlayPixels, width, height);
|
|
954
1029
|
const resultImage = new PhotonImage(blendedPixels, width, height);
|
|
955
1030
|
const base64 = await (0, external_transform_js_namespaceObject.photonToBase64)(resultImage, 90);
|
|
@@ -12,6 +12,7 @@ export declare const compositeElementInfoImg: (options: {
|
|
|
12
12
|
};
|
|
13
13
|
annotationPadding?: number;
|
|
14
14
|
borderThickness?: number;
|
|
15
|
+
centerPoint?: boolean;
|
|
15
16
|
prompt?: string;
|
|
16
17
|
}) => Promise<string>;
|
|
17
18
|
export declare const compositePointMarkerImg: (options: {
|
|
@@ -25,6 +26,7 @@ export declare const compositePointMarkerImg: (options: {
|
|
|
25
26
|
height: number;
|
|
26
27
|
};
|
|
27
28
|
radius?: number;
|
|
29
|
+
indexId?: number;
|
|
28
30
|
}) => Promise<string>;
|
|
29
31
|
export declare const processImageElementInfo: (options: {
|
|
30
32
|
inputImgBase64: string;
|
package/package.json
CHANGED
|
@@ -298,7 +298,7 @@ export abstract class BaseMidsceneTools<
|
|
|
298
298
|
this.toolDefaults,
|
|
299
299
|
);
|
|
300
300
|
|
|
301
|
-
// 4. Add common tools (screenshot,
|
|
301
|
+
// 4. Add common tools (screenshot, act, assert)
|
|
302
302
|
const commonTools = generateCommonTools(
|
|
303
303
|
(args = {}) => this.ensureAgent(this.extractAgentInitParam(args)),
|
|
304
304
|
this.getAgentInitArgSchema(),
|
package/src/img/box-select.ts
CHANGED
|
@@ -260,6 +260,48 @@ function drawCircle(
|
|
|
260
260
|
}
|
|
261
261
|
}
|
|
262
262
|
|
|
263
|
+
function drawEllipse(
|
|
264
|
+
pixels: Uint8Array,
|
|
265
|
+
width: number,
|
|
266
|
+
height: number,
|
|
267
|
+
center: { x: number; y: number },
|
|
268
|
+
radiusX: number,
|
|
269
|
+
radiusY: number,
|
|
270
|
+
color: { r: number; g: number; b: number; a: number },
|
|
271
|
+
thickness = 2,
|
|
272
|
+
) {
|
|
273
|
+
const cx = Math.round(center.x);
|
|
274
|
+
const cy = Math.round(center.y);
|
|
275
|
+
const outerRadiusX = Math.max(Math.round(radiusX), 1);
|
|
276
|
+
const outerRadiusY = Math.max(Math.round(radiusY), 1);
|
|
277
|
+
const innerRadiusX = Math.max(outerRadiusX - Math.round(thickness), 0);
|
|
278
|
+
const innerRadiusY = Math.max(outerRadiusY - Math.round(thickness), 0);
|
|
279
|
+
|
|
280
|
+
for (let py = cy - outerRadiusY; py <= cy + outerRadiusY; py++) {
|
|
281
|
+
for (let px = cx - outerRadiusX; px <= cx + outerRadiusX; px++) {
|
|
282
|
+
if (px < 0 || py < 0 || px >= width || py >= height) continue;
|
|
283
|
+
|
|
284
|
+
const dx = px - cx;
|
|
285
|
+
const dy = py - cy;
|
|
286
|
+
const outer =
|
|
287
|
+
(dx * dx) / (outerRadiusX * outerRadiusX) +
|
|
288
|
+
(dy * dy) / (outerRadiusY * outerRadiusY);
|
|
289
|
+
const inner =
|
|
290
|
+
innerRadiusX === 0 || innerRadiusY === 0
|
|
291
|
+
? 0
|
|
292
|
+
: (dx * dx) / (innerRadiusX * innerRadiusX) +
|
|
293
|
+
(dy * dy) / (innerRadiusY * innerRadiusY);
|
|
294
|
+
if (outer > 1 || inner < 1) continue;
|
|
295
|
+
|
|
296
|
+
const idx = (py * width + px) * 4;
|
|
297
|
+
pixels[idx + 0] = color.r;
|
|
298
|
+
pixels[idx + 1] = color.g;
|
|
299
|
+
pixels[idx + 2] = color.b;
|
|
300
|
+
pixels[idx + 3] = color.a;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
263
305
|
function drawLine(
|
|
264
306
|
pixels: Uint8Array,
|
|
265
307
|
width: number,
|
|
@@ -294,64 +336,132 @@ function drawLine(
|
|
|
294
336
|
}
|
|
295
337
|
}
|
|
296
338
|
|
|
297
|
-
function
|
|
339
|
+
function drawCallout(
|
|
298
340
|
pixels: Uint8Array,
|
|
299
341
|
width: number,
|
|
300
342
|
height: number,
|
|
301
|
-
|
|
302
|
-
|
|
343
|
+
targetPoint: { x: number; y: number },
|
|
344
|
+
markerRadius: number,
|
|
345
|
+
style: {
|
|
346
|
+
callout: { r: number; g: number; b: number; a: number };
|
|
347
|
+
calloutBorder: { r: number; g: number; b: number; a: number };
|
|
348
|
+
calloutText: { r: number; g: number; b: number; a: number };
|
|
349
|
+
indexId: number;
|
|
350
|
+
},
|
|
303
351
|
) {
|
|
304
|
-
const
|
|
305
|
-
const
|
|
306
|
-
const
|
|
307
|
-
const
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
352
|
+
const cx = Math.round(targetPoint.x);
|
|
353
|
+
const cy = Math.round(targetPoint.y);
|
|
354
|
+
const gap = Math.max(markerRadius + 18, 30);
|
|
355
|
+
const candidates = [
|
|
356
|
+
{ x: cx + gap, y: cy - gap },
|
|
357
|
+
{ x: cx + gap, y: cy + gap },
|
|
358
|
+
{ x: cx - gap, y: cy - gap },
|
|
359
|
+
{ x: cx - gap, y: cy + gap },
|
|
360
|
+
{ x: cx, y: cy - gap },
|
|
361
|
+
{ x: cx, y: cy + gap },
|
|
362
|
+
{ x: cx + gap, y: cy },
|
|
363
|
+
{ x: cx - gap, y: cy },
|
|
364
|
+
];
|
|
365
|
+
const calloutCenter = candidates.find(
|
|
366
|
+
(candidate) =>
|
|
367
|
+
candidate.x - markerRadius >= 0 &&
|
|
368
|
+
candidate.x + markerRadius < width &&
|
|
369
|
+
candidate.y - markerRadius >= 0 &&
|
|
370
|
+
candidate.y + markerRadius < height,
|
|
371
|
+
) || {
|
|
372
|
+
x: Math.min(Math.max(cx + gap, markerRadius), width - markerRadius - 1),
|
|
373
|
+
y: Math.min(Math.max(cy - gap, markerRadius), height - markerRadius - 1),
|
|
374
|
+
};
|
|
375
|
+
|
|
376
|
+
const lineAngle = Math.atan2(cy - calloutCenter.y, cx - calloutCenter.x);
|
|
377
|
+
const lineStart = {
|
|
378
|
+
x: calloutCenter.x + Math.cos(lineAngle) * (markerRadius + 1),
|
|
379
|
+
y: calloutCenter.y + Math.sin(lineAngle) * (markerRadius + 1),
|
|
380
|
+
};
|
|
381
|
+
|
|
382
|
+
const targetRadiusX = 30;
|
|
383
|
+
const targetRadiusY = 15;
|
|
384
|
+
const ellipseBoundaryDistance =
|
|
385
|
+
1 /
|
|
386
|
+
Math.sqrt(
|
|
387
|
+
Math.cos(lineAngle) ** 2 / targetRadiusX ** 2 +
|
|
388
|
+
Math.sin(lineAngle) ** 2 / targetRadiusY ** 2,
|
|
389
|
+
);
|
|
390
|
+
const lineEnd = {
|
|
391
|
+
x: cx - Math.cos(lineAngle) * ellipseBoundaryDistance,
|
|
392
|
+
y: cy - Math.sin(lineAngle) * ellipseBoundaryDistance,
|
|
393
|
+
};
|
|
394
|
+
|
|
395
|
+
drawLine(pixels, width, height, lineStart, lineEnd, style.callout, 2);
|
|
396
|
+
drawCircle(
|
|
318
397
|
pixels,
|
|
319
398
|
width,
|
|
320
399
|
height,
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
400
|
+
calloutCenter,
|
|
401
|
+
markerRadius + 2,
|
|
402
|
+
style.calloutBorder,
|
|
403
|
+
{
|
|
404
|
+
thickness: 3,
|
|
405
|
+
},
|
|
325
406
|
);
|
|
326
|
-
|
|
407
|
+
drawCircle(
|
|
327
408
|
pixels,
|
|
328
409
|
width,
|
|
329
410
|
height,
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
411
|
+
calloutCenter,
|
|
412
|
+
markerRadius,
|
|
413
|
+
style.callout,
|
|
414
|
+
{
|
|
415
|
+
fill: true,
|
|
416
|
+
},
|
|
334
417
|
);
|
|
335
|
-
|
|
418
|
+
drawEllipse(
|
|
336
419
|
pixels,
|
|
337
420
|
width,
|
|
338
421
|
height,
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
422
|
+
targetPoint,
|
|
423
|
+
targetRadiusX,
|
|
424
|
+
targetRadiusY,
|
|
425
|
+
style.callout,
|
|
342
426
|
2,
|
|
343
427
|
);
|
|
344
|
-
|
|
428
|
+
const textWidth = getNumberWidth(style.indexId);
|
|
429
|
+
const textHeight = FONT_HEIGHT * FONT_SCALE;
|
|
430
|
+
drawNumber(
|
|
345
431
|
pixels,
|
|
346
432
|
width,
|
|
347
433
|
height,
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
434
|
+
style.indexId,
|
|
435
|
+
Math.round(calloutCenter.x - textWidth / 2),
|
|
436
|
+
Math.round(calloutCenter.y - textHeight / 2),
|
|
437
|
+
style.calloutText,
|
|
352
438
|
);
|
|
353
439
|
}
|
|
354
440
|
|
|
441
|
+
function drawPointMarker(
|
|
442
|
+
pixels: Uint8Array,
|
|
443
|
+
width: number,
|
|
444
|
+
height: number,
|
|
445
|
+
point: { x: number; y: number },
|
|
446
|
+
radius: number,
|
|
447
|
+
indexId = 1,
|
|
448
|
+
) {
|
|
449
|
+
const markerColors = [
|
|
450
|
+
{ r: 0xc6, g: 0x23, b: 0x00, a: 0xee },
|
|
451
|
+
{ r: 0x00, g: 0x00, b: 0xff, a: 0xee },
|
|
452
|
+
];
|
|
453
|
+
const callout = markerColors[(indexId - 1) % markerColors.length];
|
|
454
|
+
const calloutBorder = { r: 0xff, g: 0xff, b: 0xff, a: 0xff };
|
|
455
|
+
const calloutText = { r: 0xff, g: 0xff, b: 0xff, a: 0xff };
|
|
456
|
+
const markerRadius = Math.max(Math.round(radius), 10);
|
|
457
|
+
drawCallout(pixels, width, height, point, markerRadius, {
|
|
458
|
+
callout,
|
|
459
|
+
calloutBorder,
|
|
460
|
+
calloutText,
|
|
461
|
+
indexId,
|
|
462
|
+
});
|
|
463
|
+
}
|
|
464
|
+
|
|
355
465
|
function blendPixels(
|
|
356
466
|
basePixels: Uint8Array,
|
|
357
467
|
overlayPixels: Uint8Array,
|
|
@@ -406,6 +516,7 @@ const createSvgOverlay = async (
|
|
|
406
516
|
boxPadding = 5,
|
|
407
517
|
borderThickness = 2,
|
|
408
518
|
prompt?: string,
|
|
519
|
+
centerPoint = false,
|
|
409
520
|
): Promise<Uint8Array> => {
|
|
410
521
|
// Create transparent overlay
|
|
411
522
|
const overlayPixels = new Uint8Array(imageWidth * imageHeight * 4);
|
|
@@ -490,6 +601,21 @@ const createSvgOverlay = async (
|
|
|
490
601
|
borderThickness,
|
|
491
602
|
);
|
|
492
603
|
|
|
604
|
+
if (centerPoint) {
|
|
605
|
+
drawCircle(
|
|
606
|
+
overlayPixels,
|
|
607
|
+
imageWidth,
|
|
608
|
+
imageHeight,
|
|
609
|
+
{
|
|
610
|
+
x: element.rect.left + element.rect.width / 2,
|
|
611
|
+
y: element.rect.top + element.rect.height / 2,
|
|
612
|
+
},
|
|
613
|
+
4,
|
|
614
|
+
color.rect,
|
|
615
|
+
{ fill: true },
|
|
616
|
+
);
|
|
617
|
+
}
|
|
618
|
+
|
|
493
619
|
// Calculate text position
|
|
494
620
|
const indexId = element.indexId;
|
|
495
621
|
if (typeof indexId !== 'number') {
|
|
@@ -595,6 +721,7 @@ export const compositeElementInfoImg = async (options: {
|
|
|
595
721
|
size?: { width: number; height: number };
|
|
596
722
|
annotationPadding?: number;
|
|
597
723
|
borderThickness?: number;
|
|
724
|
+
centerPoint?: boolean;
|
|
598
725
|
prompt?: string;
|
|
599
726
|
}) => {
|
|
600
727
|
assert(options.inputImgBase64, 'inputImgBase64 is required');
|
|
@@ -648,6 +775,7 @@ export const compositeElementInfoImg = async (options: {
|
|
|
648
775
|
options.annotationPadding,
|
|
649
776
|
options.borderThickness,
|
|
650
777
|
prompt,
|
|
778
|
+
options.centerPoint,
|
|
651
779
|
);
|
|
652
780
|
|
|
653
781
|
// Blend overlay onto base image
|
|
@@ -669,6 +797,7 @@ export const compositePointMarkerImg = async (options: {
|
|
|
669
797
|
point: { x: number; y: number };
|
|
670
798
|
size?: { width: number; height: number };
|
|
671
799
|
radius?: number;
|
|
800
|
+
indexId?: number;
|
|
672
801
|
}) => {
|
|
673
802
|
assert(options.inputImgBase64, 'inputImgBase64 is required');
|
|
674
803
|
const { PhotonImage, SamplingFilter, resize } = await getPhoton();
|
|
@@ -715,6 +844,7 @@ export const compositePointMarkerImg = async (options: {
|
|
|
715
844
|
height,
|
|
716
845
|
options.point,
|
|
717
846
|
options.radius ?? 14,
|
|
847
|
+
options.indexId ?? 1,
|
|
718
848
|
);
|
|
719
849
|
const blendedPixels = blendPixels(basePixels, overlayPixels, width, height);
|
|
720
850
|
const resultImage = new PhotonImage(blendedPixels, width, height);
|