@mauriciobenjamin700/ort-vision-sdk-web 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +128 -0
- package/LICENSE +21 -0
- package/README.md +90 -0
- package/dist/core/canvas.d.ts +21 -0
- package/dist/core/canvas.d.ts.map +1 -0
- package/dist/core/canvas.js +58 -0
- package/dist/core/canvas.js.map +1 -0
- package/dist/core/exceptions.d.ts +25 -0
- package/dist/core/exceptions.d.ts.map +1 -0
- package/dist/core/exceptions.js +28 -0
- package/dist/core/exceptions.js.map +1 -0
- package/dist/core/providers.d.ts +21 -0
- package/dist/core/providers.d.ts.map +1 -0
- package/dist/core/providers.js +29 -0
- package/dist/core/providers.js.map +1 -0
- package/dist/core/session.d.ts +47 -0
- package/dist/core/session.d.ts.map +1 -0
- package/dist/core/session.js +86 -0
- package/dist/core/session.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +21 -0
- package/dist/index.js.map +1 -0
- package/dist/io/image.d.ts +17 -0
- package/dist/io/image.d.ts.map +1 -0
- package/dist/io/image.js +104 -0
- package/dist/io/image.js.map +1 -0
- package/dist/labels.d.ts +38 -0
- package/dist/labels.d.ts.map +1 -0
- package/dist/labels.js +71 -0
- package/dist/labels.js.map +1 -0
- package/dist/postprocess/classification.d.ts +16 -0
- package/dist/postprocess/classification.d.ts.map +1 -0
- package/dist/postprocess/classification.js +46 -0
- package/dist/postprocess/classification.js.map +1 -0
- package/dist/postprocess/detection.d.ts +112 -0
- package/dist/postprocess/detection.d.ts.map +1 -0
- package/dist/postprocess/detection.js +283 -0
- package/dist/postprocess/detection.js.map +1 -0
- package/dist/postprocess/segmentation.d.ts +61 -0
- package/dist/postprocess/segmentation.d.ts.map +1 -0
- package/dist/postprocess/segmentation.js +184 -0
- package/dist/postprocess/segmentation.js.map +1 -0
- package/dist/preprocess/image.d.ts +71 -0
- package/dist/preprocess/image.d.ts.map +1 -0
- package/dist/preprocess/image.js +171 -0
- package/dist/preprocess/image.js.map +1 -0
- package/dist/results.d.ts +182 -0
- package/dist/results.d.ts.map +1 -0
- package/dist/results.js +321 -0
- package/dist/results.js.map +1 -0
- package/dist/tasks/base.d.ts +14 -0
- package/dist/tasks/base.d.ts.map +1 -0
- package/dist/tasks/base.js +17 -0
- package/dist/tasks/base.js.map +1 -0
- package/dist/tasks/classifier.d.ts +82 -0
- package/dist/tasks/classifier.d.ts.map +1 -0
- package/dist/tasks/classifier.js +135 -0
- package/dist/tasks/classifier.js.map +1 -0
- package/dist/tasks/detector.d.ts +102 -0
- package/dist/tasks/detector.d.ts.map +1 -0
- package/dist/tasks/detector.js +189 -0
- package/dist/tasks/detector.js.map +1 -0
- package/dist/tasks/segmenter.d.ts +105 -0
- package/dist/tasks/segmenter.d.ts.map +1 -0
- package/dist/tasks/segmenter.js +242 -0
- package/dist/tasks/segmenter.js.map +1 -0
- package/dist/types.d.ts +182 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +147 -0
- package/dist/types.js.map +1 -0
- package/package.json +71 -0
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Object detection task using anchor-free YOLO ONNX models (v8/v9/v10/v11/v12).
|
|
3
|
+
*/
|
|
4
|
+
import { type ModelSource, type OrtSessionOptions } from "../core/session.js";
|
|
5
|
+
import { type ImageInput } from "../io/image.js";
|
|
6
|
+
import { type LabelSpec } from "../labels.js";
|
|
7
|
+
import { DetectionResults } from "../results.js";
|
|
8
|
+
import { VisionTask } from "./base.js";
|
|
9
|
+
/**
|
|
10
|
+
* Decoder family for the model's detection head.
|
|
11
|
+
*
|
|
12
|
+
* - `"yolo"`: anchor-free YOLO head with output shape `[1, 4 + nc, N]` —
|
|
13
|
+
* covers YOLOv8, v9, v10, v11, v12, v26 detect exports.
|
|
14
|
+
*
|
|
15
|
+
* The SDK does **not** auto-detect the head from the model — the caller is
|
|
16
|
+
* responsible for picking a head that matches their export. Future families
|
|
17
|
+
* (v5/v6/v7 with `[1, N, 5+nc]`) will be added as new literal members.
|
|
18
|
+
*/
|
|
19
|
+
export type DetectorHead = "yolo";
|
|
20
|
+
export interface DetectorOptions extends OrtSessionOptions {
|
|
21
|
+
/**
|
|
22
|
+
* Decoder family for the detection head. Default `"yolo"` covers
|
|
23
|
+
* YOLOv8/v9/v10/v11/v12/v26.
|
|
24
|
+
*/
|
|
25
|
+
readonly head?: DetectorHead;
|
|
26
|
+
/** Class label spec — see {@link resolveLabels}. Defaults to the COCO 80-class preset. */
|
|
27
|
+
readonly labels?: LabelSpec;
|
|
28
|
+
/** Number of classes — used to validate the supplied labels. */
|
|
29
|
+
readonly numClasses?: number;
|
|
30
|
+
/** Model input `[width, height]` for letterboxing. Defaults to `[640, 640]`. */
|
|
31
|
+
readonly inputSize?: readonly [number, number];
|
|
32
|
+
/** Default minimum class score to keep a candidate. */
|
|
33
|
+
readonly confThreshold?: number;
|
|
34
|
+
/** Default IoU threshold for non-maximum suppression. */
|
|
35
|
+
readonly iouThreshold?: number;
|
|
36
|
+
/** Maximum number of detections per image. */
|
|
37
|
+
readonly maxDetections?: number;
|
|
38
|
+
}
|
|
39
|
+
export interface DetectorPredictOptions {
|
|
40
|
+
/** Override the default confidence threshold. */
|
|
41
|
+
readonly confThreshold?: number;
|
|
42
|
+
/** Override the default IoU threshold. */
|
|
43
|
+
readonly iouThreshold?: number;
|
|
44
|
+
/**
|
|
45
|
+
* If set, keep only detections whose `classId` is in this list.
|
|
46
|
+
* Mirrors Ultralytics' `model.predict(img, classes=[0, 16])`.
|
|
47
|
+
*/
|
|
48
|
+
readonly classes?: readonly number[];
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Object detector for anchor-free YOLO ONNX models (v8/v9/v10/v11/v12).
|
|
52
|
+
*
|
|
53
|
+
* `predict()` returns `Promise<DetectionResults[]>` (length 1 for a single
|
|
54
|
+
* image), mirroring Ultralytics' `YOLO("img.jpg")`. Iterate the envelope for
|
|
55
|
+
* per-instance dataclasses, or use the bulk `boxes` view (`.xyxy`, `.xywh`,
|
|
56
|
+
* `.xyxyn`, `.xywhn`, `.cls`, `.conf`).
|
|
57
|
+
*
|
|
58
|
+
* @example
|
|
59
|
+
* ```typescript
|
|
60
|
+
* const det = await Detector.create("/models/yolov8n.onnx");
|
|
61
|
+
* const results = await det.predict("/images/street.jpg");
|
|
62
|
+
* const r = results[0];
|
|
63
|
+
* console.log(r.boxes.xyxy, r.boxes.cls, r.boxes.conf, r.names);
|
|
64
|
+
* for (const d of r) {
|
|
65
|
+
* console.log(d.cls, d.conf, d.box.xyxy);
|
|
66
|
+
* }
|
|
67
|
+
* ```
|
|
68
|
+
*/
|
|
69
|
+
export declare class Detector extends VisionTask {
|
|
70
|
+
private readonly _head;
|
|
71
|
+
private readonly _labels;
|
|
72
|
+
private readonly _names;
|
|
73
|
+
private readonly _inputSize;
|
|
74
|
+
private readonly _confThreshold;
|
|
75
|
+
private readonly _iouThreshold;
|
|
76
|
+
private readonly _maxDetections;
|
|
77
|
+
private constructor();
|
|
78
|
+
/** Load the model and resolve labels. */
|
|
79
|
+
static create(model: ModelSource, options?: DetectorOptions): Promise<Detector>;
|
|
80
|
+
/** The decoder family used to interpret the model's output. */
|
|
81
|
+
get head(): DetectorHead;
|
|
82
|
+
/** Class labels indexed by class id. */
|
|
83
|
+
get labels(): readonly string[];
|
|
84
|
+
/** Class id → class name dict (matches Ultralytics' `model.names`). */
|
|
85
|
+
get names(): Readonly<Record<number, string>>;
|
|
86
|
+
/** Number of classes the model predicts. */
|
|
87
|
+
get numClasses(): number;
|
|
88
|
+
/**
|
|
89
|
+
* Alias for {@link predict} — call the detector like a torch `nn.Module`.
|
|
90
|
+
*
|
|
91
|
+
* Use as `det.call(img)` since JavaScript class instances are not callable;
|
|
92
|
+
* for direct invocation, prefer `det.predict(img)`. The full
|
|
93
|
+
* {@link DetectorPredictOptions} (including `classes`) is supported.
|
|
94
|
+
*/
|
|
95
|
+
call(image: ImageInput, options?: DetectorPredictOptions): Promise<DetectionResults[]>;
|
|
96
|
+
/** Run detection on a single image. */
|
|
97
|
+
predict(image: ImageInput, options?: DetectorPredictOptions): Promise<DetectionResults[]>;
|
|
98
|
+
private _preprocess;
|
|
99
|
+
private _buildResult;
|
|
100
|
+
private _buildBoxes;
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=detector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"detector.d.ts","sourceRoot":"","sources":["../../src/tasks/detector.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,iBAAiB,EAAc,MAAM,oBAAoB,CAAC;AAC1F,OAAO,EAAE,KAAK,UAAU,EAAa,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,KAAK,SAAS,EAAiB,MAAM,cAAc,CAAC;AAQ7D,OAAO,EAAS,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAOvC;;;;;;;;;GASG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAElC,MAAM,WAAW,eAAgB,SAAQ,iBAAiB;IACxD;;;OAGG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,CAAC;IAC7B,0FAA0F;IAC1F,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC;IAC5B,gEAAgE;IAChE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,gFAAgF;IAChF,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/C,uDAAuD;IACvD,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,yDAAyD;IACzD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,8CAA8C;IAC9C,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,sBAAsB;IACrC,iDAAiD;IACjD,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,0CAA0C;IAC1C,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B;;;OAGG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,QAAS,SAAQ,UAAU;IAGpC,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,cAAc;IARjC,OAAO;IAaP,yCAAyC;WAC5B,MAAM,CACjB,KAAK,EAAE,WAAW,EAClB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,QAAQ,CAAC;IAyBpB,+DAA+D;IAC/D,IAAI,IAAI,IAAI,YAAY,CAEvB;IAED,wCAAwC;IACxC,IAAI,MAAM,IAAI,SAAS,MAAM,EAAE,CAE9B;IAED,uEAAuE;IACvE,IAAI,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAE5C;IAED,4CAA4C;IAC5C,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;;;;OAMG;IACG,IAAI,CACR,KAAK,EAAE,UAAU,EACjB,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAI9B,uCAAuC;IACjC,OAAO,CACX,KAAK,EAAE,UAAU,EACjB,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAmD9B,OAAO,CAAC,WAAW;IAkBnB,OAAO,CAAC,YAAY;IA4CpB,OAAO,CAAC,WAAW;CAmBpB"}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Object detection task using anchor-free YOLO ONNX models (v8/v9/v10/v11/v12).
|
|
3
|
+
*/
|
|
4
|
+
import { OrtSession } from "../core/session.js";
|
|
5
|
+
import { loadImage } from "../io/image.js";
|
|
6
|
+
import { resolveLabels } from "../labels.js";
|
|
7
|
+
import { decodeYolo } from "../postprocess/detection.js";
|
|
8
|
+
import { letterbox, toCHW, toFloat32, toFloat32Tensor, } from "../preprocess/image.js";
|
|
9
|
+
import { Boxes, DetectionResults } from "../results.js";
|
|
10
|
+
import { VisionTask } from "./base.js";
|
|
11
|
+
import { RGBImage, } from "../types.js";
|
|
12
|
+
/**
|
|
13
|
+
* Object detector for anchor-free YOLO ONNX models (v8/v9/v10/v11/v12).
|
|
14
|
+
*
|
|
15
|
+
* `predict()` returns `Promise<DetectionResults[]>` (length 1 for a single
|
|
16
|
+
* image), mirroring Ultralytics' `YOLO("img.jpg")`. Iterate the envelope for
|
|
17
|
+
* per-instance dataclasses, or use the bulk `boxes` view (`.xyxy`, `.xywh`,
|
|
18
|
+
* `.xyxyn`, `.xywhn`, `.cls`, `.conf`).
|
|
19
|
+
*
|
|
20
|
+
* @example
|
|
21
|
+
* ```typescript
|
|
22
|
+
* const det = await Detector.create("/models/yolov8n.onnx");
|
|
23
|
+
* const results = await det.predict("/images/street.jpg");
|
|
24
|
+
* const r = results[0];
|
|
25
|
+
* console.log(r.boxes.xyxy, r.boxes.cls, r.boxes.conf, r.names);
|
|
26
|
+
* for (const d of r) {
|
|
27
|
+
* console.log(d.cls, d.conf, d.box.xyxy);
|
|
28
|
+
* }
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
export class Detector extends VisionTask {
|
|
32
|
+
_head;
|
|
33
|
+
_labels;
|
|
34
|
+
_names;
|
|
35
|
+
_inputSize;
|
|
36
|
+
_confThreshold;
|
|
37
|
+
_iouThreshold;
|
|
38
|
+
_maxDetections;
|
|
39
|
+
constructor(session, _head, _labels, _names, _inputSize, _confThreshold, _iouThreshold, _maxDetections) {
|
|
40
|
+
super(session);
|
|
41
|
+
this._head = _head;
|
|
42
|
+
this._labels = _labels;
|
|
43
|
+
this._names = _names;
|
|
44
|
+
this._inputSize = _inputSize;
|
|
45
|
+
this._confThreshold = _confThreshold;
|
|
46
|
+
this._iouThreshold = _iouThreshold;
|
|
47
|
+
this._maxDetections = _maxDetections;
|
|
48
|
+
}
|
|
49
|
+
/** Load the model and resolve labels. */
|
|
50
|
+
static async create(model, options = {}) {
|
|
51
|
+
const head = options.head ?? "yolo";
|
|
52
|
+
if (head !== "yolo") {
|
|
53
|
+
throw new Error(`Unsupported detector head '${head}'. Supported: 'yolo'.`);
|
|
54
|
+
}
|
|
55
|
+
const session = await OrtSession.create(model, options);
|
|
56
|
+
const labels = resolveLabels(options.labels ?? "coco", {
|
|
57
|
+
numClasses: options.numClasses,
|
|
58
|
+
});
|
|
59
|
+
const names = {};
|
|
60
|
+
for (let i = 0; i < labels.length; i++) {
|
|
61
|
+
names[i] = labels[i];
|
|
62
|
+
}
|
|
63
|
+
return new Detector(session, head, labels, names, options.inputSize ?? [640, 640], options.confThreshold ?? 0.25, options.iouThreshold ?? 0.45, options.maxDetections ?? 300);
|
|
64
|
+
}
|
|
65
|
+
/** The decoder family used to interpret the model's output. */
|
|
66
|
+
get head() {
|
|
67
|
+
return this._head;
|
|
68
|
+
}
|
|
69
|
+
/** Class labels indexed by class id. */
|
|
70
|
+
get labels() {
|
|
71
|
+
return this._labels;
|
|
72
|
+
}
|
|
73
|
+
/** Class id → class name dict (matches Ultralytics' `model.names`). */
|
|
74
|
+
get names() {
|
|
75
|
+
return this._names;
|
|
76
|
+
}
|
|
77
|
+
/** Number of classes the model predicts. */
|
|
78
|
+
get numClasses() {
|
|
79
|
+
return this._labels.length;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Alias for {@link predict} — call the detector like a torch `nn.Module`.
|
|
83
|
+
*
|
|
84
|
+
* Use as `det.call(img)` since JavaScript class instances are not callable;
|
|
85
|
+
* for direct invocation, prefer `det.predict(img)`. The full
|
|
86
|
+
* {@link DetectorPredictOptions} (including `classes`) is supported.
|
|
87
|
+
*/
|
|
88
|
+
async call(image, options = {}) {
|
|
89
|
+
return this.predict(image, options);
|
|
90
|
+
}
|
|
91
|
+
/** Run detection on a single image. */
|
|
92
|
+
async predict(image, options = {}) {
|
|
93
|
+
const path = typeof image === "string" ? image : null;
|
|
94
|
+
const original = await loadImage(image);
|
|
95
|
+
const { tensor, scale, padLeft, padTop } = this._preprocess(original);
|
|
96
|
+
const outputs = await this._session.run({ [this._session.inputName]: tensor });
|
|
97
|
+
const firstOutputName = this._session.outputNames[0];
|
|
98
|
+
if (firstOutputName === undefined) {
|
|
99
|
+
throw new Error("Detector model has no outputs.");
|
|
100
|
+
}
|
|
101
|
+
const raw = outputs[firstOutputName];
|
|
102
|
+
if (raw === undefined) {
|
|
103
|
+
throw new Error(`Detector model output ${firstOutputName} missing from run() result.`);
|
|
104
|
+
}
|
|
105
|
+
const decodedAll = decodeYolo(raw.data, raw.dims, {
|
|
106
|
+
originalWidth: original.width,
|
|
107
|
+
originalHeight: original.height,
|
|
108
|
+
padLeft,
|
|
109
|
+
padTop,
|
|
110
|
+
scale,
|
|
111
|
+
confThreshold: options.confThreshold ?? this._confThreshold,
|
|
112
|
+
iouThreshold: options.iouThreshold ?? this._iouThreshold,
|
|
113
|
+
maxDetections: this._maxDetections,
|
|
114
|
+
});
|
|
115
|
+
const decoded = options.classes !== undefined
|
|
116
|
+
? (() => {
|
|
117
|
+
const allowed = new Set(options.classes);
|
|
118
|
+
return decodedAll.filter((d) => allowed.has(d.classId));
|
|
119
|
+
})()
|
|
120
|
+
: decodedAll;
|
|
121
|
+
const detections = decoded.map((d) => this._buildResult(original, d.bbox, d.classId, d.confidence));
|
|
122
|
+
const orig = [original.height, original.width];
|
|
123
|
+
return [
|
|
124
|
+
new DetectionResults(this._buildBoxes(detections, orig), detections, this._names, original, orig, path),
|
|
125
|
+
];
|
|
126
|
+
}
|
|
127
|
+
_preprocess(image) {
|
|
128
|
+
const [tw, th] = this._inputSize;
|
|
129
|
+
const lb = letterbox(image, tw, th);
|
|
130
|
+
const f32 = toFloat32(lb.image);
|
|
131
|
+
const chw = toCHW(f32, lb.image.width, lb.image.height, 3);
|
|
132
|
+
return {
|
|
133
|
+
tensor: toFloat32Tensor(chw, [1, 3, lb.image.height, lb.image.width]),
|
|
134
|
+
scale: lb.scale,
|
|
135
|
+
padLeft: lb.padLeft,
|
|
136
|
+
padTop: lb.padTop,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
_buildResult(original, bbox, classId, confidence) {
|
|
140
|
+
const [x1, y1, x2, y2] = bbox.asIntXyxy();
|
|
141
|
+
const cx1 = Math.max(0, x1);
|
|
142
|
+
const cy1 = Math.max(0, y1);
|
|
143
|
+
const cx2 = Math.min(original.width, x2);
|
|
144
|
+
const cy2 = Math.min(original.height, y2);
|
|
145
|
+
let cropped;
|
|
146
|
+
if (cx2 > cx1 && cy2 > cy1) {
|
|
147
|
+
const cw = cx2 - cx1;
|
|
148
|
+
const ch = cy2 - cy1;
|
|
149
|
+
const out = new Uint8Array(cw * ch * 3);
|
|
150
|
+
for (let row = 0; row < ch; row++) {
|
|
151
|
+
const srcOffset = ((cy1 + row) * original.width + cx1) * 3;
|
|
152
|
+
out.set(original.data.subarray(srcOffset, srcOffset + cw * 3), row * cw * 3);
|
|
153
|
+
}
|
|
154
|
+
cropped = new RGBImage(out, cw, ch);
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
cropped = new RGBImage(new Uint8Array(0), 0, 0);
|
|
158
|
+
}
|
|
159
|
+
const className = this._names[classId] ?? `class_${classId}`;
|
|
160
|
+
return {
|
|
161
|
+
classId,
|
|
162
|
+
className,
|
|
163
|
+
confidence,
|
|
164
|
+
bbox,
|
|
165
|
+
cls: classId,
|
|
166
|
+
name: className,
|
|
167
|
+
conf: confidence,
|
|
168
|
+
box: bbox,
|
|
169
|
+
croppedImage: cropped,
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
_buildBoxes(detections, origShape) {
|
|
173
|
+
const n = detections.length;
|
|
174
|
+
const xyxy = new Float32Array(n * 4);
|
|
175
|
+
const cls = new Int32Array(n);
|
|
176
|
+
const conf = new Float32Array(n);
|
|
177
|
+
for (let i = 0; i < n; i++) {
|
|
178
|
+
const d = detections[i];
|
|
179
|
+
xyxy[i * 4] = d.bbox.x1;
|
|
180
|
+
xyxy[i * 4 + 1] = d.bbox.y1;
|
|
181
|
+
xyxy[i * 4 + 2] = d.bbox.x2;
|
|
182
|
+
xyxy[i * 4 + 3] = d.bbox.y2;
|
|
183
|
+
cls[i] = d.classId;
|
|
184
|
+
conf[i] = d.confidence;
|
|
185
|
+
}
|
|
186
|
+
return new Boxes(xyxy, cls, conf, origShape);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
//# sourceMappingURL=detector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"detector.js","sourceRoot":"","sources":["../../src/tasks/detector.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAA4C,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC1F,OAAO,EAAmB,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAkB,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EACL,SAAS,EACT,KAAK,EACL,SAAS,EACT,eAAe,GAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAGL,QAAQ,GACT,MAAM,aAAa,CAAC;AA8CrB;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,OAAO,QAAS,SAAQ,UAAU;IAGnB;IACA;IACA;IACA;IACA;IACA;IACA;IARnB,YACE,OAAmB,EACF,KAAmB,EACnB,OAA0B,EAC1B,MAAwC,EACxC,UAAqC,EACrC,cAAsB,EACtB,aAAqB,EACrB,cAAsB;QAEvC,KAAK,CAAC,OAAO,CAAC,CAAC;QARE,UAAK,GAAL,KAAK,CAAc;QACnB,YAAO,GAAP,OAAO,CAAmB;QAC1B,WAAM,GAAN,MAAM,CAAkC;QACxC,eAAU,GAAV,UAAU,CAA2B;QACrC,mBAAc,GAAd,cAAc,CAAQ;QACtB,kBAAa,GAAb,aAAa,CAAQ;QACrB,mBAAc,GAAd,cAAc,CAAQ;IAGzC,CAAC;IAED,yCAAyC;IACzC,MAAM,CAAC,KAAK,CAAC,MAAM,CACjB,KAAkB,EAClB,UAA2B,EAAE;QAE7B,MAAM,IAAI,GAAiB,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC;QAClD,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,uBAAuB,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,IAAI,MAAM,EAAE;YACrD,UAAU,EAAE,OAAO,CAAC,UAAU;SAC/B,CAAC,CAAC;QACH,MAAM,KAAK,GAA2B,EAAE,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAW,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,QAAQ,CACjB,OAAO,EACP,IAAI,EACJ,MAAM,EACN,KAAK,EACL,OAAO,CAAC,SAAS,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,EAC/B,OAAO,CAAC,aAAa,IAAI,IAAI,EAC7B,OAAO,CAAC,YAAY,IAAI,IAAI,EAC5B,OAAO,CAAC,aAAa,IAAI,GAAG,CAC7B,CAAC;IACJ,CAAC;IAED,+DAA+D;IAC/D,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,wCAAwC;IACxC,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,uEAAuE;IACvE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,4CAA4C;IAC5C,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,CACR,KAAiB,EACjB,UAAkC,EAAE;QAEpC,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACtC,CAAC;IAED,uCAAuC;IACvC,KAAK,CAAC,OAAO,CACX,KAAiB,EACjB,UAAkC,EAAE;QAEpC,MAAM,IAAI,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QACtD,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAE/E,MAAM,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACrD,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;QACrC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,yBAAyB,eAAe,6BAA6B,CAAC,CAAC;QACzF,CAAC;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,IAAoB,EAAE,GAAG,CAAC,IAAI,EAAE;YAChE,aAAa,EAAE,QAAQ,CAAC,KAAK;YAC7B,cAAc,EAAE,QAAQ,CAAC,MAAM;YAC/B,OAAO;YACP,MAAM;YACN,KAAK;YACL,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,cAAc;YAC3D,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,aAAa;YACxD,aAAa,EAAE,IAAI,CAAC,cAAc;SACnC,CAAC,CAAC;QAEH,MAAM,OAAO,GACX,OAAO,CAAC,OAAO,KAAK,SAAS;YAC3B,CAAC,CAAC,CAAC,GAAG,EAAE;gBACJ,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBACzC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAC1D,CAAC,CAAC,EAAE;YACN,CAAC,CAAC,UAAU,CAAC;QAEjB,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,UAAU,CAAC,CAC7D,CAAC;QAEF,MAAM,IAAI,GAA8B,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1E,OAAO;YACL,IAAI,gBAAgB,CAClB,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,IAAI,CAAC,EAClC,UAAU,EACV,IAAI,CAAC,MAAM,EACX,QAAQ,EACR,IAAI,EACJ,IAAI,CACL;SACF,CAAC;IACJ,CAAC;IAEO,WAAW,CAAC,KAAe;QAMjC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;QACjC,MAAM,EAAE,GAAG,SAAS,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QACpC,MAAM,GAAG,GAAG,SAAS,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3D,OAAO;YACL,MAAM,EAAE,eAAe,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrE,KAAK,EAAE,EAAE,CAAC,KAAK;YACf,OAAO,EAAE,EAAE,CAAC,OAAO;YACnB,MAAM,EAAE,EAAE,CAAC,MAAM;SAClB,CAAC;IACJ,CAAC;IAEO,YAAY,CAClB,QAAkB,EAClB,IAAiB,EACjB,OAAe,EACf,UAAkB;QAElB,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACzC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAE1C,IAAI,OAAiB,CAAC;QACtB,IAAI,GAAG,GAAG,GAAG,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,CAAC;YACrB,MAAM,EAAE,GAAG,GAAG,GAAG,GAAG,CAAC;YACrB,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YACxC,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,QAAQ,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC3D,GAAG,CAAC,GAAG,CACL,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,GAAG,EAAE,GAAG,CAAC,CAAC,EACrD,GAAG,GAAG,EAAE,GAAG,CAAC,CACb,CAAC;YACJ,CAAC;YACD,OAAO,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QACtC,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,IAAI,QAAQ,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,SAAS,OAAO,EAAE,CAAC;QAE7D,OAAO;YACL,OAAO;YACP,SAAS;YACT,UAAU;YACV,IAAI;YACJ,GAAG,EAAE,OAAO;YACZ,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,UAAU;YAChB,GAAG,EAAE,IAAI;YACT,YAAY,EAAE,OAAO;SACtB,CAAC;IACJ,CAAC;IAEO,WAAW,CACjB,UAAsC,EACtC,SAAoC;QAEpC,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAoB,CAAC;YAC3C,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;YACnB,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC;QACzB,CAAC;QACD,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;IAC/C,CAAC;CACF"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Instance-segmentation task using YOLO seg ONNX models (v8-seg / v11-seg / ...).
|
|
3
|
+
*/
|
|
4
|
+
import { type ModelSource, type OrtSessionOptions } from "../core/session.js";
|
|
5
|
+
import { type ImageInput } from "../io/image.js";
|
|
6
|
+
import { type LabelSpec } from "../labels.js";
|
|
7
|
+
import { SegmentationResults } from "../results.js";
|
|
8
|
+
import { VisionTask } from "./base.js";
|
|
9
|
+
/**
|
|
10
|
+
* Decoder family for the segmentation head.
|
|
11
|
+
*
|
|
12
|
+
* - `"yolo-seg"`: YOLO instance-segmentation head with two outputs —
|
|
13
|
+
* `[1, 4 + nc + nm, N]` per-anchor predictions plus `[1, nm, mh, mw]`
|
|
14
|
+
* prototype masks. Covers YOLOv8-seg, v11-seg, v26-seg.
|
|
15
|
+
*
|
|
16
|
+
* The SDK does **not** auto-detect this — the caller is responsible for
|
|
17
|
+
* picking a head that matches their export.
|
|
18
|
+
*/
|
|
19
|
+
export type SegmenterHead = "yolo-seg";
|
|
20
|
+
export interface SegmenterOptions extends OrtSessionOptions {
|
|
21
|
+
/**
|
|
22
|
+
* Decoder family for the segmentation head. Default `"yolo-seg"` covers
|
|
23
|
+
* YOLOv8-seg/v11-seg/v26-seg.
|
|
24
|
+
*/
|
|
25
|
+
readonly head?: SegmenterHead;
|
|
26
|
+
/** Class label spec — see {@link resolveLabels}. Defaults to the COCO 80-class preset. */
|
|
27
|
+
readonly labels?: LabelSpec;
|
|
28
|
+
/** Number of classes — used to validate the supplied labels. */
|
|
29
|
+
readonly numClasses?: number;
|
|
30
|
+
/** Model input `[width, height]` for letterboxing. Defaults to `[640, 640]`. */
|
|
31
|
+
readonly inputSize?: readonly [number, number];
|
|
32
|
+
/** Default minimum class score to keep a candidate. */
|
|
33
|
+
readonly confThreshold?: number;
|
|
34
|
+
/** Default IoU threshold for non-maximum suppression. */
|
|
35
|
+
readonly iouThreshold?: number;
|
|
36
|
+
/** Maximum number of instances per image. */
|
|
37
|
+
readonly maxDetections?: number;
|
|
38
|
+
/** Probability cutoff applied to soft masks. Defaults to `0.5`. */
|
|
39
|
+
readonly maskThreshold?: number;
|
|
40
|
+
}
|
|
41
|
+
export interface SegmenterPredictOptions {
|
|
42
|
+
readonly confThreshold?: number;
|
|
43
|
+
readonly iouThreshold?: number;
|
|
44
|
+
/**
|
|
45
|
+
* If set, keep only instances whose `classId` is in this list.
|
|
46
|
+
* Mirrors Ultralytics' `model.predict(img, classes=[0, 16])`.
|
|
47
|
+
*/
|
|
48
|
+
readonly classes?: readonly number[];
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Instance segmenter for YOLO seg ONNX models (v8-seg / v11-seg / ...).
|
|
52
|
+
*
|
|
53
|
+
* The model is expected to expose two outputs:
|
|
54
|
+
*
|
|
55
|
+
* 1. `output0`: `(1, 4 + numClasses + numMaskCoefs, numAnchors)` — per-anchor
|
|
56
|
+
* predictions (boxes, class scores, mask coefficients).
|
|
57
|
+
* 2. `output1`: `(1, numMaskCoefs, maskH, maskW)` — prototype masks.
|
|
58
|
+
*
|
|
59
|
+
* `predict()` returns `Promise<SegmentationResults[]>` (length 1 for a
|
|
60
|
+
* single image), mirroring Ultralytics' API. The envelope exposes:
|
|
61
|
+
*
|
|
62
|
+
* - `boxes`: bulk numpy view (`xyxy`, `xywh`, `xyxyn`, `xywhn`, `cls`, `conf`).
|
|
63
|
+
* - `masks`: per-instance binary masks cropped to each box.
|
|
64
|
+
* - per-instance {@link SegmentationResult} via iteration.
|
|
65
|
+
*
|
|
66
|
+
* @example
|
|
67
|
+
* ```typescript
|
|
68
|
+
* const seg = await Segmenter.create("/models/yolov8n-seg.onnx");
|
|
69
|
+
* const r = (await seg.predict("/images/street.jpg"))[0];
|
|
70
|
+
* for (const inst of r) {
|
|
71
|
+
* console.log(inst.cls, inst.conf, inst.box.xyxy);
|
|
72
|
+
* }
|
|
73
|
+
* ```
|
|
74
|
+
*/
|
|
75
|
+
export declare class Segmenter extends VisionTask {
|
|
76
|
+
private readonly _head;
|
|
77
|
+
private readonly _labels;
|
|
78
|
+
private readonly _names;
|
|
79
|
+
private readonly _inputSize;
|
|
80
|
+
private readonly _confThreshold;
|
|
81
|
+
private readonly _iouThreshold;
|
|
82
|
+
private readonly _maxDetections;
|
|
83
|
+
private readonly _maskThreshold;
|
|
84
|
+
private constructor();
|
|
85
|
+
/** Load the model and resolve labels. */
|
|
86
|
+
static create(model: ModelSource, options?: SegmenterOptions): Promise<Segmenter>;
|
|
87
|
+
/** The decoder family used to interpret the model's output. */
|
|
88
|
+
get head(): SegmenterHead;
|
|
89
|
+
/** Class labels indexed by class id. */
|
|
90
|
+
get labels(): readonly string[];
|
|
91
|
+
/** Class id → class name dict (matches Ultralytics' `model.names`). */
|
|
92
|
+
get names(): Readonly<Record<number, string>>;
|
|
93
|
+
/** Number of classes the model predicts. */
|
|
94
|
+
get numClasses(): number;
|
|
95
|
+
/** Alias for {@link predict} (parity with PyTorch `nn.Module.__call__`). */
|
|
96
|
+
call(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
|
|
97
|
+
/** Run instance segmentation on a single image. */
|
|
98
|
+
predict(image: ImageInput, options?: SegmenterPredictOptions): Promise<SegmentationResults[]>;
|
|
99
|
+
private _preprocess;
|
|
100
|
+
private _splitOutputs;
|
|
101
|
+
private _buildResult;
|
|
102
|
+
private _buildBoxes;
|
|
103
|
+
private _buildMasks;
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=segmenter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"segmenter.d.ts","sourceRoot":"","sources":["../../src/tasks/segmenter.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,iBAAiB,EAEvB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,KAAK,UAAU,EAAa,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,KAAK,SAAS,EAAiB,MAAM,cAAc,CAAC;AAQ7D,OAAO,EAAgB,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAQvC;;;;;;;;;GASG;AACH,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC,MAAM,WAAW,gBAAiB,SAAQ,iBAAiB;IACzD;;;OAGG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,CAAC;IAC9B,0FAA0F;IAC1F,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC;IAC5B,gEAAgE;IAChE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,gFAAgF;IAChF,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/C,uDAAuD;IACvD,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,yDAAyD;IACzD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,6CAA6C;IAC7C,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,mEAAmE;IACnE,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B;;;OAGG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,SAAU,SAAQ,UAAU;IAGrC,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,cAAc;IATjC,OAAO;IAcP,yCAAyC;WAC5B,MAAM,CACjB,KAAK,EAAE,WAAW,EAClB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,SAAS,CAAC;IA0BrB,+DAA+D;IAC/D,IAAI,IAAI,IAAI,aAAa,CAExB;IAED,wCAAwC;IACxC,IAAI,MAAM,IAAI,SAAS,MAAM,EAAE,CAE9B;IAED,uEAAuE;IACvE,IAAI,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAE5C;IAED,4CAA4C;IAC5C,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,4EAA4E;IACtE,IAAI,CACR,KAAK,EAAE,UAAU,EACjB,OAAO,GAAE,uBAA4B,GACpC,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAIjC,mDAAmD;IAC7C,OAAO,CACX,KAAK,EAAE,UAAU,EACjB,OAAO,GAAE,uBAA4B,GACpC,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAuDjC,OAAO,CAAC,WAAW;IAkBnB,OAAO,CAAC,aAAa;IAyBrB,OAAO,CAAC,YAAY;IAoEpB,OAAO,CAAC,WAAW;IAoBnB,OAAO,CAAC,WAAW;CAkBpB"}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Instance-segmentation task using YOLO seg ONNX models (v8-seg / v11-seg / ...).
|
|
3
|
+
*/
|
|
4
|
+
import { OrtSession, } from "../core/session.js";
|
|
5
|
+
import { loadImage } from "../io/image.js";
|
|
6
|
+
import { resolveLabels } from "../labels.js";
|
|
7
|
+
import { decodeYoloSeg } from "../postprocess/segmentation.js";
|
|
8
|
+
import { letterbox, toCHW, toFloat32, toFloat32Tensor, } from "../preprocess/image.js";
|
|
9
|
+
import { Boxes, Masks, SegmentationResults } from "../results.js";
|
|
10
|
+
import { VisionTask } from "./base.js";
|
|
11
|
+
import { Mask, RGBImage, } from "../types.js";
|
|
12
|
+
/**
|
|
13
|
+
* Instance segmenter for YOLO seg ONNX models (v8-seg / v11-seg / ...).
|
|
14
|
+
*
|
|
15
|
+
* The model is expected to expose two outputs:
|
|
16
|
+
*
|
|
17
|
+
* 1. `output0`: `(1, 4 + numClasses + numMaskCoefs, numAnchors)` — per-anchor
|
|
18
|
+
* predictions (boxes, class scores, mask coefficients).
|
|
19
|
+
* 2. `output1`: `(1, numMaskCoefs, maskH, maskW)` — prototype masks.
|
|
20
|
+
*
|
|
21
|
+
* `predict()` returns `Promise<SegmentationResults[]>` (length 1 for a
|
|
22
|
+
* single image), mirroring Ultralytics' API. The envelope exposes:
|
|
23
|
+
*
|
|
24
|
+
* - `boxes`: bulk numpy view (`xyxy`, `xywh`, `xyxyn`, `xywhn`, `cls`, `conf`).
|
|
25
|
+
* - `masks`: per-instance binary masks cropped to each box.
|
|
26
|
+
* - per-instance {@link SegmentationResult} via iteration.
|
|
27
|
+
*
|
|
28
|
+
* @example
|
|
29
|
+
* ```typescript
|
|
30
|
+
* const seg = await Segmenter.create("/models/yolov8n-seg.onnx");
|
|
31
|
+
* const r = (await seg.predict("/images/street.jpg"))[0];
|
|
32
|
+
* for (const inst of r) {
|
|
33
|
+
* console.log(inst.cls, inst.conf, inst.box.xyxy);
|
|
34
|
+
* }
|
|
35
|
+
* ```
|
|
36
|
+
*/
|
|
37
|
+
export class Segmenter extends VisionTask {
|
|
38
|
+
_head;
|
|
39
|
+
_labels;
|
|
40
|
+
_names;
|
|
41
|
+
_inputSize;
|
|
42
|
+
_confThreshold;
|
|
43
|
+
_iouThreshold;
|
|
44
|
+
_maxDetections;
|
|
45
|
+
_maskThreshold;
|
|
46
|
+
constructor(session, _head, _labels, _names, _inputSize, _confThreshold, _iouThreshold, _maxDetections, _maskThreshold) {
|
|
47
|
+
super(session);
|
|
48
|
+
this._head = _head;
|
|
49
|
+
this._labels = _labels;
|
|
50
|
+
this._names = _names;
|
|
51
|
+
this._inputSize = _inputSize;
|
|
52
|
+
this._confThreshold = _confThreshold;
|
|
53
|
+
this._iouThreshold = _iouThreshold;
|
|
54
|
+
this._maxDetections = _maxDetections;
|
|
55
|
+
this._maskThreshold = _maskThreshold;
|
|
56
|
+
}
|
|
57
|
+
/** Load the model and resolve labels. */
|
|
58
|
+
static async create(model, options = {}) {
|
|
59
|
+
const head = options.head ?? "yolo-seg";
|
|
60
|
+
if (head !== "yolo-seg") {
|
|
61
|
+
throw new Error(`Unsupported segmenter head '${head}'. Supported: 'yolo-seg'.`);
|
|
62
|
+
}
|
|
63
|
+
const session = await OrtSession.create(model, options);
|
|
64
|
+
const labels = resolveLabels(options.labels ?? "coco", {
|
|
65
|
+
numClasses: options.numClasses,
|
|
66
|
+
});
|
|
67
|
+
const names = {};
|
|
68
|
+
for (let i = 0; i < labels.length; i++) {
|
|
69
|
+
names[i] = labels[i];
|
|
70
|
+
}
|
|
71
|
+
return new Segmenter(session, head, labels, names, options.inputSize ?? [640, 640], options.confThreshold ?? 0.25, options.iouThreshold ?? 0.45, options.maxDetections ?? 300, options.maskThreshold ?? 0.5);
|
|
72
|
+
}
|
|
73
|
+
/** The decoder family used to interpret the model's output. */
|
|
74
|
+
get head() {
|
|
75
|
+
return this._head;
|
|
76
|
+
}
|
|
77
|
+
/** Class labels indexed by class id. */
|
|
78
|
+
get labels() {
|
|
79
|
+
return this._labels;
|
|
80
|
+
}
|
|
81
|
+
/** Class id → class name dict (matches Ultralytics' `model.names`). */
|
|
82
|
+
get names() {
|
|
83
|
+
return this._names;
|
|
84
|
+
}
|
|
85
|
+
/** Number of classes the model predicts. */
|
|
86
|
+
get numClasses() {
|
|
87
|
+
return this._labels.length;
|
|
88
|
+
}
|
|
89
|
+
/** Alias for {@link predict} (parity with PyTorch `nn.Module.__call__`). */
|
|
90
|
+
async call(image, options = {}) {
|
|
91
|
+
return this.predict(image, options);
|
|
92
|
+
}
|
|
93
|
+
/** Run instance segmentation on a single image. */
|
|
94
|
+
async predict(image, options = {}) {
|
|
95
|
+
const path = typeof image === "string" ? image : null;
|
|
96
|
+
const original = await loadImage(image);
|
|
97
|
+
const { tensor, scale, padLeft, padTop } = this._preprocess(original);
|
|
98
|
+
const outputs = await this._session.run({ [this._session.inputName]: tensor });
|
|
99
|
+
const { perAnchor, prototypes } = this._splitOutputs(outputs);
|
|
100
|
+
const decodedAll = decodeYoloSeg(perAnchor.data, perAnchor.dims, prototypes.data, prototypes.dims, {
|
|
101
|
+
numClasses: this._labels.length,
|
|
102
|
+
inputWidth: this._inputSize[0],
|
|
103
|
+
inputHeight: this._inputSize[1],
|
|
104
|
+
originalWidth: original.width,
|
|
105
|
+
originalHeight: original.height,
|
|
106
|
+
padLeft,
|
|
107
|
+
padTop,
|
|
108
|
+
scale,
|
|
109
|
+
confThreshold: options.confThreshold ?? this._confThreshold,
|
|
110
|
+
iouThreshold: options.iouThreshold ?? this._iouThreshold,
|
|
111
|
+
maxDetections: this._maxDetections,
|
|
112
|
+
maskThreshold: this._maskThreshold,
|
|
113
|
+
});
|
|
114
|
+
const decoded = options.classes !== undefined
|
|
115
|
+
? (() => {
|
|
116
|
+
const allowed = new Set(options.classes);
|
|
117
|
+
return decodedAll.filter((d) => allowed.has(d.classId));
|
|
118
|
+
})()
|
|
119
|
+
: decodedAll;
|
|
120
|
+
const detections = decoded.map((d) => this._buildResult(original, d.bbox, d.classId, d.confidence, d.mask));
|
|
121
|
+
const orig = [original.height, original.width];
|
|
122
|
+
return [
|
|
123
|
+
new SegmentationResults(this._buildBoxes(detections, orig), this._buildMasks(detections, orig), detections, this._names, original, orig, path),
|
|
124
|
+
];
|
|
125
|
+
}
|
|
126
|
+
_preprocess(image) {
|
|
127
|
+
const [tw, th] = this._inputSize;
|
|
128
|
+
const lb = letterbox(image, tw, th);
|
|
129
|
+
const f32 = toFloat32(lb.image);
|
|
130
|
+
const chw = toCHW(f32, lb.image.width, lb.image.height, 3);
|
|
131
|
+
return {
|
|
132
|
+
tensor: toFloat32Tensor(chw, [1, 3, lb.image.height, lb.image.width]),
|
|
133
|
+
scale: lb.scale,
|
|
134
|
+
padLeft: lb.padLeft,
|
|
135
|
+
padTop: lb.padTop,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
_splitOutputs(outputs) {
|
|
139
|
+
let perAnchor;
|
|
140
|
+
let prototypes;
|
|
141
|
+
for (const name of this._session.outputNames) {
|
|
142
|
+
const t = outputs[name];
|
|
143
|
+
if (t === undefined)
|
|
144
|
+
continue;
|
|
145
|
+
if (t.dims.length === 3 && perAnchor === undefined) {
|
|
146
|
+
perAnchor = t;
|
|
147
|
+
}
|
|
148
|
+
else if (t.dims.length === 4 && prototypes === undefined) {
|
|
149
|
+
prototypes = t;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (perAnchor === undefined || prototypes === undefined) {
|
|
153
|
+
const shapes = this._session.outputNames.map((n) => `${n}: ${JSON.stringify(outputs[n]?.dims ?? [])}`);
|
|
154
|
+
throw new Error(`Segmenter expected one 3-D and one 4-D output, got [${shapes.join(", ")}].`);
|
|
155
|
+
}
|
|
156
|
+
return { perAnchor, prototypes };
|
|
157
|
+
}
|
|
158
|
+
_buildResult(original, bbox, classId, confidence, mask) {
|
|
159
|
+
const [x1, y1, x2, y2] = bbox.asIntXyxy();
|
|
160
|
+
const cx1 = Math.max(0, x1);
|
|
161
|
+
const cy1 = Math.max(0, y1);
|
|
162
|
+
const cx2 = Math.min(original.width, x2);
|
|
163
|
+
const cy2 = Math.min(original.height, y2);
|
|
164
|
+
let segmentedImage;
|
|
165
|
+
let finalMask = mask;
|
|
166
|
+
if (cx2 > cx1 && cy2 > cy1 && mask.data.length > 0) {
|
|
167
|
+
const cropW = cx2 - cx1;
|
|
168
|
+
const cropH = cy2 - cy1;
|
|
169
|
+
const mw = Math.min(mask.width, cropW);
|
|
170
|
+
const mh = Math.min(mask.height, cropH);
|
|
171
|
+
const segData = new Uint8Array(mw * mh * 3);
|
|
172
|
+
for (let row = 0; row < mh; row++) {
|
|
173
|
+
const srcRowOffset = ((cy1 + row) * original.width + cx1) * 3;
|
|
174
|
+
const dstRowOffset = row * mw * 3;
|
|
175
|
+
const maskRowOffset = row * mask.width;
|
|
176
|
+
for (let col = 0; col < mw; col++) {
|
|
177
|
+
const m = mask.data[maskRowOffset + col];
|
|
178
|
+
if (m !== 0) {
|
|
179
|
+
const s = srcRowOffset + col * 3;
|
|
180
|
+
const d = dstRowOffset + col * 3;
|
|
181
|
+
segData[d] = original.data[s];
|
|
182
|
+
segData[d + 1] = original.data[s + 1];
|
|
183
|
+
segData[d + 2] = original.data[s + 2];
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
segmentedImage = new RGBImage(segData, mw, mh);
|
|
188
|
+
if (mw !== mask.width || mh !== mask.height) {
|
|
189
|
+
const trimmed = new Uint8Array(mw * mh);
|
|
190
|
+
for (let row = 0; row < mh; row++) {
|
|
191
|
+
trimmed.set(mask.data.subarray(row * mask.width, row * mask.width + mw), row * mw);
|
|
192
|
+
}
|
|
193
|
+
finalMask = new Mask(trimmed, mw, mh);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
else {
|
|
197
|
+
finalMask = new Mask(new Uint8Array(0), 0, 0);
|
|
198
|
+
segmentedImage = new RGBImage(new Uint8Array(0), 0, 0);
|
|
199
|
+
}
|
|
200
|
+
const className = this._names[classId] ?? `class_${classId}`;
|
|
201
|
+
return {
|
|
202
|
+
classId,
|
|
203
|
+
className,
|
|
204
|
+
confidence,
|
|
205
|
+
bbox,
|
|
206
|
+
cls: classId,
|
|
207
|
+
name: className,
|
|
208
|
+
conf: confidence,
|
|
209
|
+
box: bbox,
|
|
210
|
+
mask: finalMask,
|
|
211
|
+
segmentedImage,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
_buildBoxes(detections, origShape) {
|
|
215
|
+
const n = detections.length;
|
|
216
|
+
const xyxy = new Float32Array(n * 4);
|
|
217
|
+
const cls = new Int32Array(n);
|
|
218
|
+
const conf = new Float32Array(n);
|
|
219
|
+
for (let i = 0; i < n; i++) {
|
|
220
|
+
const d = detections[i];
|
|
221
|
+
xyxy[i * 4] = d.bbox.x1;
|
|
222
|
+
xyxy[i * 4 + 1] = d.bbox.y1;
|
|
223
|
+
xyxy[i * 4 + 2] = d.bbox.x2;
|
|
224
|
+
xyxy[i * 4 + 3] = d.bbox.y2;
|
|
225
|
+
cls[i] = d.classId;
|
|
226
|
+
conf[i] = d.confidence;
|
|
227
|
+
}
|
|
228
|
+
return new Boxes(xyxy, cls, conf, origShape);
|
|
229
|
+
}
|
|
230
|
+
_buildMasks(detections, origShape) {
|
|
231
|
+
const xyxy = new Float32Array(detections.length * 4);
|
|
232
|
+
for (let i = 0; i < detections.length; i++) {
|
|
233
|
+
const d = detections[i];
|
|
234
|
+
xyxy[i * 4] = d.bbox.x1;
|
|
235
|
+
xyxy[i * 4 + 1] = d.bbox.y1;
|
|
236
|
+
xyxy[i * 4 + 2] = d.bbox.x2;
|
|
237
|
+
xyxy[i * 4 + 3] = d.bbox.y2;
|
|
238
|
+
}
|
|
239
|
+
return new Masks(detections.map((d) => d.mask), xyxy, origShape);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
//# sourceMappingURL=segmenter.js.map
|