@eyepop.ai/eyepop 3.19.1 → 3.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/eyepop.index.d.mts +386 -15
- package/dist/eyepop.index.d.ts +386 -15
- package/dist/eyepop.index.js +300 -5
- package/dist/eyepop.index.js.map +1 -1
- package/dist/eyepop.index.mjs +291 -4
- package/dist/eyepop.index.mjs.map +1 -1
- package/dist/eyepop.min.js +1 -1
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -302,6 +302,6 @@ Rendering helpers are provided by [@eyepop.ai/eyepop-render-2d](https://www.npmj
|
|
|
302
302
|
|
|
303
303
|
`EyePop.dataEndpoint()` provides preview support for dataset management and model optimization workflows. This API is experimental and subject to change.
|
|
304
304
|
|
|
305
|
-
## Composable Pops
|
|
305
|
+
## Composable Pops
|
|
306
306
|
|
|
307
|
-
See [Composable Pops](composable-pops.md) for
|
|
307
|
+
See [Composable Pops](composable-pops.md) for client-side Pop composition.
|
package/dist/eyepop.index.d.mts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Logger } from 'pino';
|
|
2
|
+
import { OpenAPIV3_1 } from 'openapi-types';
|
|
2
3
|
|
|
3
4
|
interface Session {
|
|
4
5
|
readonly eyepopUrl: string;
|
|
@@ -69,7 +70,7 @@ interface Prediction extends StreamTime {
|
|
|
69
70
|
meshs?: Array<PredictedMesh>;
|
|
70
71
|
keyPoints?: Array<PredictedKeyPoints>;
|
|
71
72
|
embeddings?: Array<PredictedEmbedding>;
|
|
72
|
-
details?: Array<
|
|
73
|
+
details?: Array<Record<string, any>>;
|
|
73
74
|
motions?: Array<PredictedMotion>;
|
|
74
75
|
depth?: Depth;
|
|
75
76
|
}
|
|
@@ -95,11 +96,20 @@ interface Contour {
|
|
|
95
96
|
points: Array<Point2d>;
|
|
96
97
|
cutouts: Array<Array<Point2d>>;
|
|
97
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* A segmentation mask, optionally with a per-object point cloud.
|
|
101
|
+
*
|
|
102
|
+
* `world` is the base64 encoding of three little-endian float32 values per mask
|
|
103
|
+
* pixel, row-major, exactly width*height triples - so the point for bitmap
|
|
104
|
+
* pixel (i, j) is at triple index j * width + i. Points the worker could not
|
|
105
|
+
* place are NaN. Use decodePointCloud() to read it.
|
|
106
|
+
*/
|
|
98
107
|
interface Mask {
|
|
99
108
|
bitmap: string;
|
|
100
109
|
width: number;
|
|
101
110
|
height: number;
|
|
102
111
|
stride: number;
|
|
112
|
+
world?: string;
|
|
103
113
|
}
|
|
104
114
|
/**
|
|
105
115
|
* A frame-level depth map as produced by depth estimation abilities
|
|
@@ -110,15 +120,35 @@ interface Mask {
|
|
|
110
120
|
* source frame; map a source coordinate (x, y) to the map proportionally:
|
|
111
121
|
* (x * width / source_width, y * height / source_height).
|
|
112
122
|
*
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
*
|
|
123
|
+
* Sky pixels carry +Infinity. Use decodeDepthMap() to access the values.
|
|
124
|
+
*
|
|
125
|
+
* `semantic` says what the values mean, and is always present in prediction v2 -
|
|
126
|
+
* "unknown" included - so an absent member means a worker that predates the
|
|
127
|
+
* field rather than a map that declined to say:
|
|
128
|
+
*
|
|
129
|
+
* - "canonical_metric" - metres = value * focal_px / 300, with focal_px scaled
|
|
130
|
+
* to the map's own resolution
|
|
131
|
+
* - "metric" - the value is already metres
|
|
132
|
+
* - "relative" - scale- AND shift-invariant, so ordering is meaningful but
|
|
133
|
+
* distance is not. Not back-projectable: recovering a cloud from it yields a
|
|
134
|
+
* distorted scene rather than a scaled one
|
|
135
|
+
* - "unknown" - the ability declared nothing. Not back-projectable
|
|
136
|
+
*
|
|
137
|
+
* `world` is the scene point cloud, present when the Pop asked for
|
|
138
|
+
* `depthMap.toWorld`. Same encoding as `Mask.world` - three little-endian
|
|
139
|
+
* float32 per pixel, row-major - but on this map's own grid, so the point for
|
|
140
|
+
* pixel (i, j) is at triple index j * width + i, exactly where that pixel's
|
|
141
|
+
* value sits in `values`. Points the worker could not place are NaN; the value
|
|
142
|
+
* at the same index is what says why, which is why both are sent.
|
|
116
143
|
*/
|
|
117
144
|
interface Depth {
|
|
118
145
|
width: number;
|
|
119
146
|
height: number;
|
|
120
147
|
values: string;
|
|
148
|
+
semantic?: DepthSemantic;
|
|
149
|
+
world?: string;
|
|
121
150
|
}
|
|
151
|
+
type DepthSemantic = 'canonical_metric' | 'metric' | 'relative' | 'unknown';
|
|
122
152
|
interface PredictedObject extends PredictedClass {
|
|
123
153
|
trackId?: number;
|
|
124
154
|
x: number;
|
|
@@ -134,11 +164,31 @@ interface PredictedObject extends PredictedClass {
|
|
|
134
164
|
texts?: Array<PredictedText>;
|
|
135
165
|
meshs?: Array<PredictedMesh>;
|
|
136
166
|
keyPoints?: Array<PredictedKeyPoints>;
|
|
137
|
-
details?: Array<
|
|
167
|
+
details?: Array<Record<string, any>>;
|
|
138
168
|
}
|
|
169
|
+
/**
|
|
170
|
+
* A point in source pixels, optionally placed in 3D.
|
|
171
|
+
*
|
|
172
|
+
* `worldX`/`worldY`/`worldZ` are metres, present only when the pipeline was
|
|
173
|
+
* asked for world coordinates - a Pop with `depthMap` and a component with
|
|
174
|
+
* `toWorld` - and only in prediction v2. A point the worker could not place -
|
|
175
|
+
* sky, outside the depth map, no usable map - carries none of the three rather
|
|
176
|
+
* than a zero or a NaN, so test for undefined.
|
|
177
|
+
*
|
|
178
|
+
* Which frame they are in depends on the source's camera: with extrinsics, the
|
|
179
|
+
* world frame those define (Z up, ground at Z = 0); without them, the camera
|
|
180
|
+
* frame in the OpenCV convention (X right, Y down, Z forward from the camera).
|
|
181
|
+
*
|
|
182
|
+
* Only the carriers the worker enriches ever populate them: key points, outline
|
|
183
|
+
* points and contour points. Bounding boxes and mesh points do not - a box is
|
|
184
|
+
* not a point, and any single anchor choice would be arbitrary.
|
|
185
|
+
*/
|
|
139
186
|
interface Point2d {
|
|
140
187
|
x: number;
|
|
141
188
|
y: number;
|
|
189
|
+
worldX?: number;
|
|
190
|
+
worldY?: number;
|
|
191
|
+
worldZ?: number;
|
|
142
192
|
}
|
|
143
193
|
interface PredictedMesh {
|
|
144
194
|
category?: string;
|
|
@@ -243,17 +293,17 @@ declare enum AutoAnnotateStatus {
|
|
|
243
293
|
}
|
|
244
294
|
interface DatasetAutoAnnotateCreate {
|
|
245
295
|
auto_annotate: string;
|
|
246
|
-
auto_annotate_params?:
|
|
296
|
+
auto_annotate_params?: Record<string, any>;
|
|
247
297
|
source_model_uuid?: string;
|
|
248
298
|
status?: AutoAnnotateStatus;
|
|
249
299
|
status_message?: string;
|
|
250
300
|
source?: string;
|
|
251
|
-
metrics?:
|
|
301
|
+
metrics?: Record<string, any>;
|
|
252
302
|
}
|
|
253
303
|
interface DatasetAutoAnnotateUpdate {
|
|
254
304
|
status?: AutoAnnotateStatus;
|
|
255
305
|
status_message?: string;
|
|
256
|
-
metrics?:
|
|
306
|
+
metrics?: Record<string, any>;
|
|
257
307
|
}
|
|
258
308
|
interface DatasetAutoAnnotate {
|
|
259
309
|
created_at?: Date;
|
|
@@ -263,11 +313,11 @@ interface DatasetAutoAnnotate {
|
|
|
263
313
|
source_ability_uuid?: string;
|
|
264
314
|
source_model_uuid?: string;
|
|
265
315
|
auto_annotate?: string;
|
|
266
|
-
auto_annotate_params?:
|
|
316
|
+
auto_annotate_params?: Record<string, any>;
|
|
267
317
|
status?: AutoAnnotateStatus;
|
|
268
318
|
status_message?: string;
|
|
269
319
|
source?: string;
|
|
270
|
-
metrics?:
|
|
320
|
+
metrics?: Record<string, any>;
|
|
271
321
|
}
|
|
272
322
|
interface AssetImport {
|
|
273
323
|
url: string;
|
|
@@ -435,7 +485,7 @@ interface Model {
|
|
|
435
485
|
description?: string;
|
|
436
486
|
external_id?: string;
|
|
437
487
|
pretrained_model_uuid?: string;
|
|
438
|
-
extra_params?:
|
|
488
|
+
extra_params?: Record<string, any>;
|
|
439
489
|
task?: ModelTask;
|
|
440
490
|
classes?: string[];
|
|
441
491
|
type: ModelType;
|
|
@@ -454,7 +504,7 @@ interface ModelCreate {
|
|
|
454
504
|
description: string;
|
|
455
505
|
external_id?: string;
|
|
456
506
|
pretrained_model_uuid?: string;
|
|
457
|
-
extra_params?:
|
|
507
|
+
extra_params?: Record<string, any>;
|
|
458
508
|
task?: ModelTask;
|
|
459
509
|
classes?: string[];
|
|
460
510
|
}
|
|
@@ -590,6 +640,12 @@ interface InferRuntimeConfig {
|
|
|
590
640
|
context_length?: number;
|
|
591
641
|
roi?: Area;
|
|
592
642
|
}
|
|
643
|
+
/**
|
|
644
|
+
* Schema to constrain a VLM's output. Where one is set, the ability is
|
|
645
|
+
* guaranteed to return a structured object matching this Open API schema, and
|
|
646
|
+
* the parsed object arrives on each prediction's `details` attribute.
|
|
647
|
+
*/
|
|
648
|
+
type ResponseSchema = OpenAPIV3_1.SchemaObject;
|
|
593
649
|
interface TransformInto {
|
|
594
650
|
classes?: string[];
|
|
595
651
|
}
|
|
@@ -599,6 +655,7 @@ interface InferRequest {
|
|
|
599
655
|
config: InferRuntimeConfig;
|
|
600
656
|
refresh?: boolean;
|
|
601
657
|
transform_into?: TransformInto;
|
|
658
|
+
response_schema?: ResponseSchema;
|
|
602
659
|
}
|
|
603
660
|
interface EvaluateFilter {
|
|
604
661
|
partitions?: string[];
|
|
@@ -634,7 +691,7 @@ interface EvaluateResponse {
|
|
|
634
691
|
status: EvaluationStatus;
|
|
635
692
|
status_message?: string;
|
|
636
693
|
source: string;
|
|
637
|
-
metrics?:
|
|
694
|
+
metrics?: Record<string, any>;
|
|
638
695
|
run_info: EvaluateRunInfo;
|
|
639
696
|
}
|
|
640
697
|
interface InferRunInfo {
|
|
@@ -650,6 +707,7 @@ interface AutoPromptConfig {
|
|
|
650
707
|
num_samples: number;
|
|
651
708
|
task_description?: string;
|
|
652
709
|
infer: InferRequest;
|
|
710
|
+
response_schema?: ResponseSchema;
|
|
653
711
|
evaluate: EvaluateConfig;
|
|
654
712
|
}
|
|
655
713
|
declare enum TaskType {
|
|
@@ -685,6 +743,7 @@ interface VlmAbilityCreate {
|
|
|
685
743
|
worker_release: string;
|
|
686
744
|
text_prompt: string;
|
|
687
745
|
transform_into: TransformInto;
|
|
746
|
+
response_schema?: ResponseSchema;
|
|
688
747
|
config: InferRuntimeConfig;
|
|
689
748
|
is_public: boolean;
|
|
690
749
|
video_chunk_length_ns?: number;
|
|
@@ -696,6 +755,7 @@ interface VlmAbilityUpdate {
|
|
|
696
755
|
worker_release?: string;
|
|
697
756
|
text_prompt?: string;
|
|
698
757
|
transform_into?: TransformInto;
|
|
758
|
+
response_schema?: ResponseSchema;
|
|
699
759
|
config?: InferRuntimeConfig;
|
|
700
760
|
is_public?: boolean;
|
|
701
761
|
video_chunk_length_ns?: number;
|
|
@@ -714,6 +774,7 @@ interface VlmAbility {
|
|
|
714
774
|
worker_release?: string;
|
|
715
775
|
text_prompt?: string;
|
|
716
776
|
transform_into?: TransformInto;
|
|
777
|
+
response_schema?: ResponseSchema;
|
|
717
778
|
config?: InferRuntimeConfig;
|
|
718
779
|
video_chunk_length_ns?: number;
|
|
719
780
|
video_chunk_overlap?: number;
|
|
@@ -750,6 +811,111 @@ interface AliasResolution {
|
|
|
750
811
|
model_uuid: string | null;
|
|
751
812
|
}
|
|
752
813
|
|
|
814
|
+
/**
|
|
815
|
+
* Camera calibration for a source, as accepted by the worker's setSource.
|
|
816
|
+
*
|
|
817
|
+
* A calibration is what turns a depth map into measurable world coordinates.
|
|
818
|
+
* Without one the worker falls back to an assumed horizontal field of view,
|
|
819
|
+
* which is a development scaffold rather than something to ship: for canonical
|
|
820
|
+
* metric depth the guess cancels out of X and Y and survives only in Z, so
|
|
821
|
+
* lateral measurements stay exact while every distance along the optical axis
|
|
822
|
+
* is wrong by however wrong the guess was.
|
|
823
|
+
*
|
|
824
|
+
* Mirrors what the instance API accepts. validateCamera() checks it locally so
|
|
825
|
+
* a bad calibration is an error here rather than a 400 halfway through an
|
|
826
|
+
* upload; the worker validates authoritatively, and the two are meant to agree.
|
|
827
|
+
*/
|
|
828
|
+
/**
|
|
829
|
+
* How far a rotation's norm may sit from 1 and still be taken for a rotation.
|
|
830
|
+
* Loose enough to survive a float round trip through JSON, tight enough to
|
|
831
|
+
* catch an un-normalised or garbage one. Matches the instance and the worker.
|
|
832
|
+
*/
|
|
833
|
+
declare const QUATERNION_TOLERANCE = 0.001;
|
|
834
|
+
/**
|
|
835
|
+
* The pinhole parameters, normalised to the frame rather than given in pixels.
|
|
836
|
+
*
|
|
837
|
+
* Normalised so one calibration survives a resolution change: a camera
|
|
838
|
+
* calibrated at 1920x1080 and later streamed at 1280x720 keeps working. Pixel
|
|
839
|
+
* values carrying no calibration resolution are wrong by exactly the resolution
|
|
840
|
+
* ratio, and nothing downstream can detect it.
|
|
841
|
+
*
|
|
842
|
+
* `fx`/`fy` are the focal length as a fraction of the frame's width and height;
|
|
843
|
+
* `cx`/`cy` are the principal point in the same normalisation.
|
|
844
|
+
*/
|
|
845
|
+
interface CameraIntrinsics {
|
|
846
|
+
fx: number;
|
|
847
|
+
fy: number;
|
|
848
|
+
cx: number;
|
|
849
|
+
cy: number;
|
|
850
|
+
}
|
|
851
|
+
/** The OpenCV Brown-Conrady coefficients. All zero is a rectilinear lens. */
|
|
852
|
+
interface CameraDistortion {
|
|
853
|
+
k1?: number;
|
|
854
|
+
k2?: number;
|
|
855
|
+
p1?: number;
|
|
856
|
+
p2?: number;
|
|
857
|
+
k3?: number;
|
|
858
|
+
}
|
|
859
|
+
/** A rotation, w first. Must be a unit quaternion. */
|
|
860
|
+
interface Quaternion {
|
|
861
|
+
w: number;
|
|
862
|
+
x: number;
|
|
863
|
+
y: number;
|
|
864
|
+
z: number;
|
|
865
|
+
}
|
|
866
|
+
/** A translation in metres. */
|
|
867
|
+
interface Vector3d {
|
|
868
|
+
x: number;
|
|
869
|
+
y: number;
|
|
870
|
+
z: number;
|
|
871
|
+
}
|
|
872
|
+
/**
|
|
873
|
+
* The camera's pose, expressed camera -> world.
|
|
874
|
+
*
|
|
875
|
+
* As `P_world = R(rotation) * P_camera + translation`, so the rotation carries
|
|
876
|
+
* the camera's axes onto the world's and the translation is where the camera
|
|
877
|
+
* itself sits. The world frame is Z up with the ground plane at Z = 0.
|
|
878
|
+
*
|
|
879
|
+
* This is the inverse of what cv2.solvePnP returns; a caller holding its
|
|
880
|
+
* rvec/tvec must invert both halves (R = R_cv.T, t = -R_cv.T @ t_cv), and t_cv
|
|
881
|
+
* is *not* the camera position.
|
|
882
|
+
*
|
|
883
|
+
* Left unset, the pose is the identity and world coordinates come back in the
|
|
884
|
+
* camera frame instead - OpenCV convention, X right, Y down, Z forward.
|
|
885
|
+
*/
|
|
886
|
+
interface CameraExtrinsics {
|
|
887
|
+
rotation?: Quaternion;
|
|
888
|
+
translation?: Vector3d;
|
|
889
|
+
}
|
|
890
|
+
/**
|
|
891
|
+
* One source's calibration.
|
|
892
|
+
*
|
|
893
|
+
* Exactly one of `intrinsics` and `hfovDegrees` describes the lens. Both is
|
|
894
|
+
* rejected rather than resolved by precedence - two descriptions of one lens
|
|
895
|
+
* that disagree have no right answer - and neither is rejected too, since
|
|
896
|
+
* defaulting a focal length would be inventing a lens.
|
|
897
|
+
*
|
|
898
|
+
* `hfovDegrees` is the shorthand for an operator who knows their lens's field
|
|
899
|
+
* of view but not its calibration matrix. It assumes square pixels and a
|
|
900
|
+
* centred principal point, so it approximates a real calibration rather than
|
|
901
|
+
* replacing one - but it describes the actual lens rather than the worker's
|
|
902
|
+
* assumed fallback. It composes with distortion and extrinsics exactly as
|
|
903
|
+
* intrinsics do.
|
|
904
|
+
*/
|
|
905
|
+
interface Camera {
|
|
906
|
+
intrinsics?: CameraIntrinsics;
|
|
907
|
+
hfovDegrees?: number;
|
|
908
|
+
distortion?: CameraDistortion;
|
|
909
|
+
extrinsics?: CameraExtrinsics;
|
|
910
|
+
}
|
|
911
|
+
/**
|
|
912
|
+
* Throw if the calibration cannot describe a real camera.
|
|
913
|
+
*
|
|
914
|
+
* Checked locally so a bad one fails before an upload starts rather than as a
|
|
915
|
+
* 400 partway through it. The rules are the worker's, not stricter ones.
|
|
916
|
+
*/
|
|
917
|
+
declare function validateCamera(camera: Camera): void;
|
|
918
|
+
|
|
753
919
|
declare enum PredictionVersion {
|
|
754
920
|
V1 = 1,
|
|
755
921
|
V2 = 2
|
|
@@ -803,7 +969,6 @@ interface ResultStream extends AsyncIterable<Prediction> {
|
|
|
803
969
|
declare enum PopComponentType {
|
|
804
970
|
FORWARD = "forward",
|
|
805
971
|
INFERENCE = "inference",
|
|
806
|
-
TRACING = "tracing",
|
|
807
972
|
TRACKING = "tracking",
|
|
808
973
|
CONTOUR_FINDER = "contour_finder",
|
|
809
974
|
COMPONENT_FINDER = "component_finder"
|
|
@@ -831,6 +996,23 @@ interface BaseComponent {
|
|
|
831
996
|
type: PopComponentType;
|
|
832
997
|
id?: number;
|
|
833
998
|
forward?: PopForward;
|
|
999
|
+
/**
|
|
1000
|
+
* Enrich this component's point based predictions with world coordinates,
|
|
1001
|
+
* back-projected through the Pop's `depthMap`.
|
|
1002
|
+
*
|
|
1003
|
+
* Declared here because the instance declares it on one shared component
|
|
1004
|
+
* type, but only a component that runs its own inference can honour it -
|
|
1005
|
+
* inference and tracking - since that is what gives it an id for the worker
|
|
1006
|
+
* to select on. Asking for it on a forward, contour finder or component
|
|
1007
|
+
* finder is rejected when the Pop is compiled. A contour finder's points do
|
|
1008
|
+
* get enriched, but they belong to the object that fed it, so the request
|
|
1009
|
+
* goes on the inference component upstream.
|
|
1010
|
+
*
|
|
1011
|
+
* Enrichment needs a *metric* depth ability. A `relative` map is accepted
|
|
1012
|
+
* and silently produces no world coordinates: its shift is unknown, so a
|
|
1013
|
+
* cloud recovered from it would be distorted rather than merely unscaled.
|
|
1014
|
+
*/
|
|
1015
|
+
toWorld?: boolean;
|
|
834
1016
|
}
|
|
835
1017
|
interface ForwardComponent extends BaseComponent {
|
|
836
1018
|
}
|
|
@@ -913,10 +1095,62 @@ interface ComponentFinderComponent extends BaseComponent {
|
|
|
913
1095
|
componentClassLabel?: string;
|
|
914
1096
|
}
|
|
915
1097
|
type PopComponent = ForwardComponent | InferenceComponent | TrackingComponent | ContourFinderComponent | ComponentFinderComponent;
|
|
1098
|
+
/**
|
|
1099
|
+
* The depth ability whose frame level map feeds world coordinates.
|
|
1100
|
+
*
|
|
1101
|
+
* `ability` names it by alias and `abilityUuid` by uuid; give exactly one. A
|
|
1102
|
+
* Pop has one depth map because the worker back-projects every prediction
|
|
1103
|
+
* through it, so a second depth source would have nowhere to go. Naming one
|
|
1104
|
+
* makes the converter build the depth branch itself and keep it out of the
|
|
1105
|
+
* response - the caller asked for coordinates, not for a megabyte of base64
|
|
1106
|
+
* depth per frame.
|
|
1107
|
+
*
|
|
1108
|
+
* `toWorld` back-projects the map itself, so the response carries a point cloud
|
|
1109
|
+
* of the whole scene rather than one per segmented object. It is also what
|
|
1110
|
+
* reveals the map: without it the injected branch stays out of the response
|
|
1111
|
+
* entirely. Read the result with cloudOfDepth().
|
|
1112
|
+
*
|
|
1113
|
+
* Use a *metric* depth ability. A `relative` one is accepted and yields no
|
|
1114
|
+
* world coordinates at all.
|
|
1115
|
+
*/
|
|
1116
|
+
interface PopDepthMap {
|
|
1117
|
+
ability?: string;
|
|
1118
|
+
abilityUuid?: string;
|
|
1119
|
+
toWorld?: boolean;
|
|
1120
|
+
}
|
|
1121
|
+
/**
|
|
1122
|
+
* Source level parameters a Pop applies to every source it processes, each
|
|
1123
|
+
* overridable per source.
|
|
1124
|
+
*
|
|
1125
|
+
* Merged per field against whatever a source supplies, so a source giving its
|
|
1126
|
+
* own roi but no camera keeps its roi and takes the default camera. Because the
|
|
1127
|
+
* camera merges as one field, a source declaring its own lens replaces a
|
|
1128
|
+
* defaulted one outright rather than mixing the two.
|
|
1129
|
+
*/
|
|
1130
|
+
interface SourceDefaults {
|
|
1131
|
+
camera?: Camera;
|
|
1132
|
+
roi?: Area;
|
|
1133
|
+
fps?: string;
|
|
1134
|
+
motionDetect?: boolean;
|
|
1135
|
+
motionSensitivity?: number;
|
|
1136
|
+
motionThreshold?: number;
|
|
1137
|
+
motionGap?: number;
|
|
1138
|
+
motionGridX?: number;
|
|
1139
|
+
motionGridY?: number;
|
|
1140
|
+
}
|
|
916
1141
|
interface Pop {
|
|
917
1142
|
components: PopComponent[];
|
|
918
1143
|
postTransform?: string;
|
|
1144
|
+
defaults?: SourceDefaults;
|
|
1145
|
+
depthMap?: PopDepthMap;
|
|
919
1146
|
}
|
|
1147
|
+
/**
|
|
1148
|
+
* Throw if a Pop cannot mean what it says.
|
|
1149
|
+
*
|
|
1150
|
+
* Only the parts a caller can get wrong locally: the worker rejects both of
|
|
1151
|
+
* these too, but as a 400 once the Pop is already in flight.
|
|
1152
|
+
*/
|
|
1153
|
+
declare function validatePop(pop: Pop): void;
|
|
920
1154
|
interface MotionDetectConfig {
|
|
921
1155
|
motionDetect: boolean;
|
|
922
1156
|
motionSensitivity?: number;
|
|
@@ -936,6 +1170,16 @@ interface ProcessParams {
|
|
|
936
1170
|
motionDetect?: MotionDetectConfig | undefined;
|
|
937
1171
|
roi?: Area | undefined;
|
|
938
1172
|
fps?: string | undefined;
|
|
1173
|
+
/**
|
|
1174
|
+
* This source's camera calibration, overriding the Pop's `defaults.camera`.
|
|
1175
|
+
*
|
|
1176
|
+
* Without one the worker assumes a 60 degree horizontal field of view,
|
|
1177
|
+
* which is a development scaffold: for canonical metric depth the guess
|
|
1178
|
+
* cancels out of X and Y and survives only in Z, so lateral measurements
|
|
1179
|
+
* stay exact while every distance along the optical axis is wrong by
|
|
1180
|
+
* however wrong the guess was.
|
|
1181
|
+
*/
|
|
1182
|
+
camera?: Camera | undefined;
|
|
939
1183
|
}
|
|
940
1184
|
interface ProcessRequest extends ProcessParams {
|
|
941
1185
|
source: Source;
|
|
@@ -1189,6 +1433,16 @@ declare class WorkerEndpoint extends Endpoint<WorkerEndpoint> {
|
|
|
1189
1433
|
*/
|
|
1190
1434
|
process({ source, componentParams, motionDetect, roi }: ProcessRequest): Promise<ResultStream>;
|
|
1191
1435
|
private processV1;
|
|
1436
|
+
/**
|
|
1437
|
+
* Reject a source's calibration before any work is started for it.
|
|
1438
|
+
*
|
|
1439
|
+
* Called by every entry point that takes ProcessParams rather than at the
|
|
1440
|
+
* one boundary they share, because they have none: the group methods build
|
|
1441
|
+
* their jobs directly instead of going through processV2. It runs before
|
|
1442
|
+
* the concurrency limit is acquired, so a request that can never be sent
|
|
1443
|
+
* does not hold a slot.
|
|
1444
|
+
*/
|
|
1445
|
+
private validateProcessParams;
|
|
1192
1446
|
private processV2;
|
|
1193
1447
|
uploadGroup(paths: string[], params?: ProcessParams): Promise<ResultStream>;
|
|
1194
1448
|
uploadStreamGroup(streams: Array<ReadableStream<Uint8Array> | Blob | BufferSource>, mimeTypes?: string[], params?: ProcessParams): Promise<ResultStream>;
|
|
@@ -1385,6 +1639,123 @@ declare class DepthMap {
|
|
|
1385
1639
|
*/
|
|
1386
1640
|
declare function decodeDepthMap(depth: Depth): DepthMap;
|
|
1387
1641
|
|
|
1642
|
+
/**
|
|
1643
|
+
* Decoding and sampling of point clouds.
|
|
1644
|
+
*
|
|
1645
|
+
* When a pipeline is asked for world coordinates, an object carrying a
|
|
1646
|
+
* segmentation mask also carries `mask.world`: three little-endian float32
|
|
1647
|
+
* values per mask pixel, row-major, exactly width*height triples. The point for
|
|
1648
|
+
* bitmap pixel (i, j) sits at triple index j * width + i, so the cloud indexes
|
|
1649
|
+
* exactly like the bitmap with no separate lookup.
|
|
1650
|
+
*
|
|
1651
|
+
* A Pop asking for `depthMap.toWorld` also gets `depth.world`, the same
|
|
1652
|
+
* encoding on the depth map's own grid: one point per map pixel rather than per
|
|
1653
|
+
* mask pixel, covering the whole scene.
|
|
1654
|
+
*
|
|
1655
|
+
* Points the worker could not place are NaN - sky pixels, samples outside the
|
|
1656
|
+
* depth map, and pixels the mask does not cover. NaN is the omission sentinel
|
|
1657
|
+
* here, which is why this does not share decodeDepthMap()'s validation: that
|
|
1658
|
+
* one rejects a payload of the wrong length, but nothing here may reject a NaN.
|
|
1659
|
+
*/
|
|
1660
|
+
|
|
1661
|
+
/** Where a cloud's grid sits in the source frame, so at() can be reached from source pixels. */
|
|
1662
|
+
interface CloudBox {
|
|
1663
|
+
x: number;
|
|
1664
|
+
y: number;
|
|
1665
|
+
width: number;
|
|
1666
|
+
height: number;
|
|
1667
|
+
}
|
|
1668
|
+
/**
|
|
1669
|
+
* A decoded point cloud, per object or per scene.
|
|
1670
|
+
*
|
|
1671
|
+
* Create with decodePointCloud(), cloudOfObject(), cloudOfDepth() or
|
|
1672
|
+
* cloudsOfPrediction(). Coordinates are metres, in the frame the source's
|
|
1673
|
+
* camera extrinsics define (Z up, ground at Z = 0) or the OpenCV camera frame
|
|
1674
|
+
* when it supplied none.
|
|
1675
|
+
*
|
|
1676
|
+
* The grid is whatever the cloud was made from: an object's mask, which spans
|
|
1677
|
+
* its bounding box rather than the frame, or the depth map itself, which spans
|
|
1678
|
+
* the whole frame. Either way (i, j) is a source coordinate only via that box,
|
|
1679
|
+
* which cloudOfObject() and cloudOfDepth() record - so atSource() needs one of
|
|
1680
|
+
* those two.
|
|
1681
|
+
*/
|
|
1682
|
+
declare class PointCloud {
|
|
1683
|
+
readonly width: number;
|
|
1684
|
+
readonly height: number;
|
|
1685
|
+
/** xyz per pixel, row-major; an omitted point is NaN in all three. */
|
|
1686
|
+
readonly points: Float32Array;
|
|
1687
|
+
private readonly box;
|
|
1688
|
+
constructor(width: number, height: number, points: Float32Array, box?: CloudBox);
|
|
1689
|
+
/**
|
|
1690
|
+
* The world point for pixel (i, j), or undefined where the worker placed none.
|
|
1691
|
+
*
|
|
1692
|
+
* (i, j) indexes the cloud's own grid - the mask bitmap, or the depth map
|
|
1693
|
+
* for a scene cloud: i is the column, j is the row.
|
|
1694
|
+
*/
|
|
1695
|
+
at(i: number, j: number): Vector3 | undefined;
|
|
1696
|
+
/**
|
|
1697
|
+
* The world point for a source frame coordinate inside the cloud's box.
|
|
1698
|
+
*
|
|
1699
|
+
* Needs a cloud built with cloudOfObject(), or with cloudOfDepth() and the
|
|
1700
|
+
* frame size: the grid covers a box, and nothing else knows where it sits.
|
|
1701
|
+
*/
|
|
1702
|
+
atSource(x: number, y: number): Vector3 | undefined;
|
|
1703
|
+
isPlaced(i: number, j: number): boolean;
|
|
1704
|
+
/**
|
|
1705
|
+
* Just the points the worker placed.
|
|
1706
|
+
*
|
|
1707
|
+
* The shape a scatter plot or an export wants: the grid with its NaN holes
|
|
1708
|
+
* dropped, and no mask to apply first.
|
|
1709
|
+
*/
|
|
1710
|
+
get placedPoints(): Vector3[];
|
|
1711
|
+
/**
|
|
1712
|
+
* Per axis (min, max) in metres over the placed points, or undefined if none were.
|
|
1713
|
+
*
|
|
1714
|
+
* The counterpart to DepthMap's finiteMin/finiteMax, which are one axis
|
|
1715
|
+
* because a depth map has one value per pixel.
|
|
1716
|
+
*/
|
|
1717
|
+
get bounds(): CloudBounds | undefined;
|
|
1718
|
+
}
|
|
1719
|
+
interface Vector3 {
|
|
1720
|
+
x: number;
|
|
1721
|
+
y: number;
|
|
1722
|
+
z: number;
|
|
1723
|
+
}
|
|
1724
|
+
interface CloudBounds {
|
|
1725
|
+
min: Vector3;
|
|
1726
|
+
max: Vector3;
|
|
1727
|
+
}
|
|
1728
|
+
/** A mask's point cloud, or undefined if it carries none. */
|
|
1729
|
+
declare function decodePointCloud(mask: Mask | undefined, box?: CloudBox): PointCloud | undefined;
|
|
1730
|
+
/**
|
|
1731
|
+
* The point cloud of a predicted object's mask, or undefined if it has none.
|
|
1732
|
+
*
|
|
1733
|
+
* Records the object's bounding box, so atSource() works.
|
|
1734
|
+
*/
|
|
1735
|
+
declare function cloudOfObject(obj: PredictedObject | undefined): PointCloud | undefined;
|
|
1736
|
+
/**
|
|
1737
|
+
* The scene point cloud of a frame level depth map, or undefined if it has none.
|
|
1738
|
+
*
|
|
1739
|
+
* Present when the Pop asked for `depthMap.toWorld`. Unlike a mask cloud this
|
|
1740
|
+
* one spans the whole frame, so its grid is the depth map's rather than an
|
|
1741
|
+
* object's bounding box - and at(i, j) indexes map pixels, the same index that
|
|
1742
|
+
* pixel's value sits at in `depth.values`.
|
|
1743
|
+
*
|
|
1744
|
+
* Pass the prediction's source_width/source_height to enable atSource(); the
|
|
1745
|
+
* frame is the box in this case.
|
|
1746
|
+
*/
|
|
1747
|
+
declare function cloudOfDepth(depth: Depth | undefined, sourceWidth?: number, sourceHeight?: number): PointCloud | undefined;
|
|
1748
|
+
/**
|
|
1749
|
+
* Every point cloud in a prediction, outermost object first.
|
|
1750
|
+
*
|
|
1751
|
+
* An array rather than a single value like decodeDepthMap(), because a cloud
|
|
1752
|
+
* belongs to one object's mask and a prediction carries as many as it has
|
|
1753
|
+
* masked objects. Nested objects are included. A scene cloud, if the Pop asked
|
|
1754
|
+
* for one, comes last - appended rather than inserted so an existing index into
|
|
1755
|
+
* the object clouds keeps meaning what it did.
|
|
1756
|
+
*/
|
|
1757
|
+
declare function cloudsOfPrediction(prediction: Prediction | undefined): PointCloud[];
|
|
1758
|
+
|
|
1388
1759
|
type Bytes = string | ArrayBuffer | Uint8Array | Buffer | null | undefined;
|
|
1389
1760
|
interface StreamEvent {
|
|
1390
1761
|
type: string;
|
|
@@ -1563,4 +1934,4 @@ declare namespace EyePop {
|
|
|
1563
1934
|
function dataEndpoint(opts?: DataOptions): DataEndpoint;
|
|
1564
1935
|
}
|
|
1565
1936
|
|
|
1566
|
-
export { type AbilityAliasEntry, AbstractJob, type AccessTokenAuth, type AliasResolution, type Annotation, AnnotationInclusionMode, AnnotationType, type ApiKeyAuth, type Area, AreaType, ArtifactType, type Asset, type AssetImport, AssetInclusionMode, AssetStatus, AssetUrlType, type AssetUuidSource, type Auth0Options, type Authentication, type AutoAnnotate, type AutoAnnotateParams, type AutoAnnotatePrompt, AutoAnnotateStatus, AutoAnnotateTask, type AutoPromptConfig, type AutoTask, type BaseArea, type BaseComponent, type BaseTask, type Box, type ChangeEvent, ChangeType, type ClassificationTask, type ComponentFinderComponent, type ComponentParams, type Contour, type ContourArea, type ContourFinderComponent, ContourType, type CreateWorkflow, type CreateWorkflowConfig, DEFAULT_PREDICTION_VERSION, DataApiType, DataEndpoint, type DataOptions, type DataSession, type Dataset, type DatasetAutoAnnotate, type DatasetAutoAnnotateCreate, type DatasetAutoAnnotateUpdate, type DatasetCreate, type DatasetHeroAssetUpdate, type DatasetParent, type DatasetUpdate, type DatasetVersion, type DatasetVersionAssetStats, type Depth, DepthMap, type DownloadResponse, Endpoint, EndpointState, type EvaluateConfig, type EvaluateFilter, type EvaluateRequest, type EvaluateResponse, type EvaluateRunInfo, EvaluationStatus, ExportedBy, EyePop, EyePopDataWebSocketError, EyepopLineDecoder, type FileSource, type ForwardComponent, ForwardOperatorType, type HttpClient, type InferRequest, type InferRunInfo, type InferRuntimeConfig, type InferenceComponent, InferenceType, type ListWorkFlowItem, type ListWorkFlowItemMetadata, type ListWorkflowItemMetadataLabels, LoadFromAssetUuidJob, LoadFromGroupJob, LoadFromJob, LoadMediaStreamJob, type LocalAuth, type Mask, type MediaStreamSource, type Model, type ModelCreate, type ModelExport, ModelExportStatus, ModelFormat, type ModelMetrics, ModelPrecisionType, type ModelSample, ModelStatus, ModelTask, type ModelTrainingProgress, ModelTrainingStage, ModelType, type ModelUpdate, type MotionDetectConfig, MotionModel, type MultipartContent, type MultipartPart, type OAuth2Auth, type OnChangeEvent, type Options, type PathSource, type PlatformSupport, type Point2d, type Point3d, type Pop, type PopComponent, PopComponentType, type PopCrop, type PopForward, type PopForwardOperator, type PredictedClass, type PredictedEmbedding, type PredictedKeyPoint, type PredictedKeyPoints, type PredictedMesh, type PredictedMotion, type PredictedObject, type PredictedText, type Prediction, PredictionVersion, type ProcessParams, type ProcessRequest, type QcAiHubExportParams, type RectangleArea, type ResultStream, type Roi, Semaphore, type Session, type SessionAuth, type Source, Stream, type StreamEvent, type StreamSource, type StreamTime, TaskMode, TaskType, type TimeSpan, type TrackingComponent, TranscodeMode, type TransformInto, TransientPopId, UploadGroupJob, UploadJob, type UploadSource, type UrlSource, UserReview, VideoMode, type VlmAbility, type VlmAbilityCreate, type VlmAbilityGroup, type VlmAbilityGroupCreate, type VlmAbilityGroupUpdate, VlmAbilityStatus, type VlmAbilityUpdate, WebrtcBase, WebrtcWhep, WebrtcWhip, WorkerEndpoint, type WorkerOptions, type WorkerSession, type Workflow, WorkflowPhase, accessTokenCredential, apiKeyCredential, decodeDepthMap, EyePop as default, encodeMultipartFormData, getBaseUrl, randomMultipartBoundary, readableStreamFromString, resolveAuth };
|
|
1937
|
+
export { type AbilityAliasEntry, AbstractJob, type AccessTokenAuth, type AliasResolution, type Annotation, AnnotationInclusionMode, AnnotationType, type ApiKeyAuth, type Area, AreaType, ArtifactType, type Asset, type AssetImport, AssetInclusionMode, AssetStatus, AssetUrlType, type AssetUuidSource, type Auth0Options, type Authentication, type AutoAnnotate, type AutoAnnotateParams, type AutoAnnotatePrompt, AutoAnnotateStatus, AutoAnnotateTask, type AutoPromptConfig, type AutoTask, type BaseArea, type BaseComponent, type BaseTask, type Box, type Camera, type CameraDistortion, type CameraExtrinsics, type CameraIntrinsics, type ChangeEvent, ChangeType, type ClassificationTask, type CloudBounds, type CloudBox, type ComponentFinderComponent, type ComponentParams, type Contour, type ContourArea, type ContourFinderComponent, ContourType, type CreateWorkflow, type CreateWorkflowConfig, DEFAULT_PREDICTION_VERSION, DataApiType, DataEndpoint, type DataOptions, type DataSession, type Dataset, type DatasetAutoAnnotate, type DatasetAutoAnnotateCreate, type DatasetAutoAnnotateUpdate, type DatasetCreate, type DatasetHeroAssetUpdate, type DatasetParent, type DatasetUpdate, type DatasetVersion, type DatasetVersionAssetStats, type Depth, DepthMap, type DepthSemantic, type DownloadResponse, Endpoint, EndpointState, type EvaluateConfig, type EvaluateFilter, type EvaluateRequest, type EvaluateResponse, type EvaluateRunInfo, EvaluationStatus, ExportedBy, EyePop, EyePopDataWebSocketError, EyepopLineDecoder, type FileSource, type ForwardComponent, ForwardOperatorType, type HttpClient, type InferRequest, type InferRunInfo, type InferRuntimeConfig, type InferenceComponent, InferenceType, type ListWorkFlowItem, type ListWorkFlowItemMetadata, type ListWorkflowItemMetadataLabels, LoadFromAssetUuidJob, LoadFromGroupJob, LoadFromJob, LoadMediaStreamJob, type LocalAuth, type Mask, type MediaStreamSource, type Model, type ModelCreate, type ModelExport, ModelExportStatus, ModelFormat, type ModelMetrics, ModelPrecisionType, type ModelSample, ModelStatus, ModelTask, type ModelTrainingProgress, ModelTrainingStage, ModelType, type ModelUpdate, type MotionDetectConfig, MotionModel, type MultipartContent, type MultipartPart, type OAuth2Auth, type OnChangeEvent, type Options, type PathSource, type PlatformSupport, type Point2d, type Point3d, PointCloud, type Pop, type PopComponent, PopComponentType, type PopCrop, type PopDepthMap, type PopForward, type PopForwardOperator, type PredictedClass, type PredictedEmbedding, type PredictedKeyPoint, type PredictedKeyPoints, type PredictedMesh, type PredictedMotion, type PredictedObject, type PredictedText, type Prediction, PredictionVersion, type ProcessParams, type ProcessRequest, QUATERNION_TOLERANCE, type QcAiHubExportParams, type Quaternion, type RectangleArea, type ResponseSchema, type ResultStream, type Roi, Semaphore, type Session, type SessionAuth, type Source, type SourceDefaults, Stream, type StreamEvent, type StreamSource, type StreamTime, TaskMode, TaskType, type TimeSpan, type TrackingComponent, TranscodeMode, type TransformInto, TransientPopId, UploadGroupJob, UploadJob, type UploadSource, type UrlSource, UserReview, type Vector3, type Vector3d, VideoMode, type VlmAbility, type VlmAbilityCreate, type VlmAbilityGroup, type VlmAbilityGroupCreate, type VlmAbilityGroupUpdate, VlmAbilityStatus, type VlmAbilityUpdate, WebrtcBase, WebrtcWhep, WebrtcWhip, WorkerEndpoint, type WorkerOptions, type WorkerSession, type Workflow, WorkflowPhase, accessTokenCredential, apiKeyCredential, cloudOfDepth, cloudOfObject, cloudsOfPrediction, decodeDepthMap, decodePointCloud, EyePop as default, encodeMultipartFormData, getBaseUrl, randomMultipartBoundary, readableStreamFromString, resolveAuth, validateCamera, validatePop };
|