@camstack/addon-post-analysis 1.2.346 → 1.2.348
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{clip-model-registry--UsI8Ud4.js → clip-model-registry-AqAEU7Eo.js} +1759 -147
- package/dist/{clip-model-registry-BftmbgcV.mjs → clip-model-registry-CrUfogC-.mjs} +1759 -147
- package/dist/embedding-encoder/index.js +1 -1
- package/dist/embedding-encoder/index.mjs +1 -1
- package/dist/pipeline-analytics/_stub.js +2 -2
- package/dist/pipeline-analytics/{_virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-CeJTtS8B.mjs → _virtual_mf-localSharedImportMap___mfe_internal__addon_pipeline_analytics_widgets-zK8c1RbI.mjs} +3 -3
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-Du_tzgi2.mjs +26 -0
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-DLOjWwaN.mjs +26 -0
- package/dist/pipeline-analytics/{hostInit-DkInRTZ0.mjs → hostInit-DAeozYqI.mjs} +3 -3
- package/dist/pipeline-analytics/index.js +423 -242
- package/dist/pipeline-analytics/index.mjs +423 -242
- package/dist/pipeline-analytics/remoteEntry.js +1 -1
- package/package.json +1 -1
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_types__loadShare__.js-4Fu7jNn9.mjs +0 -26
- package/dist/pipeline-analytics/_virtual_mf___mfe_internal__addon_pipeline_analytics_widgets__loadShare___mf_0_camstack_mf_1_ui_mf_2_library__loadShare__.js-Cutp7k0_.mjs +0 -26
|
@@ -2,7 +2,7 @@ Object.defineProperties(exports, {
|
|
|
2
2
|
__esModule: { value: true },
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
|
-
const require_clip_model_registry = require("../clip-model-registry
|
|
5
|
+
const require_clip_model_registry = require("../clip-model-registry-AqAEU7Eo.js");
|
|
6
6
|
let node_fs = require("node:fs");
|
|
7
7
|
node_fs = require_clip_model_registry.__toESM(node_fs, 1);
|
|
8
8
|
let node_path = require("node:path");
|
|
@@ -83507,6 +83507,260 @@ function withDatasetVersion(manifest, version) {
|
|
|
83507
83507
|
};
|
|
83508
83508
|
}
|
|
83509
83509
|
//#endregion
|
|
83510
|
+
//#region src/pipeline-analytics/retrain/retrain-dataset-version-store.ts
|
|
83511
|
+
/**
|
|
83512
|
+
* @durable class=ledger owner=pipeline-analytics
|
|
83513
|
+
* write="one row per dataset version the operator creates: a frozen frameIds list,
|
|
83514
|
+
* the sha256 of the annotations those frames carried, the export parameters and
|
|
83515
|
+
* the counts at creation. Never updated after the insert — a different selection
|
|
83516
|
+
* is a new version."
|
|
83517
|
+
* retention="none — rows leave only when the operator deletes the version. The
|
|
83518
|
+
* frames themselves are owned by the retrain corpus; deleting a version deletes no
|
|
83519
|
+
* frame, and deleting a frame does not edit a version (verify reports it missing)."
|
|
83520
|
+
*/
|
|
83521
|
+
var RETRAIN_DATASET_VERSIONS_COLLECTION = "pipeline-analytics:retrain-dataset-versions";
|
|
83522
|
+
var COLUMNS = [
|
|
83523
|
+
{
|
|
83524
|
+
name: "id",
|
|
83525
|
+
type: "TEXT",
|
|
83526
|
+
primaryKey: true,
|
|
83527
|
+
notNull: true
|
|
83528
|
+
},
|
|
83529
|
+
{
|
|
83530
|
+
name: "name",
|
|
83531
|
+
type: "TEXT",
|
|
83532
|
+
notNull: true
|
|
83533
|
+
},
|
|
83534
|
+
{
|
|
83535
|
+
name: "nameKey",
|
|
83536
|
+
type: "TEXT",
|
|
83537
|
+
notNull: true
|
|
83538
|
+
},
|
|
83539
|
+
{
|
|
83540
|
+
name: "createdAt",
|
|
83541
|
+
type: "INTEGER",
|
|
83542
|
+
notNull: true
|
|
83543
|
+
},
|
|
83544
|
+
{
|
|
83545
|
+
name: "selection",
|
|
83546
|
+
type: "JSON",
|
|
83547
|
+
notNull: true
|
|
83548
|
+
},
|
|
83549
|
+
{
|
|
83550
|
+
name: "layout",
|
|
83551
|
+
type: "TEXT",
|
|
83552
|
+
notNull: true
|
|
83553
|
+
},
|
|
83554
|
+
{
|
|
83555
|
+
name: "completeOnly",
|
|
83556
|
+
type: "BOOLEAN",
|
|
83557
|
+
notNull: true
|
|
83558
|
+
},
|
|
83559
|
+
{
|
|
83560
|
+
name: "frameIds",
|
|
83561
|
+
type: "JSON",
|
|
83562
|
+
notNull: true
|
|
83563
|
+
},
|
|
83564
|
+
{
|
|
83565
|
+
name: "annotationHash",
|
|
83566
|
+
type: "TEXT",
|
|
83567
|
+
notNull: true
|
|
83568
|
+
},
|
|
83569
|
+
{
|
|
83570
|
+
name: "manifestSchema",
|
|
83571
|
+
type: "TEXT",
|
|
83572
|
+
notNull: true
|
|
83573
|
+
},
|
|
83574
|
+
{
|
|
83575
|
+
name: "stats",
|
|
83576
|
+
type: "JSON",
|
|
83577
|
+
notNull: true
|
|
83578
|
+
}
|
|
83579
|
+
];
|
|
83580
|
+
var INDEXES = [{
|
|
83581
|
+
name: "idx_retrain_dataset_versions_name_key",
|
|
83582
|
+
columns: ["nameKey"],
|
|
83583
|
+
unique: true
|
|
83584
|
+
}, {
|
|
83585
|
+
name: "idx_retrain_dataset_versions_created",
|
|
83586
|
+
columns: ["createdAt"]
|
|
83587
|
+
}];
|
|
83588
|
+
function summaryOfVersion(version) {
|
|
83589
|
+
const { frameIds, ...rest } = version;
|
|
83590
|
+
return {
|
|
83591
|
+
...rest,
|
|
83592
|
+
frameCount: frameIds.length
|
|
83593
|
+
};
|
|
83594
|
+
}
|
|
83595
|
+
var nameKeyOf = (name) => name.trim().toLowerCase();
|
|
83596
|
+
function numberList(value) {
|
|
83597
|
+
return Array.isArray(value) ? value.filter((v) => typeof v === "number") : void 0;
|
|
83598
|
+
}
|
|
83599
|
+
function stringList(value) {
|
|
83600
|
+
return Array.isArray(value) ? value.filter((v) => typeof v === "string") : void 0;
|
|
83601
|
+
}
|
|
83602
|
+
function parseJson(value) {
|
|
83603
|
+
if (typeof value !== "string") return value;
|
|
83604
|
+
try {
|
|
83605
|
+
return JSON.parse(value);
|
|
83606
|
+
} catch {
|
|
83607
|
+
return;
|
|
83608
|
+
}
|
|
83609
|
+
}
|
|
83610
|
+
function asRecord(value) {
|
|
83611
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? { ...value } : {};
|
|
83612
|
+
}
|
|
83613
|
+
var EMPTY_STATS = {
|
|
83614
|
+
frames: 0,
|
|
83615
|
+
completeFrames: 0,
|
|
83616
|
+
completeness: 0,
|
|
83617
|
+
subjects: 0,
|
|
83618
|
+
modelErrors: 0,
|
|
83619
|
+
hardNegatives: 0,
|
|
83620
|
+
cells: [],
|
|
83621
|
+
cameras: []
|
|
83622
|
+
};
|
|
83623
|
+
/** A stored stats object, trusted only as far as its shape. */
|
|
83624
|
+
function readStats(value) {
|
|
83625
|
+
const raw = asRecord(parseJson(value));
|
|
83626
|
+
const num = (k) => typeof raw[k] === "number" ? raw[k] : 0;
|
|
83627
|
+
const cells = Array.isArray(raw["cells"]) ? raw["cells"] : [];
|
|
83628
|
+
const cameras = Array.isArray(raw["cameras"]) ? raw["cameras"] : [];
|
|
83629
|
+
return {
|
|
83630
|
+
...EMPTY_STATS,
|
|
83631
|
+
frames: num("frames"),
|
|
83632
|
+
completeFrames: num("completeFrames"),
|
|
83633
|
+
completeness: num("completeness"),
|
|
83634
|
+
subjects: num("subjects"),
|
|
83635
|
+
modelErrors: num("modelErrors"),
|
|
83636
|
+
hardNegatives: num("hardNegatives"),
|
|
83637
|
+
cells: cells.map((c) => {
|
|
83638
|
+
const r = asRecord(c);
|
|
83639
|
+
const lighting = r["lighting"] === "day" || r["lighting"] === "night" ? r["lighting"] : "unknown";
|
|
83640
|
+
return {
|
|
83641
|
+
macroClass: String(r["macroClass"] ?? ""),
|
|
83642
|
+
deviceId: Number(r["deviceId"] ?? 0),
|
|
83643
|
+
lighting,
|
|
83644
|
+
subjects: Number(r["subjects"] ?? 0),
|
|
83645
|
+
frames: Number(r["frames"] ?? 0)
|
|
83646
|
+
};
|
|
83647
|
+
}),
|
|
83648
|
+
cameras: cameras.map((c) => {
|
|
83649
|
+
const r = asRecord(c);
|
|
83650
|
+
return {
|
|
83651
|
+
deviceId: Number(r["deviceId"] ?? 0),
|
|
83652
|
+
frames: Number(r["frames"] ?? 0),
|
|
83653
|
+
completeFrames: Number(r["completeFrames"] ?? 0)
|
|
83654
|
+
};
|
|
83655
|
+
})
|
|
83656
|
+
};
|
|
83657
|
+
}
|
|
83658
|
+
var RetrainDatasetVersionStore = class {
|
|
83659
|
+
store;
|
|
83660
|
+
logger;
|
|
83661
|
+
constructor(deps) {
|
|
83662
|
+
this.store = deps.store;
|
|
83663
|
+
this.logger = deps.logger;
|
|
83664
|
+
}
|
|
83665
|
+
static async declare(store) {
|
|
83666
|
+
await store.declareCollection.mutate({
|
|
83667
|
+
collection: RETRAIN_DATASET_VERSIONS_COLLECTION,
|
|
83668
|
+
columns: [...COLUMNS],
|
|
83669
|
+
indexes: [...INDEXES]
|
|
83670
|
+
});
|
|
83671
|
+
}
|
|
83672
|
+
insert = async (version) => {
|
|
83673
|
+
await this.store.set.mutate({
|
|
83674
|
+
collection: RETRAIN_DATASET_VERSIONS_COLLECTION,
|
|
83675
|
+
key: version.id,
|
|
83676
|
+
value: {
|
|
83677
|
+
name: version.name,
|
|
83678
|
+
nameKey: nameKeyOf(version.name),
|
|
83679
|
+
createdAt: version.createdAt,
|
|
83680
|
+
selection: version.selection,
|
|
83681
|
+
layout: version.layout,
|
|
83682
|
+
completeOnly: version.completeOnly,
|
|
83683
|
+
frameIds: version.frameIds,
|
|
83684
|
+
annotationHash: version.annotationHash,
|
|
83685
|
+
manifestSchema: version.manifestSchema,
|
|
83686
|
+
stats: version.stats
|
|
83687
|
+
}
|
|
83688
|
+
});
|
|
83689
|
+
};
|
|
83690
|
+
get = async (id) => {
|
|
83691
|
+
try {
|
|
83692
|
+
const row = await this.store.get.query({
|
|
83693
|
+
collection: RETRAIN_DATASET_VERSIONS_COLLECTION,
|
|
83694
|
+
key: id
|
|
83695
|
+
});
|
|
83696
|
+
return row === null || row === void 0 ? null : this.toVersion(id, asRecord(row));
|
|
83697
|
+
} catch (err) {
|
|
83698
|
+
this.logger.warn("retrain dataset version read failed", { meta: {
|
|
83699
|
+
versionId: id,
|
|
83700
|
+
error: String(err)
|
|
83701
|
+
} });
|
|
83702
|
+
return null;
|
|
83703
|
+
}
|
|
83704
|
+
};
|
|
83705
|
+
list = async () => {
|
|
83706
|
+
return (await this.listRows()).map(summaryOfVersion);
|
|
83707
|
+
};
|
|
83708
|
+
/** Every row WITH its frame list — the migration into `training` reads this. */
|
|
83709
|
+
listRows = async () => {
|
|
83710
|
+
try {
|
|
83711
|
+
return (await this.store.query.query({
|
|
83712
|
+
collection: RETRAIN_DATASET_VERSIONS_COLLECTION,
|
|
83713
|
+
filter: { orderBy: {
|
|
83714
|
+
field: "createdAt",
|
|
83715
|
+
direction: "desc"
|
|
83716
|
+
} }
|
|
83717
|
+
})).map((r) => this.toVersion(r.id, asRecord(r.data)));
|
|
83718
|
+
} catch (err) {
|
|
83719
|
+
this.logger.warn("retrain dataset versions list failed", { meta: { error: String(err) } });
|
|
83720
|
+
return [];
|
|
83721
|
+
}
|
|
83722
|
+
};
|
|
83723
|
+
findIdByName = async (name) => {
|
|
83724
|
+
return (await this.store.query.query({
|
|
83725
|
+
collection: RETRAIN_DATASET_VERSIONS_COLLECTION,
|
|
83726
|
+
filter: {
|
|
83727
|
+
where: { nameKey: nameKeyOf(name) },
|
|
83728
|
+
limit: 1
|
|
83729
|
+
}
|
|
83730
|
+
}))[0]?.id ?? null;
|
|
83731
|
+
};
|
|
83732
|
+
remove = async (id) => {
|
|
83733
|
+
if (await this.get(id) === null) return false;
|
|
83734
|
+
await this.store.delete.mutate({
|
|
83735
|
+
collection: RETRAIN_DATASET_VERSIONS_COLLECTION,
|
|
83736
|
+
key: id
|
|
83737
|
+
});
|
|
83738
|
+
return true;
|
|
83739
|
+
};
|
|
83740
|
+
toVersion(id, data) {
|
|
83741
|
+
const selection = asRecord(parseJson(data["selection"]));
|
|
83742
|
+
const deviceIds = numberList(selection["deviceIds"]);
|
|
83743
|
+
const collectionIds = stringList(selection["collectionIds"]);
|
|
83744
|
+
const frameIdsSel = stringList(selection["frameIds"]);
|
|
83745
|
+
return {
|
|
83746
|
+
id,
|
|
83747
|
+
name: String(data["name"] ?? ""),
|
|
83748
|
+
createdAt: Number(data["createdAt"] ?? 0),
|
|
83749
|
+
selection: {
|
|
83750
|
+
...deviceIds !== void 0 ? { deviceIds } : {},
|
|
83751
|
+
...collectionIds !== void 0 ? { collectionIds } : {},
|
|
83752
|
+
...frameIdsSel !== void 0 ? { frameIds: frameIdsSel } : {}
|
|
83753
|
+
},
|
|
83754
|
+
layout: data["layout"] === "crops" ? "crops" : "frames",
|
|
83755
|
+
completeOnly: data["completeOnly"] === true || data["completeOnly"] === 1,
|
|
83756
|
+
frameIds: stringList(parseJson(data["frameIds"])) ?? [],
|
|
83757
|
+
annotationHash: String(data["annotationHash"] ?? ""),
|
|
83758
|
+
manifestSchema: String(data["manifestSchema"] ?? ""),
|
|
83759
|
+
stats: readStats(data["stats"])
|
|
83760
|
+
};
|
|
83761
|
+
}
|
|
83762
|
+
};
|
|
83763
|
+
//#endregion
|
|
83510
83764
|
//#region src/pipeline-analytics/retrain/retrain-geometry.ts
|
|
83511
83765
|
function clamp01(v) {
|
|
83512
83766
|
return v < 0 ? 0 : v > 1 ? 1 : v;
|
|
@@ -83795,13 +84049,6 @@ function filterOf(selection, completeOnly) {
|
|
|
83795
84049
|
...completeOnly ? { completeOnly: true } : {}
|
|
83796
84050
|
};
|
|
83797
84051
|
}
|
|
83798
|
-
function summaryOf(version) {
|
|
83799
|
-
const { frameIds, ...rest } = version;
|
|
83800
|
-
return {
|
|
83801
|
-
...rest,
|
|
83802
|
-
frameCount: frameIds.length
|
|
83803
|
-
};
|
|
83804
|
-
}
|
|
83805
84052
|
var RetrainDatasetService = class {
|
|
83806
84053
|
deps;
|
|
83807
84054
|
constructor(deps) {
|
|
@@ -83854,11 +84101,11 @@ var RetrainDatasetService = class {
|
|
|
83854
84101
|
} });
|
|
83855
84102
|
return {
|
|
83856
84103
|
kind: "created",
|
|
83857
|
-
version:
|
|
84104
|
+
version: summaryOfVersion(version)
|
|
83858
84105
|
};
|
|
83859
84106
|
}
|
|
83860
84107
|
async list() {
|
|
83861
|
-
return
|
|
84108
|
+
return this.deps.versions.list();
|
|
83862
84109
|
}
|
|
83863
84110
|
async delete(input) {
|
|
83864
84111
|
const deleted = await this.deps.versions.remove(input.id);
|
|
@@ -83916,246 +84163,62 @@ var RetrainDatasetService = class {
|
|
|
83916
84163
|
}
|
|
83917
84164
|
};
|
|
83918
84165
|
//#endregion
|
|
83919
|
-
//#region src/pipeline-analytics/retrain/retrain-dataset-version-
|
|
83920
|
-
|
|
83921
|
-
|
|
83922
|
-
|
|
83923
|
-
|
|
83924
|
-
|
|
83925
|
-
* is a new version."
|
|
83926
|
-
* retention="none — rows leave only when the operator deletes the version. The
|
|
83927
|
-
* frames themselves are owned by the retrain corpus; deleting a version deletes no
|
|
83928
|
-
* frame, and deleting a frame does not edit a version (verify reports it missing)."
|
|
83929
|
-
*/
|
|
83930
|
-
var RETRAIN_DATASET_VERSIONS_COLLECTION = "pipeline-analytics:retrain-dataset-versions";
|
|
83931
|
-
var COLUMNS = [
|
|
83932
|
-
{
|
|
83933
|
-
name: "id",
|
|
83934
|
-
type: "TEXT",
|
|
83935
|
-
primaryKey: true,
|
|
83936
|
-
notNull: true
|
|
83937
|
-
},
|
|
83938
|
-
{
|
|
83939
|
-
name: "name",
|
|
83940
|
-
type: "TEXT",
|
|
83941
|
-
notNull: true
|
|
83942
|
-
},
|
|
83943
|
-
{
|
|
83944
|
-
name: "nameKey",
|
|
83945
|
-
type: "TEXT",
|
|
83946
|
-
notNull: true
|
|
83947
|
-
},
|
|
83948
|
-
{
|
|
83949
|
-
name: "createdAt",
|
|
83950
|
-
type: "INTEGER",
|
|
83951
|
-
notNull: true
|
|
83952
|
-
},
|
|
83953
|
-
{
|
|
83954
|
-
name: "selection",
|
|
83955
|
-
type: "JSON",
|
|
83956
|
-
notNull: true
|
|
83957
|
-
},
|
|
83958
|
-
{
|
|
83959
|
-
name: "layout",
|
|
83960
|
-
type: "TEXT",
|
|
83961
|
-
notNull: true
|
|
83962
|
-
},
|
|
83963
|
-
{
|
|
83964
|
-
name: "completeOnly",
|
|
83965
|
-
type: "BOOLEAN",
|
|
83966
|
-
notNull: true
|
|
83967
|
-
},
|
|
83968
|
-
{
|
|
83969
|
-
name: "frameIds",
|
|
83970
|
-
type: "JSON",
|
|
83971
|
-
notNull: true
|
|
83972
|
-
},
|
|
83973
|
-
{
|
|
83974
|
-
name: "annotationHash",
|
|
83975
|
-
type: "TEXT",
|
|
83976
|
-
notNull: true
|
|
83977
|
-
},
|
|
83978
|
-
{
|
|
83979
|
-
name: "manifestSchema",
|
|
83980
|
-
type: "TEXT",
|
|
83981
|
-
notNull: true
|
|
83982
|
-
},
|
|
83983
|
-
{
|
|
83984
|
-
name: "stats",
|
|
83985
|
-
type: "JSON",
|
|
83986
|
-
notNull: true
|
|
83987
|
-
}
|
|
83988
|
-
];
|
|
83989
|
-
var INDEXES = [{
|
|
83990
|
-
name: "idx_retrain_dataset_versions_name_key",
|
|
83991
|
-
columns: ["nameKey"],
|
|
83992
|
-
unique: true
|
|
83993
|
-
}, {
|
|
83994
|
-
name: "idx_retrain_dataset_versions_created",
|
|
83995
|
-
columns: ["createdAt"]
|
|
83996
|
-
}];
|
|
83997
|
-
var nameKeyOf = (name) => name.trim().toLowerCase();
|
|
83998
|
-
function numberList(value) {
|
|
83999
|
-
return Array.isArray(value) ? value.filter((v) => typeof v === "number") : void 0;
|
|
84000
|
-
}
|
|
84001
|
-
function stringList(value) {
|
|
84002
|
-
return Array.isArray(value) ? value.filter((v) => typeof v === "string") : void 0;
|
|
84003
|
-
}
|
|
84004
|
-
function parseJson(value) {
|
|
84005
|
-
if (typeof value !== "string") return value;
|
|
84006
|
-
try {
|
|
84007
|
-
return JSON.parse(value);
|
|
84008
|
-
} catch {
|
|
84009
|
-
return;
|
|
84010
|
-
}
|
|
84011
|
-
}
|
|
84012
|
-
function asRecord(value) {
|
|
84013
|
-
return value !== null && typeof value === "object" && !Array.isArray(value) ? { ...value } : {};
|
|
84014
|
-
}
|
|
84015
|
-
var EMPTY_STATS = {
|
|
84016
|
-
frames: 0,
|
|
84017
|
-
completeFrames: 0,
|
|
84018
|
-
completeness: 0,
|
|
84019
|
-
subjects: 0,
|
|
84020
|
-
modelErrors: 0,
|
|
84021
|
-
hardNegatives: 0,
|
|
84022
|
-
cells: [],
|
|
84023
|
-
cameras: []
|
|
84024
|
-
};
|
|
84025
|
-
/** A stored stats object, trusted only as far as its shape. */
|
|
84026
|
-
function readStats(value) {
|
|
84027
|
-
const raw = asRecord(parseJson(value));
|
|
84028
|
-
const num = (k) => typeof raw[k] === "number" ? raw[k] : 0;
|
|
84029
|
-
const cells = Array.isArray(raw["cells"]) ? raw["cells"] : [];
|
|
84030
|
-
const cameras = Array.isArray(raw["cameras"]) ? raw["cameras"] : [];
|
|
84031
|
-
return {
|
|
84032
|
-
...EMPTY_STATS,
|
|
84033
|
-
frames: num("frames"),
|
|
84034
|
-
completeFrames: num("completeFrames"),
|
|
84035
|
-
completeness: num("completeness"),
|
|
84036
|
-
subjects: num("subjects"),
|
|
84037
|
-
modelErrors: num("modelErrors"),
|
|
84038
|
-
hardNegatives: num("hardNegatives"),
|
|
84039
|
-
cells: cells.map((c) => {
|
|
84040
|
-
const r = asRecord(c);
|
|
84041
|
-
const lighting = r["lighting"] === "day" || r["lighting"] === "night" ? r["lighting"] : "unknown";
|
|
84042
|
-
return {
|
|
84043
|
-
macroClass: String(r["macroClass"] ?? ""),
|
|
84044
|
-
deviceId: Number(r["deviceId"] ?? 0),
|
|
84045
|
-
lighting,
|
|
84046
|
-
subjects: Number(r["subjects"] ?? 0),
|
|
84047
|
-
frames: Number(r["frames"] ?? 0)
|
|
84048
|
-
};
|
|
84049
|
-
}),
|
|
84050
|
-
cameras: cameras.map((c) => {
|
|
84051
|
-
const r = asRecord(c);
|
|
84052
|
-
return {
|
|
84053
|
-
deviceId: Number(r["deviceId"] ?? 0),
|
|
84054
|
-
frames: Number(r["frames"] ?? 0),
|
|
84055
|
-
completeFrames: Number(r["completeFrames"] ?? 0)
|
|
84056
|
-
};
|
|
84057
|
-
})
|
|
84058
|
-
};
|
|
84059
|
-
}
|
|
84060
|
-
var RetrainDatasetVersionStore = class {
|
|
84061
|
-
store;
|
|
84062
|
-
logger;
|
|
84166
|
+
//#region src/pipeline-analytics/retrain/retrain-dataset-version-routing.ts
|
|
84167
|
+
var RoutingDatasetVersionRepository = class {
|
|
84168
|
+
deps;
|
|
84169
|
+
lastPresence = false;
|
|
84170
|
+
migrated = false;
|
|
84171
|
+
migration = null;
|
|
84063
84172
|
constructor(deps) {
|
|
84064
|
-
this.
|
|
84065
|
-
this.logger = deps.logger;
|
|
84066
|
-
}
|
|
84067
|
-
static async declare(store) {
|
|
84068
|
-
await store.declareCollection.mutate({
|
|
84069
|
-
collection: RETRAIN_DATASET_VERSIONS_COLLECTION,
|
|
84070
|
-
columns: [...COLUMNS],
|
|
84071
|
-
indexes: [...INDEXES]
|
|
84072
|
-
});
|
|
84173
|
+
this.deps = deps;
|
|
84073
84174
|
}
|
|
84074
84175
|
insert = async (version) => {
|
|
84075
|
-
await this.
|
|
84076
|
-
|
|
84077
|
-
|
|
84078
|
-
|
|
84079
|
-
|
|
84080
|
-
nameKey: nameKeyOf(version.name),
|
|
84081
|
-
createdAt: version.createdAt,
|
|
84082
|
-
selection: version.selection,
|
|
84083
|
-
layout: version.layout,
|
|
84084
|
-
completeOnly: version.completeOnly,
|
|
84085
|
-
frameIds: version.frameIds,
|
|
84086
|
-
annotationHash: version.annotationHash,
|
|
84087
|
-
manifestSchema: version.manifestSchema,
|
|
84088
|
-
stats: version.stats
|
|
84089
|
-
}
|
|
84090
|
-
});
|
|
84176
|
+
if (await this.route() === "local") {
|
|
84177
|
+
await this.deps.local.insert(version);
|
|
84178
|
+
return;
|
|
84179
|
+
}
|
|
84180
|
+
await this.deps.training.put(version);
|
|
84091
84181
|
};
|
|
84092
|
-
get = async (id) =>
|
|
84182
|
+
get = async (id) => await this.route() === "local" ? this.deps.local.get(id) : this.deps.training.get(id);
|
|
84183
|
+
list = async () => await this.route() === "local" ? this.deps.local.list() : this.deps.training.list();
|
|
84184
|
+
findIdByName = async (name) => await this.route() === "local" ? this.deps.local.findIdByName(name) : this.deps.training.findIdByName(name);
|
|
84185
|
+
remove = async (id) => await this.route() === "local" ? this.deps.local.remove(id) : this.deps.training.remove(id);
|
|
84186
|
+
async route() {
|
|
84187
|
+
if (!await this.presence()) return "local";
|
|
84188
|
+
if (this.migrated) return "training";
|
|
84189
|
+
return await this.migrateOnce() ? "training" : "local";
|
|
84190
|
+
}
|
|
84191
|
+
async presence() {
|
|
84192
|
+
let answer;
|
|
84093
84193
|
try {
|
|
84094
|
-
|
|
84095
|
-
|
|
84096
|
-
|
|
84097
|
-
});
|
|
84098
|
-
return row === null || row === void 0 ? null : this.toVersion(id, asRecord(row));
|
|
84099
|
-
} catch (err) {
|
|
84100
|
-
this.logger.warn("retrain dataset version read failed", { meta: {
|
|
84101
|
-
versionId: id,
|
|
84102
|
-
error: String(err)
|
|
84103
|
-
} });
|
|
84104
|
-
return null;
|
|
84194
|
+
answer = await this.deps.trainingPresent();
|
|
84195
|
+
} catch {
|
|
84196
|
+
answer = null;
|
|
84105
84197
|
}
|
|
84106
|
-
|
|
84107
|
-
|
|
84198
|
+
if (answer !== null) this.lastPresence = answer;
|
|
84199
|
+
return this.lastPresence;
|
|
84200
|
+
}
|
|
84201
|
+
migrateOnce() {
|
|
84202
|
+
if (this.migration === null) this.migration = this.migrate().finally(() => {
|
|
84203
|
+
this.migration = null;
|
|
84204
|
+
});
|
|
84205
|
+
return this.migration;
|
|
84206
|
+
}
|
|
84207
|
+
async migrate() {
|
|
84208
|
+
const { local, training, logger } = this.deps;
|
|
84108
84209
|
try {
|
|
84109
|
-
|
|
84110
|
-
|
|
84111
|
-
|
|
84112
|
-
|
|
84113
|
-
|
|
84114
|
-
|
|
84115
|
-
|
|
84210
|
+
const rows = await local.listRows();
|
|
84211
|
+
for (const row of rows) {
|
|
84212
|
+
await training.put(row);
|
|
84213
|
+
await local.remove(row.id);
|
|
84214
|
+
}
|
|
84215
|
+
this.migrated = true;
|
|
84216
|
+
if (rows.length > 0) logger.info("retrain dataset versions moved to the training capability", { meta: { versions: rows.length } });
|
|
84217
|
+
return true;
|
|
84116
84218
|
} catch (err) {
|
|
84117
|
-
|
|
84118
|
-
return
|
|
84219
|
+
logger.warn("retrain dataset versions could not move to training yet — kept local", { meta: { error: err instanceof Error ? err.message : String(err) } });
|
|
84220
|
+
return false;
|
|
84119
84221
|
}
|
|
84120
|
-
};
|
|
84121
|
-
findIdByName = async (name) => {
|
|
84122
|
-
return (await this.store.query.query({
|
|
84123
|
-
collection: RETRAIN_DATASET_VERSIONS_COLLECTION,
|
|
84124
|
-
filter: {
|
|
84125
|
-
where: { nameKey: nameKeyOf(name) },
|
|
84126
|
-
limit: 1
|
|
84127
|
-
}
|
|
84128
|
-
}))[0]?.id ?? null;
|
|
84129
|
-
};
|
|
84130
|
-
remove = async (id) => {
|
|
84131
|
-
if (await this.get(id) === null) return false;
|
|
84132
|
-
await this.store.delete.mutate({
|
|
84133
|
-
collection: RETRAIN_DATASET_VERSIONS_COLLECTION,
|
|
84134
|
-
key: id
|
|
84135
|
-
});
|
|
84136
|
-
return true;
|
|
84137
|
-
};
|
|
84138
|
-
toVersion(id, data) {
|
|
84139
|
-
const selection = asRecord(parseJson(data["selection"]));
|
|
84140
|
-
const deviceIds = numberList(selection["deviceIds"]);
|
|
84141
|
-
const collectionIds = stringList(selection["collectionIds"]);
|
|
84142
|
-
const frameIdsSel = stringList(selection["frameIds"]);
|
|
84143
|
-
return {
|
|
84144
|
-
id,
|
|
84145
|
-
name: String(data["name"] ?? ""),
|
|
84146
|
-
createdAt: Number(data["createdAt"] ?? 0),
|
|
84147
|
-
selection: {
|
|
84148
|
-
...deviceIds !== void 0 ? { deviceIds } : {},
|
|
84149
|
-
...collectionIds !== void 0 ? { collectionIds } : {},
|
|
84150
|
-
...frameIdsSel !== void 0 ? { frameIds: frameIdsSel } : {}
|
|
84151
|
-
},
|
|
84152
|
-
layout: data["layout"] === "crops" ? "crops" : "frames",
|
|
84153
|
-
completeOnly: data["completeOnly"] === true || data["completeOnly"] === 1,
|
|
84154
|
-
frameIds: stringList(parseJson(data["frameIds"])) ?? [],
|
|
84155
|
-
annotationHash: String(data["annotationHash"] ?? ""),
|
|
84156
|
-
manifestSchema: String(data["manifestSchema"] ?? ""),
|
|
84157
|
-
stats: readStats(data["stats"])
|
|
84158
|
-
};
|
|
84159
84222
|
}
|
|
84160
84223
|
};
|
|
84161
84224
|
//#endregion
|
|
@@ -84825,6 +84888,61 @@ async function writeRetrainArchiveTo(deps) {
|
|
|
84825
84888
|
return stats;
|
|
84826
84889
|
}
|
|
84827
84890
|
//#endregion
|
|
84891
|
+
//#region src/pipeline-analytics/retrain/retrain-archive-offer.ts
|
|
84892
|
+
/** A reader silent this long is gone; the body is cut. */
|
|
84893
|
+
var RETRAIN_OFFER_STALL_MS = 6e4;
|
|
84894
|
+
function responseArchiveSink(res, stallMs = RETRAIN_OFFER_STALL_MS) {
|
|
84895
|
+
let closed = false;
|
|
84896
|
+
res.once("close", () => {
|
|
84897
|
+
closed = true;
|
|
84898
|
+
});
|
|
84899
|
+
return { write: (chunk) => new Promise((resolve, reject) => {
|
|
84900
|
+
if (closed || res.destroyed) {
|
|
84901
|
+
reject(/* @__PURE__ */ new Error("retrain archive reader went away"));
|
|
84902
|
+
return;
|
|
84903
|
+
}
|
|
84904
|
+
if (res.write(chunk)) {
|
|
84905
|
+
resolve();
|
|
84906
|
+
return;
|
|
84907
|
+
}
|
|
84908
|
+
const onDrain = () => {
|
|
84909
|
+
cleanup();
|
|
84910
|
+
resolve();
|
|
84911
|
+
};
|
|
84912
|
+
const onClose = () => {
|
|
84913
|
+
cleanup();
|
|
84914
|
+
reject(/* @__PURE__ */ new Error("retrain archive reader went away"));
|
|
84915
|
+
};
|
|
84916
|
+
const timer = setTimeout(() => {
|
|
84917
|
+
cleanup();
|
|
84918
|
+
res.destroy(/* @__PURE__ */ new Error("retrain archive reader stalled"));
|
|
84919
|
+
reject(/* @__PURE__ */ new Error(`retrain archive reader stalled for ${stallMs} ms`));
|
|
84920
|
+
}, stallMs);
|
|
84921
|
+
const cleanup = () => {
|
|
84922
|
+
clearTimeout(timer);
|
|
84923
|
+
res.off("drain", onDrain);
|
|
84924
|
+
res.off("close", onClose);
|
|
84925
|
+
};
|
|
84926
|
+
res.once("drain", onDrain);
|
|
84927
|
+
res.once("close", onClose);
|
|
84928
|
+
}) };
|
|
84929
|
+
}
|
|
84930
|
+
/**
|
|
84931
|
+
* Serve one archive: headers, the writer's chunks, the end. A writer that
|
|
84932
|
+
* throws destroys the response (the reader sees a truncated body, never a
|
|
84933
|
+
* clean end) and the error propagates to the caller's log.
|
|
84934
|
+
*/
|
|
84935
|
+
async function serveRetrainArchive(_req, res, writeArchive) {
|
|
84936
|
+
res.writeHead(200, { "content-type": "application/x-tar" });
|
|
84937
|
+
try {
|
|
84938
|
+
await writeArchive(responseArchiveSink(res));
|
|
84939
|
+
await new Promise((resolve) => res.end(resolve));
|
|
84940
|
+
} catch (err) {
|
|
84941
|
+
res.destroy(err instanceof Error ? err : new Error(String(err)));
|
|
84942
|
+
throw err;
|
|
84943
|
+
}
|
|
84944
|
+
}
|
|
84945
|
+
//#endregion
|
|
84828
84946
|
//#region src/pipeline-analytics/retrain/retrain-crop-cut.ts
|
|
84829
84947
|
/**
|
|
84830
84948
|
* The `crops` layout's cutter: one subject crop from a retrain frame copy.
|
|
@@ -111121,8 +111239,26 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends require_clip_m
|
|
|
111121
111239
|
logger: logger.child("RetrainDatasets")
|
|
111122
111240
|
});
|
|
111123
111241
|
this.retrainDatasetVersions = retrainDatasetVersions;
|
|
111242
|
+
const routedDatasetVersions = new RoutingDatasetVersionRepository({
|
|
111243
|
+
local: retrainDatasetVersions,
|
|
111244
|
+
training: {
|
|
111245
|
+
put: (row) => api.training.putDatasetVersion.mutate({ ...row }),
|
|
111246
|
+
get: (id) => api.training.getDatasetVersion.query({ id }),
|
|
111247
|
+
list: async () => api.training.listDatasetVersions.query({}),
|
|
111248
|
+
findIdByName: async (name) => (await api.training.findDatasetVersionIdByName.query({ name })).id,
|
|
111249
|
+
remove: async (id) => (await api.training.removeDatasetVersion.mutate({ id })).removed
|
|
111250
|
+
},
|
|
111251
|
+
trainingPresent: async () => {
|
|
111252
|
+
try {
|
|
111253
|
+
return (await api.addons.listCapabilityProviders.query({ capName: "training" })).length > 0;
|
|
111254
|
+
} catch {
|
|
111255
|
+
return null;
|
|
111256
|
+
}
|
|
111257
|
+
},
|
|
111258
|
+
logger: logger.child("RetrainDatasets")
|
|
111259
|
+
});
|
|
111124
111260
|
this.retrainDatasets = new RetrainDatasetService({
|
|
111125
|
-
versions:
|
|
111261
|
+
versions: routedDatasetVersions,
|
|
111126
111262
|
buildPlan: (request) => this.buildRetrainExportPlan(request),
|
|
111127
111263
|
now: () => Date.now(),
|
|
111128
111264
|
newId: () => (0, node_crypto.randomUUID)(),
|
|
@@ -112487,6 +112623,51 @@ var PipelineAnalyticsAddon = class PipelineAnalyticsAddon extends require_clip_m
|
|
|
112487
112623
|
}
|
|
112488
112624
|
}
|
|
112489
112625
|
/**
|
|
112626
|
+
* The same archive behind a one-shot `ctx.peerBytes` ticket (D805): the
|
|
112627
|
+
* caller (a training provider, same host) dials it at once and streams it
|
|
112628
|
+
* into its cloud. The plan is built now, so an empty or unknown selection
|
|
112629
|
+
* fails here rather than as an empty body; the bytes and their sha256 are
|
|
112630
|
+
* the reader's to measure.
|
|
112631
|
+
*/
|
|
112632
|
+
async offerRetrainArchive(input) {
|
|
112633
|
+
const frameStore = this.retrainFrames;
|
|
112634
|
+
if (frameStore === null) throw new Error("retrain export: analytics stores unavailable");
|
|
112635
|
+
const peerBytes = this.ctx.peerBytes;
|
|
112636
|
+
if (peerBytes === void 0) throw new Error("retrain archive offer: no peer-bytes facility");
|
|
112637
|
+
const plan = await this.buildRetrainExportPlan({
|
|
112638
|
+
filter: input.filter,
|
|
112639
|
+
layout: input.layout
|
|
112640
|
+
});
|
|
112641
|
+
if (plan.frames.length === 0) throw new Error("retrain archive offer: the selection holds no frame");
|
|
112642
|
+
const logger = this.ctx.logger;
|
|
112643
|
+
const offered = await peerBytes.offer({
|
|
112644
|
+
label: "retrain-archive",
|
|
112645
|
+
handler: (req, res) => serveRetrainArchive(req, res, async (sink) => {
|
|
112646
|
+
await writeRetrainArchiveTo({
|
|
112647
|
+
plan,
|
|
112648
|
+
readFrame: (frame) => frameStore.readFrame(frame),
|
|
112649
|
+
cutCrop: this.retrainCropCutter(),
|
|
112650
|
+
sink,
|
|
112651
|
+
markExported: (frameIds, batchId) => this.markRetrainExported(frameIds, batchId),
|
|
112652
|
+
logger,
|
|
112653
|
+
now: () => Date.now()
|
|
112654
|
+
});
|
|
112655
|
+
}).catch((err) => {
|
|
112656
|
+
logger.warn("retrain archive offer failed mid-body", { meta: { error: require_clip_model_registry.errMsg(err) } });
|
|
112657
|
+
})
|
|
112658
|
+
});
|
|
112659
|
+
if (offered.kind === "refused") throw new Error(`retrain archive offer refused: ${offered.code} — ${offered.detail}`);
|
|
112660
|
+
logger.info("retrain archive offered", { meta: {
|
|
112661
|
+
frames: plan.frames.length,
|
|
112662
|
+
layout: input.layout
|
|
112663
|
+
} });
|
|
112664
|
+
return {
|
|
112665
|
+
ticket: offered.ticket,
|
|
112666
|
+
frames: plan.frames.length,
|
|
112667
|
+
annotations: plan.manifest.annotationCount
|
|
112668
|
+
};
|
|
112669
|
+
}
|
|
112670
|
+
/**
|
|
112490
112671
|
* The store seam the `debug.*` actions and the debug bundle both read through.
|
|
112491
112672
|
*
|
|
112492
112673
|
* Index reads, plus ONE existence probe per media row — never a blob. The
|