@camstack/addon-pipeline 1.2.335 → 1.2.336
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{custom-model-catalog-Buehc666.mjs → custom-model-catalog-DSN-2fxO.mjs} +1 -1
- package/dist/{custom-model-catalog-CL-xJa8w.js → custom-model-catalog-jyP-7H7P.js} +1 -1
- package/dist/detection-pipeline/index.js +1 -1
- package/dist/detection-pipeline/index.mjs +1 -1
- package/dist/pipeline-runner/index.js +1 -1
- package/dist/pipeline-runner/index.mjs +1 -1
- package/package.json +1 -1
- package/python/coreml_load_probe.py +223 -0
- package/python/inference_pool.py +29 -3
- package/python/test_coreml_load_probe.py +295 -0
- package/python/test_inference_pool_coreml_cache.py +37 -1
|
@@ -426,7 +426,7 @@ var yolo26Reduced = (tier, res, sizes) => {
|
|
|
426
426
|
sizeMB: sizes.onnx
|
|
427
427
|
},
|
|
428
428
|
coreml: {
|
|
429
|
-
url: hf(`objectDetection/yolo26/coreml/camstack-yolo26${tier}-${res}.mlpackage`),
|
|
429
|
+
url: hf(`objectDetection/yolo26/coreml/camstack-yolo26${tier}-${res}-static.mlpackage`),
|
|
430
430
|
sizeMB: sizes.coreml,
|
|
431
431
|
isDirectory: true,
|
|
432
432
|
files: [...MLPACKAGE_FILES],
|
|
@@ -426,7 +426,7 @@ var yolo26Reduced = (tier, res, sizes) => {
|
|
|
426
426
|
sizeMB: sizes.onnx
|
|
427
427
|
},
|
|
428
428
|
coreml: {
|
|
429
|
-
url: hf(`objectDetection/yolo26/coreml/camstack-yolo26${tier}-${res}.mlpackage`),
|
|
429
|
+
url: hf(`objectDetection/yolo26/coreml/camstack-yolo26${tier}-${res}-static.mlpackage`),
|
|
430
430
|
sizeMB: sizes.coreml,
|
|
431
431
|
isDirectory: true,
|
|
432
432
|
files: [...MLPACKAGE_FILES],
|
|
@@ -5,7 +5,7 @@ Object.defineProperties(exports, {
|
|
|
5
5
|
const require_chunk = require("../chunk-emK7D4bc.js");
|
|
6
6
|
const require_dist = require("../dist-C5DlS_lW.js");
|
|
7
7
|
const require_node = require("../node-BUafKArW.js");
|
|
8
|
-
const require_custom_model_catalog = require("../custom-model-catalog-
|
|
8
|
+
const require_custom_model_catalog = require("../custom-model-catalog-jyP-7H7P.js");
|
|
9
9
|
const require_lazy_sharp = require("../lazy-sharp-BuwKlQLR.js");
|
|
10
10
|
const require_process_memory = require("../process-memory-DWOQCbg5.js");
|
|
11
11
|
let node_os = require("node:os");
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { n as __require } from "../chunk-DnnnRqeS.mjs";
|
|
2
2
|
import { At as loadContributionCapability, Bn as parseJsonUnknown, Bt as overlayClusterStepSettings, Dn as BaseAddon, Fn as hydrateSchema, G as YAMNET_TO_MACRO, Gn as array, Gt as pipelineExecutorCapability, Ht as pickClusterStepSettings, J as applyModelBoundSettings, N as PoolMemoryWatchdog, O as LABEL_SUPPLY_MACRO, Ot as linkedLabelStepId, Qt as resolveClusterStepModelId, Tn as errMsg, Tt as hfModelUrl, Un as sleep, Zn as object, cn as stepModelRunsOnFormat, d as CLUSTER_MODEL_SCOPED_STEPS, er as string, ft as detectionPipelineCapability, g as DEFAULT_CLUSTER_STEP_SETTINGS, gt as evaluateZoneRules, ht as enumerateInferenceDevices, jn as createEvent, n as APPLE_SA_TO_MACRO, nr as EventCategory, on as rootSuppliesOf, pn as supportedRuntimes$1, rn as resolvePoolMemoryPolicy, rt as containingParentIndex, sn as runtimeDevices$1, st as defaultDeviceFor$1, tr as union, zn as nodePin } from "../dist-CPhDwlSN.mjs";
|
|
3
3
|
import { s as readProcessCost } from "../node-DYfEOwjW.mjs";
|
|
4
|
-
import { C as resolveModelDefaultConfidence, S as landmarkPrecisionVerdict, _ as ALL_STEPS, b as getStepDefinition, f as localFrameRegistry, g as ALL_PIPELINE_STEPS, l as resolveDefaultDetectionModel, n as mergeCustomModels, o as deviceKeyOf, r as buildSubtreeCanRun, s as resolveDeviceEngine, t as CustomModelCatalog, u as macroGateVerdict, v as getDefaultModelForFormat, x as resolveModelForFormat, y as getStep } from "../custom-model-catalog-
|
|
4
|
+
import { C as resolveModelDefaultConfidence, S as landmarkPrecisionVerdict, _ as ALL_STEPS, b as getStepDefinition, f as localFrameRegistry, g as ALL_PIPELINE_STEPS, l as resolveDefaultDetectionModel, n as mergeCustomModels, o as deviceKeyOf, r as buildSubtreeCanRun, s as resolveDeviceEngine, t as CustomModelCatalog, u as macroGateVerdict, v as getDefaultModelForFormat, x as resolveModelForFormat, y as getStep } from "../custom-model-catalog-DSN-2fxO.mjs";
|
|
5
5
|
import { t as getSharp } from "../lazy-sharp-DXsqwpph.mjs";
|
|
6
6
|
import { n as pickNodePlatformArch, t as readProcessMemory } from "../process-memory-WPRYnN8c.mjs";
|
|
7
7
|
import * as os from "node:os";
|
|
@@ -5,7 +5,7 @@ Object.defineProperties(exports, {
|
|
|
5
5
|
const require_chunk = require("../chunk-emK7D4bc.js");
|
|
6
6
|
const require_dist = require("../dist-C5DlS_lW.js");
|
|
7
7
|
const require_node = require("../node-BUafKArW.js");
|
|
8
|
-
const require_custom_model_catalog = require("../custom-model-catalog-
|
|
8
|
+
const require_custom_model_catalog = require("../custom-model-catalog-jyP-7H7P.js");
|
|
9
9
|
const require_hub_hostname = require("../hub-hostname-DAJXlOgV.js");
|
|
10
10
|
const require_lazy_sharp = require("../lazy-sharp-BuwKlQLR.js");
|
|
11
11
|
const require_restream_intent = require("../restream-intent-7uDNqMKs.js");
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { $ as cameraOwnsMotionViaSensor, A as NativeLeaseAdmissionSchema, At as loadContributionCapability, Dn as BaseAddon, Gn as array, Jn as lazy, Kn as boolean, Kt as pipelineRunnerCapability, Mn as customAction, Nn as defineCustomActions, T as FailureCounters, Tn as errMsg, Ut as pickDetailCropConvention, Wn as _enum, Wt as pickNativeLeaseOverride, Xn as number, Yn as literal, Zn as object, _ as DEFAULT_DETAIL_CROP_CONVENTION, a as BackgroundRunRegistry, an as resolveRootSupply, bt as failureContributionCapability, dt as deriveRootFaceRefineRect, er as string, j as NativeLeaseSettingsSchema, jn as createEvent, kt as linkedStepOf, lt as deriveDetailCropRect, m as CROP_SUPPLY_SOURCES, nn as resolveModelBoundSetting, nr as EventCategory, on as rootSuppliesOf, q as allowedSuppliedEffect, y as DEFAULT_NATIVE_LEASE_SETTINGS, z as ROOT_SUPPLIABLE_STEP_IDS, zn as nodePin } from "../dist-CPhDwlSN.mjs";
|
|
2
2
|
import { i as NO_COST_CLAIM, t as ChildCostRegistry } from "../node-DYfEOwjW.mjs";
|
|
3
|
-
import { a as stepsForElectedPool, b as getStepDefinition, d as rootObjectGate, f as localFrameRegistry, h as resolveFrameViewGeometry, i as placeDetailStep, l as resolveDefaultDetectionModel, m as createSessionDecodeCoordinator, p as isSessionDecodeReadGone, s as resolveDeviceEngine, t as CustomModelCatalog, u as macroGateVerdict, y as getStep } from "../custom-model-catalog-
|
|
3
|
+
import { a as stepsForElectedPool, b as getStepDefinition, d as rootObjectGate, f as localFrameRegistry, h as resolveFrameViewGeometry, i as placeDetailStep, l as resolveDefaultDetectionModel, m as createSessionDecodeCoordinator, p as isSessionDecodeReadGone, s as resolveDeviceEngine, t as CustomModelCatalog, u as macroGateVerdict, y as getStep } from "../custom-model-catalog-DSN-2fxO.mjs";
|
|
4
4
|
import { t as resolveHubHostname } from "../hub-hostname-cCknRYKj.mjs";
|
|
5
5
|
import { t as getSharp } from "../lazy-sharp-DXsqwpph.mjs";
|
|
6
6
|
import { c as withDetectionIntent, o as preRollRequestOf, t as DETECTION_PRE_ROLL_REQUEST_MS } from "../restream-intent-DyGmLAU9.mjs";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@camstack/addon-pipeline",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.336",
|
|
4
4
|
"description": "Pipeline bundle — runner, detection, motion, audio + stream broker. Multi-entry npm package shipping pipeline addons under a single bundle.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"camstack",
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""Load a CoreML package in a CHILD process before the pool loads it in-process.
|
|
2
|
+
|
|
3
|
+
Why (2026-10-07, M3 Pro, macOS 26.2): the batch-flexible YOLO26 @320/@256
|
|
4
|
+
packages abort ANY process that loads them — SIGABRT from libmalloc inside
|
|
5
|
+
`Espresso::generic_gather_nd_kernel::top_shapes_for_bottom_shapes`, on every
|
|
6
|
+
compute unit, with coremltools 8.3 and 9.0 alike. Python cannot catch an
|
|
7
|
+
abort, so the pool worker died, the host respawned it, and the respawned
|
|
8
|
+
worker went straight back into the same load.
|
|
9
|
+
|
|
10
|
+
So the first load of a package (per fingerprint, compute unit and OS build)
|
|
11
|
+
happens in a throwaway interpreter. The verdict is remembered next to the
|
|
12
|
+
compiled-model cache:
|
|
13
|
+
|
|
14
|
+
ok -> load in-process, never probe this package again
|
|
15
|
+
crashed -> refuse with `CoremlLoadCrashed` (reason `native-load-crashed`),
|
|
16
|
+
now and on every later load, without re-probing
|
|
17
|
+
inconclusive -> (a Python exception in the child, a probe past its bound, a
|
|
18
|
+
child that could not start) remembered NOWHERE; the
|
|
19
|
+
in-process load runs exactly as it did before this gate
|
|
20
|
+
|
|
21
|
+
Only a death by SIGNAL is a crash. A Python-level failure raises the same
|
|
22
|
+
exception in-process, which the pool already turns into an error reply.
|
|
23
|
+
|
|
24
|
+
The child runs the pool's own `_acquire_coreml_model`, so on success it has
|
|
25
|
+
also written the compiled-model cache and the in-process load is a cache HIT:
|
|
26
|
+
the probe costs one interpreter start and one cached load, once per package.
|
|
27
|
+
"""
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import json
|
|
31
|
+
import os
|
|
32
|
+
import platform
|
|
33
|
+
import signal
|
|
34
|
+
import subprocess
|
|
35
|
+
import sys
|
|
36
|
+
from typing import Callable, Optional
|
|
37
|
+
|
|
38
|
+
# One-way: inference_pool imports this module lazily, inside its CoreML branch.
|
|
39
|
+
from inference_pool import COREML_CACHE_DIR_NAME, _coreml_model_fingerprint
|
|
40
|
+
|
|
41
|
+
VERDICT_OK = "ok"
|
|
42
|
+
VERDICT_CRASHED = "crashed"
|
|
43
|
+
VERDICT_INCONCLUSIVE = "inconclusive"
|
|
44
|
+
|
|
45
|
+
PROBE_VERDICT_VERSION = 1
|
|
46
|
+
PROBE_MARKER = "@@CAMSTACK_COREML_PROBE_OK@@"
|
|
47
|
+
# Long enough for the slowest cold compile measured on a Mac (rfdetr-package,
|
|
48
|
+
# 58 s); the pool's own load bound reports a slow load to the host meanwhile.
|
|
49
|
+
PROBE_TIMEOUT_SEC = 600.0
|
|
50
|
+
|
|
51
|
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
52
|
+
|
|
53
|
+
# The child loads the package exactly as the pool does (same function, same
|
|
54
|
+
# compute units, same compiled-model cache) and runs one predict on a neutral
|
|
55
|
+
# input, then prints the marker. It never touches the parent's state.
|
|
56
|
+
_CHILD_CODE = f"""
|
|
57
|
+
import json, sys
|
|
58
|
+
args = json.loads(sys.argv[1])
|
|
59
|
+
sys.path.insert(0, args["pool_dir"])
|
|
60
|
+
import coremltools as ct
|
|
61
|
+
import numpy as np
|
|
62
|
+
from PIL import Image
|
|
63
|
+
import inference_pool as ip
|
|
64
|
+
units = {{"cpu": ct.ComputeUnit.CPU_ONLY, "gpu": ct.ComputeUnit.CPU_AND_GPU,
|
|
65
|
+
"ane": ct.ComputeUnit.CPU_AND_NE, "all": ct.ComputeUnit.ALL}}
|
|
66
|
+
model, spec = ip._acquire_coreml_model(ct, args["path"], units.get(args["device"], ct.ComputeUnit.ALL))
|
|
67
|
+
feed = {{}}
|
|
68
|
+
for inp in spec.description.input:
|
|
69
|
+
kind = inp.type.WhichOneof("Type")
|
|
70
|
+
if kind == "imageType":
|
|
71
|
+
feed[inp.name] = Image.new("RGB", (inp.type.imageType.width, inp.type.imageType.height), (114, 114, 114))
|
|
72
|
+
elif kind == "multiArrayType":
|
|
73
|
+
mat = inp.type.multiArrayType
|
|
74
|
+
if mat.WhichOneof("ShapeFlexibility") == "shapeRange":
|
|
75
|
+
shape = [max(1, r.lowerBound) for r in mat.shapeRange.sizeRanges]
|
|
76
|
+
else:
|
|
77
|
+
shape = [max(1, d) for d in mat.shape]
|
|
78
|
+
feed[inp.name] = np.zeros(shape, dtype=np.float32)
|
|
79
|
+
if feed:
|
|
80
|
+
model.predict(feed)
|
|
81
|
+
print({PROBE_MARKER!r}, flush=True)
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
ProbeRunner = Callable[[str, str], "tuple[str, str]"]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class CoremlLoadCrashed(Exception):
|
|
88
|
+
"""The package kills the native loader; it is refused instead of loaded."""
|
|
89
|
+
|
|
90
|
+
reason = "native-load-crashed"
|
|
91
|
+
|
|
92
|
+
def __init__(self, model_path: str, device: str, detail: str) -> None:
|
|
93
|
+
self.model_path = model_path
|
|
94
|
+
self.device = device
|
|
95
|
+
self.detail = detail
|
|
96
|
+
super().__init__(
|
|
97
|
+
f"CoreML package {os.path.basename(model_path)} crashes the native loader "
|
|
98
|
+
f"on device={device} ({detail}); refused so it cannot take the pool down. "
|
|
99
|
+
f"Run this model through ONNX Runtime, or replace the package"
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def classify_probe_exit(returncode: int, marker_seen: bool) -> str:
|
|
104
|
+
"""Pure: the probe child's exit -> verdict. Only a signal death is a crash."""
|
|
105
|
+
if returncode < 0:
|
|
106
|
+
return VERDICT_CRASHED
|
|
107
|
+
if returncode == 0 and marker_seen:
|
|
108
|
+
return VERDICT_OK
|
|
109
|
+
return VERDICT_INCONCLUSIVE
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _signal_name(signum: int) -> str:
|
|
113
|
+
try:
|
|
114
|
+
return signal.Signals(signum).name
|
|
115
|
+
except ValueError:
|
|
116
|
+
return str(signum)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def run_probe(
|
|
120
|
+
model_path: str,
|
|
121
|
+
device: str,
|
|
122
|
+
child_code: str = _CHILD_CODE,
|
|
123
|
+
timeout_sec: float = PROBE_TIMEOUT_SEC,
|
|
124
|
+
) -> "tuple[str, str]":
|
|
125
|
+
"""Load `model_path` in a fresh interpreter; return (verdict, detail)."""
|
|
126
|
+
payload = json.dumps({"path": model_path, "device": device, "pool_dir": _HERE})
|
|
127
|
+
try:
|
|
128
|
+
proc = subprocess.run(
|
|
129
|
+
[sys.executable, "-c", child_code, payload],
|
|
130
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
|
131
|
+
timeout=timeout_sec,
|
|
132
|
+
)
|
|
133
|
+
except subprocess.TimeoutExpired:
|
|
134
|
+
return VERDICT_INCONCLUSIVE, f"probe timed out after {timeout_sec:g}s"
|
|
135
|
+
except OSError as exc:
|
|
136
|
+
return VERDICT_INCONCLUSIVE, f"probe could not start ({exc})"
|
|
137
|
+
marker_seen = any(line.strip() == PROBE_MARKER for line in proc.stdout.splitlines())
|
|
138
|
+
verdict = classify_probe_exit(proc.returncode, marker_seen)
|
|
139
|
+
tail = " | ".join(proc.stderr.strip().splitlines()[-3:])
|
|
140
|
+
if proc.returncode < 0:
|
|
141
|
+
detail = f"probe child died by signal {-proc.returncode} ({_signal_name(-proc.returncode)})"
|
|
142
|
+
else:
|
|
143
|
+
detail = f"probe child exited {proc.returncode}"
|
|
144
|
+
return verdict, f"{detail}{': ' + tail if tail else ''}"
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# The runner `ensure_coreml_loadable` uses when it is given none — looked up
|
|
148
|
+
# at call time, so a test can replace it.
|
|
149
|
+
DEFAULT_RUNNER: ProbeRunner = run_probe
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def current_os_build() -> str:
|
|
153
|
+
"""The OS the verdict was measured on: a macOS update may fix or break a package."""
|
|
154
|
+
release = platform.mac_ver()[0]
|
|
155
|
+
return f"{release or platform.system()} {platform.version()}"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def probe_verdict_path(model_path: str, device: str) -> str:
|
|
159
|
+
models_dir = os.path.dirname(os.path.abspath(model_path))
|
|
160
|
+
stem = os.path.splitext(os.path.basename(model_path))[0]
|
|
161
|
+
return os.path.join(models_dir, COREML_CACHE_DIR_NAME, f"{stem}.{device}.probe.json")
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _read_verdict(path: str, key: dict) -> Optional[dict]:
|
|
165
|
+
try:
|
|
166
|
+
with open(path, "r", encoding="utf-8") as fh:
|
|
167
|
+
stored = json.load(fh)
|
|
168
|
+
except (OSError, ValueError):
|
|
169
|
+
return None
|
|
170
|
+
if not isinstance(stored, dict) or stored.get("key") != key:
|
|
171
|
+
return None
|
|
172
|
+
if stored.get("verdict") not in (VERDICT_OK, VERDICT_CRASHED):
|
|
173
|
+
return None
|
|
174
|
+
return stored
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _write_verdict(path: str, key: dict, verdict: str, detail: str) -> None:
|
|
178
|
+
"""Atomic replace; a verdict that cannot be written only costs a re-probe."""
|
|
179
|
+
tmp = f"{path}.tmp-{os.getpid()}"
|
|
180
|
+
try:
|
|
181
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
182
|
+
with open(tmp, "w", encoding="utf-8") as fh:
|
|
183
|
+
json.dump({"key": key, "verdict": verdict, "detail": detail}, fh)
|
|
184
|
+
os.replace(tmp, path)
|
|
185
|
+
except OSError as exc:
|
|
186
|
+
sys.stderr.write(f"CoreML probe: could not persist the verdict for {path} ({exc})\n")
|
|
187
|
+
sys.stderr.flush()
|
|
188
|
+
try:
|
|
189
|
+
os.remove(tmp)
|
|
190
|
+
except OSError:
|
|
191
|
+
pass
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def ensure_coreml_loadable(
|
|
195
|
+
model_path: str,
|
|
196
|
+
device: str,
|
|
197
|
+
run: Optional[ProbeRunner] = None,
|
|
198
|
+
os_build: Optional[str] = None,
|
|
199
|
+
) -> None:
|
|
200
|
+
"""Return when `model_path` may be loaded in-process; raise when it would crash."""
|
|
201
|
+
key = {
|
|
202
|
+
"version": PROBE_VERDICT_VERSION,
|
|
203
|
+
"fingerprint": _coreml_model_fingerprint(model_path), # a re-download is a new package
|
|
204
|
+
"device": device,
|
|
205
|
+
"os": os_build if os_build is not None else current_os_build(),
|
|
206
|
+
}
|
|
207
|
+
verdict_path = probe_verdict_path(model_path, device)
|
|
208
|
+
known = _read_verdict(verdict_path, key)
|
|
209
|
+
name = os.path.basename(model_path)
|
|
210
|
+
if known is not None:
|
|
211
|
+
if known["verdict"] == VERDICT_CRASHED:
|
|
212
|
+
sys.stderr.write(f"CoreML probe: {name} device={device} is a KNOWN loader crash — refused\n")
|
|
213
|
+
sys.stderr.flush()
|
|
214
|
+
raise CoremlLoadCrashed(model_path, device, str(known.get("detail") or "known crash"))
|
|
215
|
+
return
|
|
216
|
+
runner = run if run is not None else DEFAULT_RUNNER
|
|
217
|
+
verdict, detail = runner(model_path, device)
|
|
218
|
+
sys.stderr.write(f"CoreML probe: {name} device={device} -> {verdict} ({detail})\n")
|
|
219
|
+
sys.stderr.flush()
|
|
220
|
+
if verdict in (VERDICT_OK, VERDICT_CRASHED):
|
|
221
|
+
_write_verdict(verdict_path, key, verdict, detail)
|
|
222
|
+
if verdict == VERDICT_CRASHED:
|
|
223
|
+
raise CoremlLoadCrashed(model_path, device, detail)
|
package/python/inference_pool.py
CHANGED
|
@@ -614,6 +614,13 @@ def _coreml_model_fingerprint(model_path: str) -> dict:
|
|
|
614
614
|
Works for both a plain file (.mlmodel) and a .mlpackage DIRECTORY (the
|
|
615
615
|
normal case), whose top-level mtime alone would not reflect content edits.
|
|
616
616
|
JSON-serializable so it round-trips through the stamp file unchanged.
|
|
617
|
+
|
|
618
|
+
The package's top-level `Manifest.json` counts by size only, never by
|
|
619
|
+
mtime: coremltools rewrites it (same bytes) on EVERY open of the package —
|
|
620
|
+
`MLModel` and `utils.load_spec` alike (measured, coremltools 8.3, macOS
|
|
621
|
+
26.2). The cache HIT path calls `load_spec`, so keyed on that mtime every
|
|
622
|
+
HIT invalidated the stamp for the next spawn: HIT, MISS, HIT, MISS. A
|
|
623
|
+
re-download rewrites the model and weights files too, so it still shows.
|
|
617
624
|
"""
|
|
618
625
|
if not os.path.isdir(model_path):
|
|
619
626
|
st = os.stat(model_path)
|
|
@@ -621,13 +628,17 @@ def _coreml_model_fingerprint(model_path: str) -> dict:
|
|
|
621
628
|
total_size = 0
|
|
622
629
|
latest_mtime = 0
|
|
623
630
|
file_count = 0
|
|
631
|
+
manifest = os.path.join(model_path, "Manifest.json")
|
|
624
632
|
for root, _dirs, files in os.walk(model_path):
|
|
625
633
|
for name in files:
|
|
626
|
-
|
|
634
|
+
file_path = os.path.join(root, name)
|
|
635
|
+
st = os.stat(file_path)
|
|
627
636
|
total_size += st.st_size
|
|
637
|
+
file_count += 1
|
|
638
|
+
if file_path == manifest:
|
|
639
|
+
continue
|
|
628
640
|
if st.st_mtime_ns > latest_mtime:
|
|
629
641
|
latest_mtime = st.st_mtime_ns
|
|
630
|
-
file_count += 1
|
|
631
642
|
return {"kind": "dir", "files": file_count, "size": total_size, "mtimeNs": latest_mtime}
|
|
632
643
|
|
|
633
644
|
|
|
@@ -1032,6 +1043,14 @@ def _load_model(slot: ModelSlot, config: dict) -> None:
|
|
|
1032
1043
|
# compile + persist on miss) and returns the spec read via
|
|
1033
1044
|
# ct.utils.load_spec on hits — falling back to the plain
|
|
1034
1045
|
# MLModel/get_spec() path on ANY cache failure.
|
|
1046
|
+
#
|
|
1047
|
+
# First, the package is loaded once in a CHILD process: a package that
|
|
1048
|
+
# aborts Core ML's native loader (SIGABRT, which Python cannot catch)
|
|
1049
|
+
# is refused here with `CoremlLoadCrashed` instead of killing this
|
|
1050
|
+
# worker and, through the respawn, every worker after it.
|
|
1051
|
+
import coreml_load_probe
|
|
1052
|
+
|
|
1053
|
+
coreml_load_probe.ensure_coreml_loadable(path, device)
|
|
1035
1054
|
model, spec = _acquire_coreml_model(
|
|
1036
1055
|
ct, path, compute_map.get(device, ct.ComputeUnit.ALL),
|
|
1037
1056
|
)
|
|
@@ -2870,7 +2889,14 @@ async def _handle_model_command(
|
|
|
2870
2889
|
"error": str(exc),
|
|
2871
2890
|
}
|
|
2872
2891
|
except Exception as exc:
|
|
2873
|
-
|
|
2892
|
+
reply = {"cmd": action, "index": index, "status": "error", "error": str(exc)}
|
|
2893
|
+
# A typed refusal (e.g. `native-load-crashed`) names its class, so the
|
|
2894
|
+
# host can tell it from an ordinary failed load.
|
|
2895
|
+
reason = getattr(exc, "reason", None)
|
|
2896
|
+
if isinstance(reason, str):
|
|
2897
|
+
reply["reason"] = reason
|
|
2898
|
+
reply["modelId"] = model_id
|
|
2899
|
+
return reply
|
|
2874
2900
|
models[index] = slot
|
|
2875
2901
|
load_ms = round((time.perf_counter() - t0) * 1000)
|
|
2876
2902
|
verb = "loaded" if action == "load" else "replaced"
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
"""Unit tests for the CoreML child-process load probe (coreml_load_probe).
|
|
2
|
+
|
|
3
|
+
A CoreML package that aborts the native loader (SIGABRT inside Core ML) takes
|
|
4
|
+
the whole pool process with it, and the host respawns the pool straight into
|
|
5
|
+
the same load. The probe loads the package in a CHILD first, remembers the
|
|
6
|
+
verdict per (package fingerprint, compute unit, OS build), and refuses a
|
|
7
|
+
package whose child died by a signal — with an exception, not a crash.
|
|
8
|
+
|
|
9
|
+
Covers:
|
|
10
|
+
- classify_probe_exit — the exit -> verdict mapping (signal = crashed only).
|
|
11
|
+
- run_probe — a REAL child that calls os.abort() is classified `crashed`,
|
|
12
|
+
a child that prints the marker is `ok`.
|
|
13
|
+
- ensure_coreml_loadable — verdict persistence: a crash refuses now and on
|
|
14
|
+
the next call without re-probing; `ok` is not re-probed; a changed package
|
|
15
|
+
is re-probed; an inconclusive probe is not remembered and does not refuse.
|
|
16
|
+
- inference_pool._load_model (coreml branch) — a package the probe calls
|
|
17
|
+
crashed never reaches `ct.models.MLModel`.
|
|
18
|
+
|
|
19
|
+
Pure-python: stubs numpy / PIL / postprocessors when absent, like the other
|
|
20
|
+
inference_pool tests.
|
|
21
|
+
|
|
22
|
+
Run: python3 -m unittest test_coreml_load_probe -v
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
import os
|
|
28
|
+
import sys
|
|
29
|
+
import tempfile
|
|
30
|
+
import types
|
|
31
|
+
import unittest
|
|
32
|
+
|
|
33
|
+
try:
|
|
34
|
+
import numpy # noqa: F401
|
|
35
|
+
except ImportError:
|
|
36
|
+
_np = types.ModuleType("numpy")
|
|
37
|
+
_np.array = lambda seq, dtype=None: list(seq) # noqa: E731
|
|
38
|
+
_np.float32 = "float32"
|
|
39
|
+
sys.modules["numpy"] = _np
|
|
40
|
+
|
|
41
|
+
try:
|
|
42
|
+
import PIL.Image # noqa: F401
|
|
43
|
+
except ImportError:
|
|
44
|
+
_pil = types.ModuleType("PIL")
|
|
45
|
+
_pil_image = types.ModuleType("PIL.Image")
|
|
46
|
+
_pil.Image = _pil_image
|
|
47
|
+
sys.modules["PIL"] = _pil
|
|
48
|
+
sys.modules["PIL.Image"] = _pil_image
|
|
49
|
+
|
|
50
|
+
if "postprocessors" not in sys.modules:
|
|
51
|
+
_pp = types.ModuleType("postprocessors")
|
|
52
|
+
_pp.POSTPROCESSORS = {}
|
|
53
|
+
sys.modules["postprocessors"] = _pp
|
|
54
|
+
|
|
55
|
+
import coreml_load_probe as probe
|
|
56
|
+
import inference_pool as ip
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _write(path: str, content: bytes = b"x") -> None:
|
|
60
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
61
|
+
with open(path, "wb") as fh:
|
|
62
|
+
fh.write(content)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _make_mlpackage(root: str, name: str = "camstack-yolo26n-320.mlpackage") -> str:
|
|
66
|
+
pkg = os.path.join(root, name)
|
|
67
|
+
_write(os.path.join(pkg, "Manifest.json"), b"{}")
|
|
68
|
+
_write(os.path.join(pkg, "Data", "com.apple.CoreML", "model.mlmodel"), b"graph")
|
|
69
|
+
return pkg
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class _Runner:
|
|
73
|
+
"""A fake probe runner: answers a fixed verdict and counts its calls."""
|
|
74
|
+
|
|
75
|
+
def __init__(self, verdict: str, detail: str = "") -> None:
|
|
76
|
+
self.verdict = verdict
|
|
77
|
+
self.detail = detail
|
|
78
|
+
self.calls: list[tuple[str, str]] = []
|
|
79
|
+
|
|
80
|
+
def __call__(self, model_path: str, device: str) -> "tuple[str, str]":
|
|
81
|
+
self.calls.append((model_path, device))
|
|
82
|
+
return self.verdict, self.detail
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class ClassifyProbeExitTest(unittest.TestCase):
|
|
86
|
+
def test_marker_and_zero_exit_is_ok(self) -> None:
|
|
87
|
+
self.assertEqual(probe.classify_probe_exit(0, True), probe.VERDICT_OK)
|
|
88
|
+
|
|
89
|
+
def test_death_by_signal_is_crashed(self) -> None:
|
|
90
|
+
# subprocess reports a signal death as -signum (SIGABRT = 6).
|
|
91
|
+
self.assertEqual(probe.classify_probe_exit(-6, False), probe.VERDICT_CRASHED)
|
|
92
|
+
self.assertEqual(probe.classify_probe_exit(-11, False), probe.VERDICT_CRASHED)
|
|
93
|
+
|
|
94
|
+
def test_python_exception_is_inconclusive(self) -> None:
|
|
95
|
+
# A Python-level failure raises the same exception in-process: harmless,
|
|
96
|
+
# so it is not a reason to refuse the package.
|
|
97
|
+
self.assertEqual(probe.classify_probe_exit(1, False), probe.VERDICT_INCONCLUSIVE)
|
|
98
|
+
|
|
99
|
+
def test_clean_exit_without_marker_is_inconclusive(self) -> None:
|
|
100
|
+
self.assertEqual(probe.classify_probe_exit(0, False), probe.VERDICT_INCONCLUSIVE)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class RunProbeRealChildTest(unittest.TestCase):
|
|
104
|
+
"""The child is a real interpreter: a native abort is seen as a crash."""
|
|
105
|
+
|
|
106
|
+
def test_child_that_aborts_is_crashed(self) -> None:
|
|
107
|
+
verdict, detail = probe.run_probe(
|
|
108
|
+
"/nonexistent.mlpackage", "all", child_code="import os; os.abort()",
|
|
109
|
+
)
|
|
110
|
+
self.assertEqual(verdict, probe.VERDICT_CRASHED)
|
|
111
|
+
self.assertIn("signal", detail)
|
|
112
|
+
|
|
113
|
+
def test_child_that_prints_marker_is_ok(self) -> None:
|
|
114
|
+
code = f"print({probe.PROBE_MARKER!r}, flush=True)"
|
|
115
|
+
verdict, _ = probe.run_probe("/nonexistent.mlpackage", "all", child_code=code)
|
|
116
|
+
self.assertEqual(verdict, probe.VERDICT_OK)
|
|
117
|
+
|
|
118
|
+
def test_child_that_raises_is_inconclusive(self) -> None:
|
|
119
|
+
verdict, _ = probe.run_probe(
|
|
120
|
+
"/nonexistent.mlpackage", "all", child_code="raise RuntimeError('bad spec')",
|
|
121
|
+
)
|
|
122
|
+
self.assertEqual(verdict, probe.VERDICT_INCONCLUSIVE)
|
|
123
|
+
|
|
124
|
+
def test_child_past_its_bound_is_inconclusive(self) -> None:
|
|
125
|
+
verdict, detail = probe.run_probe(
|
|
126
|
+
"/nonexistent.mlpackage", "all", child_code="import time; time.sleep(30)",
|
|
127
|
+
timeout_sec=0.5,
|
|
128
|
+
)
|
|
129
|
+
self.assertEqual(verdict, probe.VERDICT_INCONCLUSIVE)
|
|
130
|
+
self.assertIn("timed out", detail)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class EnsureCoremlLoadableTest(unittest.TestCase):
|
|
134
|
+
def setUp(self) -> None:
|
|
135
|
+
self._tmp = tempfile.TemporaryDirectory()
|
|
136
|
+
self.pkg = _make_mlpackage(self._tmp.name)
|
|
137
|
+
|
|
138
|
+
def tearDown(self) -> None:
|
|
139
|
+
self._tmp.cleanup()
|
|
140
|
+
|
|
141
|
+
def test_crash_refuses_with_a_typed_reason(self) -> None:
|
|
142
|
+
runner = _Runner(probe.VERDICT_CRASHED, "child died by signal 6 (SIGABRT)")
|
|
143
|
+
with self.assertRaises(probe.CoremlLoadCrashed) as ctx:
|
|
144
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=runner)
|
|
145
|
+
self.assertEqual(ctx.exception.reason, "native-load-crashed")
|
|
146
|
+
self.assertIn("camstack-yolo26n-320", str(ctx.exception))
|
|
147
|
+
|
|
148
|
+
def test_crash_is_remembered_and_not_reprobed(self) -> None:
|
|
149
|
+
runner = _Runner(probe.VERDICT_CRASHED)
|
|
150
|
+
with self.assertRaises(probe.CoremlLoadCrashed):
|
|
151
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=runner)
|
|
152
|
+
with self.assertRaises(probe.CoremlLoadCrashed):
|
|
153
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=runner)
|
|
154
|
+
self.assertEqual(len(runner.calls), 1)
|
|
155
|
+
|
|
156
|
+
def test_ok_is_remembered_and_not_reprobed(self) -> None:
|
|
157
|
+
runner = _Runner(probe.VERDICT_OK)
|
|
158
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=runner)
|
|
159
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=runner)
|
|
160
|
+
self.assertEqual(len(runner.calls), 1)
|
|
161
|
+
|
|
162
|
+
def test_a_load_that_rewrites_the_package_is_not_reprobed(self) -> None:
|
|
163
|
+
# Core ML rewrites a file inside the .mlpackage when it loads it (seen
|
|
164
|
+
# on macOS 26.2: the package's max mtime moves on every MLModel load),
|
|
165
|
+
# so the verdict must be keyed on the package as the probe LEFT it.
|
|
166
|
+
manifest = os.path.join(self.pkg, "Manifest.json")
|
|
167
|
+
|
|
168
|
+
def touching_runner(model_path: str, device: str) -> "tuple[str, str]":
|
|
169
|
+
touching_runner.calls += 1 # type: ignore[attr-defined]
|
|
170
|
+
st = os.stat(manifest)
|
|
171
|
+
os.utime(manifest, ns=(st.st_atime_ns, st.st_mtime_ns + 1_000_000_000))
|
|
172
|
+
return probe.VERDICT_CRASHED, "child died by signal 6"
|
|
173
|
+
|
|
174
|
+
touching_runner.calls = 0 # type: ignore[attr-defined]
|
|
175
|
+
for _ in range(2):
|
|
176
|
+
with self.assertRaises(probe.CoremlLoadCrashed):
|
|
177
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=touching_runner)
|
|
178
|
+
self.assertEqual(touching_runner.calls, 1) # type: ignore[attr-defined]
|
|
179
|
+
|
|
180
|
+
def test_verdict_is_per_compute_unit(self) -> None:
|
|
181
|
+
runner = _Runner(probe.VERDICT_OK)
|
|
182
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=runner)
|
|
183
|
+
probe.ensure_coreml_loadable(self.pkg, "cpu", run=runner)
|
|
184
|
+
self.assertEqual([d for _, d in runner.calls], ["all", "cpu"])
|
|
185
|
+
|
|
186
|
+
def test_a_changed_package_is_reprobed(self) -> None:
|
|
187
|
+
crashed = _Runner(probe.VERDICT_CRASHED)
|
|
188
|
+
with self.assertRaises(probe.CoremlLoadCrashed):
|
|
189
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=crashed)
|
|
190
|
+
# A re-download replaces the package: different size -> different fingerprint.
|
|
191
|
+
_write(os.path.join(self.pkg, "Data", "com.apple.CoreML", "model.mlmodel"), b"fixed graph")
|
|
192
|
+
ok = _Runner(probe.VERDICT_OK)
|
|
193
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=ok)
|
|
194
|
+
self.assertEqual(len(ok.calls), 1)
|
|
195
|
+
|
|
196
|
+
def test_a_different_os_build_is_reprobed(self) -> None:
|
|
197
|
+
runner = _Runner(probe.VERDICT_CRASHED)
|
|
198
|
+
with self.assertRaises(probe.CoremlLoadCrashed):
|
|
199
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=runner, os_build="26.2")
|
|
200
|
+
ok = _Runner(probe.VERDICT_OK)
|
|
201
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=ok, os_build="26.3")
|
|
202
|
+
self.assertEqual(len(ok.calls), 1)
|
|
203
|
+
|
|
204
|
+
def test_inconclusive_neither_refuses_nor_is_remembered(self) -> None:
|
|
205
|
+
runner = _Runner(probe.VERDICT_INCONCLUSIVE, "child exited 1")
|
|
206
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=runner)
|
|
207
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=runner)
|
|
208
|
+
self.assertEqual(len(runner.calls), 2)
|
|
209
|
+
|
|
210
|
+
def test_verdict_lives_in_the_coreml_cache_dir(self) -> None:
|
|
211
|
+
probe.ensure_coreml_loadable(self.pkg, "ane", run=_Runner(probe.VERDICT_OK))
|
|
212
|
+
verdict_path = probe.probe_verdict_path(self.pkg, "ane")
|
|
213
|
+
self.assertEqual(
|
|
214
|
+
os.path.dirname(verdict_path),
|
|
215
|
+
os.path.join(self._tmp.name, ip.COREML_CACHE_DIR_NAME),
|
|
216
|
+
)
|
|
217
|
+
with open(verdict_path, "r", encoding="utf-8") as fh:
|
|
218
|
+
self.assertEqual(json.load(fh)["verdict"], probe.VERDICT_OK)
|
|
219
|
+
|
|
220
|
+
def test_an_unreadable_verdict_file_is_reprobed(self) -> None:
|
|
221
|
+
_write(probe.probe_verdict_path(self.pkg, "all"), b"{not json")
|
|
222
|
+
runner = _Runner(probe.VERDICT_OK)
|
|
223
|
+
probe.ensure_coreml_loadable(self.pkg, "all", run=runner)
|
|
224
|
+
self.assertEqual(len(runner.calls), 1)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class _FakeCt:
|
|
228
|
+
"""The slice of coremltools `_load_model` touches before the model loads."""
|
|
229
|
+
|
|
230
|
+
class ComputeUnit:
|
|
231
|
+
CPU_ONLY = "CPU_ONLY"
|
|
232
|
+
CPU_AND_GPU = "CPU_AND_GPU"
|
|
233
|
+
CPU_AND_NE = "CPU_AND_NE"
|
|
234
|
+
ALL = "ALL"
|
|
235
|
+
|
|
236
|
+
def __init__(self) -> None:
|
|
237
|
+
self.mlmodel_calls = 0
|
|
238
|
+
outer = self
|
|
239
|
+
|
|
240
|
+
class _Models:
|
|
241
|
+
@staticmethod
|
|
242
|
+
def MLModel(*_a, **_k): # noqa: N802 — coremltools' name
|
|
243
|
+
outer.mlmodel_calls += 1
|
|
244
|
+
raise AssertionError("the native loader must not be reached")
|
|
245
|
+
|
|
246
|
+
@staticmethod
|
|
247
|
+
def CompiledMLModel(*_a, **_k): # noqa: N802
|
|
248
|
+
outer.mlmodel_calls += 1
|
|
249
|
+
raise AssertionError("the native loader must not be reached")
|
|
250
|
+
|
|
251
|
+
self.models = _Models
|
|
252
|
+
self.utils = types.SimpleNamespace(load_spec=lambda _p: None)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
class LoadModelGateTest(unittest.TestCase):
|
|
256
|
+
"""`_load_model` asks the probe BEFORE the in-process native load."""
|
|
257
|
+
|
|
258
|
+
def test_crashed_package_never_reaches_the_native_loader(self) -> None:
|
|
259
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
260
|
+
pkg = _make_mlpackage(tmp)
|
|
261
|
+
fake_ct = _FakeCt()
|
|
262
|
+
saved = (ip._runtime, ip._runtime_lib, probe.DEFAULT_RUNNER)
|
|
263
|
+
try:
|
|
264
|
+
ip._runtime = "coreml"
|
|
265
|
+
ip._runtime_lib = fake_ct
|
|
266
|
+
probe.DEFAULT_RUNNER = _Runner(probe.VERDICT_CRASHED, "child died by signal 6")
|
|
267
|
+
with self.assertRaises(probe.CoremlLoadCrashed):
|
|
268
|
+
ip._load_model(ip.ModelSlot(), {"path": pkg, "device": "all"})
|
|
269
|
+
finally:
|
|
270
|
+
(ip._runtime, ip._runtime_lib, probe.DEFAULT_RUNNER) = saved
|
|
271
|
+
self.assertEqual(fake_ct.mlmodel_calls, 0)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
class ModelCommandReasonTest(unittest.TestCase):
|
|
275
|
+
"""A refused load answers the host with the machine-readable reason."""
|
|
276
|
+
|
|
277
|
+
def test_load_reply_carries_native_load_crashed(self) -> None:
|
|
278
|
+
import asyncio
|
|
279
|
+
|
|
280
|
+
def failing_load(_slot, config):
|
|
281
|
+
raise probe.CoremlLoadCrashed(config["path"], "all", "child died by signal 6")
|
|
282
|
+
|
|
283
|
+
async def run() -> dict:
|
|
284
|
+
loader = ip.ModelLoader(timeout_sec=5.0, load_fn=failing_load)
|
|
285
|
+
cmd = {"cmd": "load", "index": 0, "config": {"path": "/m/camstack-yolo26n-320.mlpackage"}}
|
|
286
|
+
return await ip._handle_model_command([], cmd, loader)
|
|
287
|
+
|
|
288
|
+
reply = asyncio.run(run())
|
|
289
|
+
self.assertEqual(reply["status"], "error")
|
|
290
|
+
self.assertEqual(reply["reason"], "native-load-crashed")
|
|
291
|
+
self.assertEqual(reply["modelId"], "camstack-yolo26n-320")
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
if __name__ == "__main__":
|
|
295
|
+
unittest.main()
|
|
@@ -124,6 +124,13 @@ class CoremlModelFingerprintTest(unittest.TestCase):
|
|
|
124
124
|
after = _coreml_model_fingerprint(pkg)
|
|
125
125
|
self.assertNotEqual(before, after)
|
|
126
126
|
|
|
127
|
+
def test_manifest_rewrite_with_same_bytes_keeps_fingerprint(self) -> None:
|
|
128
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
129
|
+
pkg = _make_mlpackage(tmp)
|
|
130
|
+
before = _coreml_model_fingerprint(pkg)
|
|
131
|
+
_touch_manifest(pkg)
|
|
132
|
+
self.assertEqual(_coreml_model_fingerprint(pkg), before)
|
|
133
|
+
|
|
127
134
|
def test_fingerprint_survives_json_round_trip(self) -> None:
|
|
128
135
|
with tempfile.TemporaryDirectory() as tmp:
|
|
129
136
|
pkg = _make_mlpackage(tmp)
|
|
@@ -241,10 +248,22 @@ class _FakeCompiledMLModel:
|
|
|
241
248
|
self.compute_units = compute_units
|
|
242
249
|
|
|
243
250
|
|
|
251
|
+
def _touch_manifest(pkg: str) -> None:
|
|
252
|
+
"""What coremltools does on EVERY open of an .mlpackage (MLModel AND
|
|
253
|
+
utils.load_spec, measured with coremltools 8.3 on macOS 26.2): it rewrites
|
|
254
|
+
Manifest.json with the same bytes, so only its mtime moves."""
|
|
255
|
+
manifest = os.path.join(pkg, "Manifest.json")
|
|
256
|
+
st = os.stat(manifest)
|
|
257
|
+
os.utime(manifest, ns=(st.st_atime_ns, st.st_mtime_ns + 1_000_000_000))
|
|
258
|
+
|
|
259
|
+
|
|
244
260
|
class _FakeCt:
|
|
245
261
|
"""Minimal stand-in for the coremltools module surface the helper uses."""
|
|
246
262
|
|
|
247
|
-
def __init__(
|
|
263
|
+
def __init__(
|
|
264
|
+
self, tmp: str, fail_compiled: bool = False, fail_load_spec: bool = False,
|
|
265
|
+
touch_manifest: bool = False,
|
|
266
|
+
) -> None:
|
|
248
267
|
self.tmp = tmp
|
|
249
268
|
self.mlmodel_calls: list[tuple[str, object]] = []
|
|
250
269
|
self.compiled_calls: list[tuple[str, object]] = []
|
|
@@ -256,6 +275,8 @@ class _FakeCt:
|
|
|
256
275
|
class _MLModel:
|
|
257
276
|
def __new__(cls, path: str, compute_units: object = None) -> _FakeMLModel:
|
|
258
277
|
fake.mlmodel_calls.append((path, compute_units))
|
|
278
|
+
if touch_manifest:
|
|
279
|
+
_touch_manifest(path)
|
|
259
280
|
compiled = os.path.join(fake.tmp, f"compiled-{len(fake.mlmodel_calls)}.mlmodelc")
|
|
260
281
|
_write(os.path.join(compiled, "coremldata.bin"), b"compiled")
|
|
261
282
|
return _FakeMLModel(compiled)
|
|
@@ -269,6 +290,8 @@ class _FakeCt:
|
|
|
269
290
|
|
|
270
291
|
def _load_spec(path: str) -> _FakeSpec:
|
|
271
292
|
fake.load_spec_calls.append(path)
|
|
293
|
+
if touch_manifest:
|
|
294
|
+
_touch_manifest(path)
|
|
272
295
|
if fail_load_spec:
|
|
273
296
|
raise RuntimeError("load_spec failed")
|
|
274
297
|
return fake.loaded_spec
|
|
@@ -311,6 +334,19 @@ class AcquireCoremlModelTest(unittest.TestCase):
|
|
|
311
334
|
self.assertEqual(ct.load_spec_calls, [pkg])
|
|
312
335
|
self.assertEqual(ct.mlmodel_calls, [])
|
|
313
336
|
|
|
337
|
+
def test_a_hit_does_not_invalidate_the_next_hit(self) -> None:
|
|
338
|
+
# The HIT path reads the spec with `ct.utils.load_spec`, which rewrites
|
|
339
|
+
# the package's Manifest.json. Keyed on that mtime, every HIT
|
|
340
|
+
# invalidated the cache for the NEXT spawn: HIT, MISS, HIT, MISS.
|
|
341
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
342
|
+
pkg = _make_mlpackage(tmp)
|
|
343
|
+
_acquire_coreml_model(_FakeCt(tmp, touch_manifest=True), pkg, self.CU) # MISS
|
|
344
|
+
for _ in range(3):
|
|
345
|
+
ct = _FakeCt(tmp, touch_manifest=True)
|
|
346
|
+
model, _ = _acquire_coreml_model(ct, pkg, self.CU)
|
|
347
|
+
self.assertIsInstance(model, _FakeCompiledMLModel)
|
|
348
|
+
self.assertEqual(ct.mlmodel_calls, [])
|
|
349
|
+
|
|
314
350
|
def test_changed_model_invalidates_cache_and_recompiles(self) -> None:
|
|
315
351
|
with tempfile.TemporaryDirectory() as tmp:
|
|
316
352
|
pkg = _make_mlpackage(tmp)
|