dsh-labscript 0.2.3 → 0.2.5

package/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## 0.2.5
+
+- labs_status can list every panel global: keys=["*"] prints all of them, sorted
+  (raw=true prints the raw expressions). This makes leftover array globals visible,
+  which is what makes labs_protocol refuse to run with N_SHOTS_MISMATCH.
+- labs_status reports scan axes: the array-valued globals with their lengths and
+  n_shots_from_arrays (their product), and flags when the panel n_shots differs from
+  that product.
+- labs_wait fails immediately when an explicit run folder does not exist and is not
+  this session's pending run (it used to wait for the full timeout), and the result
+  shows whether the folder exists.
+- labs_data description: run_folder defaults to the pending run, else the last run
+  submitted through this plugin (persisted across restarts).
+- cmd_progress returns folder_exists.
+- Tests: scan axes, all-globals rendering, wait fast-fail, progress folder_exists.
+
+## 0.2.4
+
+- Fix: labs_data no longer answers h5: 0 for a run folder that does not exist. A missing
+  folder raises RUN_FOLDER_MISSING with the raw and mapped paths plus a hint: runmanager's
+  shot output folder is the folder the NEXT run will use and only exists after engage, so
+  historical runs must be passed explicitly. An existing folder with no shots reports
+  empty: true instead.
+- Fix: labs_status with keys=[...] drops keys that are not panel globals instead of
+  returning null, so the rendered output shows <missing>. UI group names (for example
+  Select_Sequence) are not globals: read their leaf keys or a run's h5.
+- Descriptions: labs_status says the shot output folder is the NEXT run folder and that
+  keys are leaf global names; labs_data explains the run_folder default and the error.
+- Tests: missing run folder, empty run folder, missing keys dropped.
+
 ## 0.2.3
 
 - Fix: labs_status now prints panel values. keys=[...] and raw=true were accepted but the

package/README.md

@@ -125,8 +125,8 @@ is not a physics simulation.
 ## Development
 
     npm install              # only runtime dependency: yaml
-    npm test                 # node --test, 36 tests (fake bridge, no python needed)
-    python python/tests/run_tests.py   # 15 tests (needs numpy + h5py)
+    npm test                 # node --test, 38 tests (fake bridge, no python needed)
+    python python/tests/run_tests.py   # 19 tests (needs numpy + h5py)
     LABSCRIPT_PYTHON=/path/to/python3 node scripts/demo.mjs   # end-to-end mock demo
 
 A fresh clone must run `npm install` before `npm test`; the CI workflow does this.
@@ -157,7 +157,9 @@ A fresh clone must run `npm install` before `npm test`; the CI workflow does thi
   partial). compiled_not_executed means shots exist but show no acquisition data yet;
   BLACS may be paused or not executing the queue.
 - Real runmanager engage does not return a run folder; the bridge captures the
-  output folder before engage and reports it as the run folder.
+  output folder before engage and reports it as the run folder. That shot output
+  folder is the NEXT run's folder and does not exist until engage, so labs_data
+  errors (RUN_FOLDER_MISSING) instead of reporting h5: 0 for a path that is not there.
 - The panel is snapshotted before every write; labs_restore restores only the keys the
   write touched (full=true restores the whole panel), so unrelated concurrent edits survive.
 - Actions are logged to events.jsonl next to the snapshots.

package/lib/tools.js

@@ -46,11 +46,16 @@ function globalText(value, limit = 240) {
 function globalLines(args, value) {
   const globals = value.globals || {}
   const keys = Array.isArray(args.keys) ? args.keys : []
+  const raw = args.raw === true && value.raw_globals ? value.raw_globals : null
+  if (keys.length === 1 && keys[0] === '*') {
+    const shown = raw || globals
+    const names = Object.keys(shown).sort()
+    return ['globals: all ' + names.length].concat(names.map((key) => '  ' + key + ' = ' + globalText(shown[key])))
+  }
   if (keys.length === 0) {
     const count = Object.keys(globals).length
-    return [count > 0 ? 'globals: ' + count + ' keys (pass keys=[...] to read values)' : 'globals: none']
+    return [count > 0 ? 'globals: ' + count + ' keys (pass keys=[...] or keys=["*"] to read values)' : 'globals: none']
   }
-  const raw = args.raw === true && value.raw_globals ? value.raw_globals : null
   const out = ['globals:']
   for (const key of keys) {
     const shown = raw ? raw[key] : globals[key]
@@ -59,6 +64,16 @@ function globalLines(args, value) {
   return out
 }
 
+function scanLine(value) {
+  const axes = value.scan_axes && typeof value.scan_axes === 'object' ? value.scan_axes : {}
+  const names = Object.keys(axes).sort()
+  const product = value.n_shots_from_arrays
+  const mismatch = typeof product === 'number' && typeof value.n_shots === 'number' && value.n_shots !== product
+  if (names.length === 0 && !mismatch) return ''
+  const detail = names.length ? names.map((key) => key + '=' + axes[key]).join(', ') : 'none'
+  return 'scan axes: ' + detail + '  n_shots_from_arrays=' + product + (mismatch ? '  (panel n_shots=' + value.n_shots + ' differs)' : '')
+}
+
 async function pendingProgress(runtime, exec) {
   const pending = runtime.state.pending
   if (!pending) return null
@@ -129,11 +144,11 @@ export function registerTools(ctx, runtime) {
 
   ctx.tools.register({
     name: 'labs_status',
-    description: 'Read the labscript runmanager panel: loaded labscript, globals (evaluated; raw expressions with raw=true), n_shots, error_in_globals, run_shots, shot output folder, pending run progress, and the panel snapshot directory. Read-only; call this before any write. Pass keys=[...] to print the selected panel values (240 chars each; raw=true prints the raw expression). Paths in the output are already mapped for this machine through pathMap; do not map them again.',
+    description: 'Read the labscript runmanager panel: loaded labscript, globals (evaluated; raw expressions with raw=true), n_shots, error_in_globals, run_shots, shot output folder, pending run progress, and the panel snapshot directory. Read-only; call this before any write. Pass keys=[...] (leaf global names; UI group names are not globals) to print the selected panel values (240 chars each; raw=true prints the raw expression), missing keys are shown as <missing>; keys=["*"] prints every global. scan axes lists array-valued globals (their lengths multiply to n_shots_from_arrays) and a differing panel n_shots is flagged. The shot output folder is the folder the NEXT run will use; it does not exist until engage. Paths in the output are already mapped for this machine through pathMap; do not map them again.',
     parameters: {
       type: 'object',
       properties: {
-        keys: { type: 'array', items: { type: 'string' }, description: 'Only return these panel globals' },
+        keys: { type: 'array', items: { type: 'string' }, description: 'Only return these panel globals; ["*"] returns every global' },
         raw: { type: 'boolean', description: 'Also return raw (expression) values' },
       },
       additionalProperties: false,
@@ -146,11 +161,14 @@ export function registerTools(ctx, runtime) {
       'n_shots: ' + value.n_shots + '  error_in_globals: ' + value.error_in_globals + '  run_shots: ' + value.run_shots,
       progressLine(value.last_run),
       'panel history: ' + (value.panel_history ? value.panel_history.dir + ' (' + value.panel_history.count + ' snapshots)' : 'n/a'),
+      scanLine(value),
       ...globalLines(args, value),
     ]))),
     async execute(args, exec) {
+      const keys = Array.isArray(args.keys) ? args.keys : []
+      const all = keys.length === 1 && keys[0] === '*'
       const data = await runtime.runBridge('status', {
-        keys: Array.isArray(args.keys) ? args.keys : undefined,
+        keys: all || keys.length === 0 ? undefined : keys,
         raw: args.raw === true,
       }, { signal: exec.signal })
       if (runtime.state.pending) data.last_run = await pendingProgress(runtime, exec)
@@ -230,7 +248,7 @@ export function registerTools(ctx, runtime) {
 
   ctx.tools.register({
     name: 'labs_wait',
-    description: 'Wait for the pending run to finish (blocking, timeout in seconds). If the pending run came from labs_protocol run, this call also finalizes it: postprocess, finish write-back of panel and calibration history, and panel restore. Returns expected vs actual shots and, for protocols, the postprocess/finish/restore results.',
+    description: 'Wait for the pending run to finish (blocking, timeout in seconds). If the pending run came from labs_protocol run, this call also finalizes it: postprocess, finish write-back of panel and calibration history, and panel restore. Returns expected vs actual shots and, for protocols, the postprocess/finish/restore results. A run folder that does not exist and is not the pending run of this session fails immediately instead of waiting.',
     parameters: {
       type: 'object',
       properties: {
@@ -244,6 +262,7 @@ export function registerTools(ctx, runtime) {
     output: looseOutput((_args, value) => text(lines([
       'run: ' + (value.mapped_run_folder || value.run_folder),
       value.aborted ? 'wait cancelled by labs_abort (run aborted)' : '',
+      value.folder_exists === false ? 'folder: missing' : '',
       'shots: ' + value.n_h5 + '/' + (value.expected_n_shots || '?') + '  complete: ' + value.complete + (value.timed_out ? '  TIMED OUT' : '') + (value.diagnosis ? '  diagnosis: ' + value.diagnosis : ''),
       value.hint ? 'hint: ' + value.hint : '',
       value.finalize && value.finalize.postprocess ? 'postprocess: ' + JSON.stringify(value.finalize.postprocess).slice(0, 400) : '',
@@ -256,6 +275,20 @@ export function registerTools(ctx, runtime) {
         const runFolder = args.run_folder || (pending ? pending.run_folder : '')
         if (!runFolder) throw new LabError('NO_PENDING_RUN', 'no run folder: submit with labs_engage or labs_protocol run first, or pass run_folder')
         const expected = pending && pending.run_folder === runFolder ? pending.expected_n_shots : undefined
+        if (!(pending && pending.run_folder === runFolder)) {
+          let probe = null
+          try {
+            probe = await runtime.runBridge('progress', {
+              run_folder: runFolder,
+              expected_n_shots: expected,
+            }, { signal: exec.signal })
+          } catch (error) {
+            probe = null
+          }
+          if (probe && probe.folder_exists === false) {
+            throw new LabError('RUN_FOLDER_MISSING', 'run folder does not exist: ' + (probe.mapped_run_folder || runFolder) + '. Pass the folder of a run that exists (labs_engage / labs_protocol return one), or check pathMap.')
+          }
+        }
         const timeoutS = Number(args.timeout_s || runtime.config.engage.timeoutS || 900)
         let waited
         try {
@@ -339,11 +372,11 @@ export function registerTools(ctx, runtime) {
 
   ctx.tools.register({
     name: 'labs_data',
-    description: 'Read a run folder HDF5: globals of the first shot (execution facts) and, with scan: a per-shot table of scan values and numeric results. Read-only.',
+    description: 'Read a run folder HDF5: globals of the first shot (execution facts) and, with scan: a per-shot table of scan values and numeric results. Read-only. run_folder defaults to the pending run, else the last run submitted through this plugin (persisted across restarts); for a historical run pass its folder explicitly. A missing folder is an error: runmanager\'s shot output is the NEXT run folder and only exists after engage.',
     parameters: {
       type: 'object',
       properties: {
-        run_folder: { type: 'string', description: 'Run folder; defaults to the pending/last submitted run' },
+        run_folder: { type: 'string', description: 'Run folder; defaults to the run submitted from this DSH session (pass a historical run folder explicitly)' },
         scan: { type: 'string', description: 'Panel global name used as the scan axis; adds a per-shot table' },
         globals: { type: 'boolean', description: 'Include the first shot globals (default true)' },
         results: { type: 'boolean', description: 'Include the results layout' },
@@ -353,7 +386,7 @@ export function registerTools(ctx, runtime) {
     isConcurrencySafe: () => true,
     output: looseOutput((_args, value) => text(lines([
       'run: ' + (value.mapped_run_folder || value.run_folder),
-      'h5: ' + value.n_h5,
+      'h5: ' + value.n_h5 + (value.empty ? ' (folder exists, no shots yet)' : ''),
       value.scan_table ? 'scan ' + value.scan_table.scan + ': ' + value.scan_table.n_points + ' shots' : '',
       value.globals ? 'globals: ' + Object.keys(value.globals).length + ' keys' : '',
     ]))),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-labscript",
-  "version": "0.2.3",
+  "version": "0.2.5",
   "description": "Control labscript-suite experiments from DeepSeek Harness: runmanager globals and scans, submits and waits, YAML fixed-operation protocols with postprocess write-back, panel snapshots, a built-in mock backend and a read-only doctor.",
   "type": "module",
   "main": "lib/index.js",

package/python/bridge.py

@@ -919,17 +919,33 @@ def cmd_status(config, args):
     backend = make_backend(config)
     keys = args.get("keys")
     raw = bool(args.get("raw"))
-    values = backend.get_globals(raw=False)
+    panel_values = backend.get_globals(raw=False)
     raw_values = backend.get_globals(raw=True)
+    scan_axes = {}
+    n_shots_from_arrays = 1
+    for name, value in panel_values.items():
+        if isinstance(value, str) or not hasattr(value, "__len__"):
+            continue
+        try:
+            size = int(len(value))
+        except Exception:
+            continue
+        if size <= 1:
+            continue
+        scan_axes[str(name)] = size
+        n_shots_from_arrays *= size
+    values = dict(panel_values)
     if keys:
-        values = dict((k, values.get(k)) for k in keys)
-        raw_values = dict((k, raw_values.get(k)) for k in keys)
+        values = dict((k, panel_values[k]) for k in keys if k in panel_values)
+        raw_values = dict((k, raw_values[k]) for k in keys if k in raw_values)
     out = {
         "backend": config.get("backend"),
         "version": backend.version() if hasattr(backend, "version") else None,
         "labscript": map_path(config, backend.labscript_file()),
         "shot_output_folder": map_path(config, backend.shot_output_folder()),
         "globals": jsonable(values),
+        "scan_axes": scan_axes,
+        "n_shots_from_arrays": n_shots_from_arrays,
         "n_shots": backend.n_shots(),
         "error_in_globals": backend.error_in_globals(),
         "run_shots": backend.run_shots(),
@@ -1127,7 +1143,7 @@ def cmd_progress(config, args):
     expected = int(args.get("expected_n_shots") or 0)
     folder = map_path(config, run_folder)
     shots = sorted(glob.glob(os.path.join(folder, "**", "*.h5"), recursive=True)) if folder else []
-    return {"run_folder": run_folder, "mapped_run_folder": folder, "n_h5": len(shots), "expected_n_shots": expected, "complete": bool(expected and len(shots) >= expected)}
+    return {"run_folder": run_folder, "mapped_run_folder": folder, "folder_exists": bool(folder) and os.path.isdir(folder), "n_h5": len(shots), "expected_n_shots": expected, "complete": bool(expected and len(shots) >= expected)}
 
 
 
@@ -1182,9 +1198,16 @@ def cmd_data(config, args):
     if not run_folder:
         raise BridgeError("BAD_ARGS", "run_folder is required")
     folder = map_path(config, run_folder)
+    if not os.path.isdir(folder):
+        raise BridgeError("RUN_FOLDER_MISSING", "run folder does not exist: " + str(folder), {
+            "run_folder": run_folder,
+            "mapped_run_folder": folder,
+            "hint": "runmanager's shot output folder is the folder the NEXT run will use; it does not exist until engage. Pass a historical run folder explicitly.",
+        })
     shots = sorted(glob.glob(os.path.join(folder, "**", "*.h5"), recursive=True))
     out = {"run_folder": run_folder, "mapped_run_folder": folder, "n_h5": len(shots)}
     if not shots:
+        out["empty"] = True
         return out
     scan = args.get("scan")
     with h5py.File(shots[0], "r", locking=False) as handle:

package/python/tests/run_tests.py

@@ -344,6 +344,65 @@ def test_progress_reports_mapped_run_folder():
     out = bridge.cmd_progress(cfg, {"run_folder": "E:\\Experiments\\nope", "expected_n_shots": 1})
     assert out["run_folder"] == "E:\\Experiments\\nope", out
     assert out["mapped_run_folder"] == "\\\\lab-pc\\Experiments\\nope", out
+    assert out["folder_exists"] is False, out
+
+
+def test_data_missing_run_folder():
+    tmp = tempfile.mkdtemp(prefix="dsh-labscript-data-missing-")
+    try:
+        cfg = make_config(tmp)
+        missing = os.path.join(tmp, "no_such_run")
+        try:
+            bridge.cmd_data(cfg, {"run_folder": missing})
+        except bridge.BridgeError as exc:
+            assert exc.code == "RUN_FOLDER_MISSING", exc.message
+            assert exc.detail["mapped_run_folder"] == missing, exc.detail
+            assert "next" in exc.detail["hint"].lower(), exc.detail
+        else:
+            raise AssertionError("expected RUN_FOLDER_MISSING")
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+
+def test_data_empty_run_folder():
+    tmp = tempfile.mkdtemp(prefix="dsh-labscript-data-empty-")
+    try:
+        cfg = make_config(tmp)
+        run_dir = os.path.join(tmp, "empty_run")
+        os.makedirs(run_dir, exist_ok=True)
+        out = bridge.cmd_data(cfg, {"run_folder": run_dir})
+        assert out["n_h5"] == 0, out
+        assert out.get("empty") is True, out
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+
+def test_status_drops_missing_keys():
+    tmp = tempfile.mkdtemp(prefix="dsh-labscript-status-missing-")
+    try:
+        cfg = make_config(tmp)
+        out = bridge.cmd_status(cfg, {"keys": ["amplitude", "Select_Sequence"], "raw": True})
+        assert "amplitude" in out["globals"], out["globals"]
+        assert "Select_Sequence" not in out["globals"], out["globals"]
+        assert "Select_Sequence" not in out["raw_globals"], out["raw_globals"]
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+
+def test_status_scan_axes():
+    tmp = tempfile.mkdtemp(prefix="dsh-labscript-scan-axes-")
+    try:
+        cfg = make_config(tmp)
+        bridge.cmd_set(cfg, {"values": {"duration_test_us": [0.0, 0.5, 1.0]}})
+        out = bridge.cmd_status(cfg, {})
+        assert out["scan_axes"].get("duration_test_us") == 3, out["scan_axes"]
+        assert out["n_shots_from_arrays"] == 3, out["n_shots_from_arrays"]
+        assert out["n_shots"] == 3, out["n_shots"]
+        partial = bridge.cmd_status(cfg, {"keys": ["amplitude"]})
+        assert "duration_test_us" not in partial["globals"], partial["globals"]
+        assert partial["scan_axes"].get("duration_test_us") == 3, partial["scan_axes"]
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
 
 
 def main():