dsh-labscript 0.2.4 → 0.2.6

package/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## 0.2.6
+
+- Fix: labs_data rendered only counts. It now prints selected first-shot globals
+  (keys=[...] to choose, ["*"] for all, <missing> for absent keys), the results group
+  names when results: true, and a scan axis summary (unique points, min/max, median
+  step, numeric column count). Pass limit to print up to 20 raw scan rows; the bridge
+  only returns rows when limit is given.
+- Tests: first-shot key selection, scan axis summary and columns, on-demand rows, and
+  the rendered output for keys/results/scan/limit.
+
+## 0.2.5
+
+- labs_status can list every panel global: keys=["*"] prints all of them, sorted
+  (raw=true prints the raw expressions). This makes leftover array globals visible,
+  which is what makes labs_protocol refuse to run with N_SHOTS_MISMATCH.
+- labs_status reports scan axes: the array-valued globals with their lengths and
+  n_shots_from_arrays (their product), and flags when the panel n_shots differs from
+  that product.
+- labs_wait fails immediately when an explicit run folder does not exist and is not
+  this session's pending run (it used to wait for the full timeout), and the result
+  shows whether the folder exists.
+- labs_data description: run_folder defaults to the pending run, else the last run
+  submitted through this plugin (persisted across restarts).
+- cmd_progress returns folder_exists.
+- Tests: scan axes, all-globals rendering, wait fast-fail, progress folder_exists.
+
 ## 0.2.4
 
 - Fix: labs_data no longer answers h5: 0 for a run folder that does not exist. A missing

package/README.md

@@ -125,8 +125,8 @@ is not a physics simulation.
 ## Development
 
     npm install              # only runtime dependency: yaml
-    npm test                 # node --test, 36 tests (fake bridge, no python needed)
-    python python/tests/run_tests.py   # 18 tests (needs numpy + h5py)
+    npm test                 # node --test, 39 tests (fake bridge, no python needed)
+    python python/tests/run_tests.py   # 20 tests (needs numpy + h5py)
     LABSCRIPT_PYTHON=/path/to/python3 node scripts/demo.mjs   # end-to-end mock demo
 
 A fresh clone must run `npm install` before `npm test`; the CI workflow does this.

package/lib/tools.js

@@ -46,11 +46,16 @@ function globalText(value, limit = 240) {
 function globalLines(args, value) {
   const globals = value.globals || {}
   const keys = Array.isArray(args.keys) ? args.keys : []
+  const raw = args.raw === true && value.raw_globals ? value.raw_globals : null
+  if (keys.length === 1 && keys[0] === '*') {
+    const shown = raw || globals
+    const names = Object.keys(shown).sort()
+    return ['globals: all ' + names.length].concat(names.map((key) => '  ' + key + ' = ' + globalText(shown[key])))
+  }
   if (keys.length === 0) {
     const count = Object.keys(globals).length
-    return [count > 0 ? 'globals: ' + count + ' keys (pass keys=[...] to read values)' : 'globals: none']
+    return [count > 0 ? 'globals: ' + count + ' keys (pass keys=[...] or keys=["*"] to read values)' : 'globals: none']
   }
-  const raw = args.raw === true && value.raw_globals ? value.raw_globals : null
   const out = ['globals:']
   for (const key of keys) {
     const shown = raw ? raw[key] : globals[key]
@@ -59,6 +64,64 @@ function globalLines(args, value) {
   return out
 }
 
+function scanLine(value) {
+  const axes = value.scan_axes && typeof value.scan_axes === 'object' ? value.scan_axes : {}
+  const names = Object.keys(axes).sort()
+  const product = value.n_shots_from_arrays
+  const mismatch = typeof product === 'number' && typeof value.n_shots === 'number' && value.n_shots !== product
+  if (names.length === 0 && !mismatch) return ''
+  const detail = names.length ? names.map((key) => key + '=' + axes[key]).join(', ') : 'none'
+  return 'scan axes: ' + detail + '  n_shots_from_arrays=' + product + (mismatch ? '  (panel n_shots=' + value.n_shots + ' differs)' : '')
+}
+
+function resultsLine(value) {
+  const layout = value.results_layout
+  if (!layout) return ''
+  const names = Object.keys(layout).sort()
+  if (names.length === 0) return 'results: none'
+  return 'results: ' + names.map((group) => group + ' (' + layout[group].length + ' fields)').join(', ')
+}
+
+function dataGlobalsLines(args, value) {
+  const globals = value.globals
+  if (!globals) return []
+  const keys = Array.isArray(args.keys) ? args.keys.map(String) : []
+  if (keys.length === 0) {
+    return ['globals: ' + Object.keys(globals).length + ' keys (pass keys=[...] or keys=["*"] to print values)']
+  }
+  const wanted = keys.length === 1 && keys[0] === '*' ? Object.keys(globals).sort() : keys
+  return ['globals:'].concat(wanted.map((key) => '  ' + key + ' = ' + (Object.prototype.hasOwnProperty.call(globals, key) ? globalText(globals[key]) : '<missing>')))
+}
+
+function scanSummaryLine(value) {
+  const table = value.scan_table
+  if (!table) return ''
+  const axis = table.axis || {}
+  const unique = typeof axis.points === 'number' && axis.points > 0
+    ? axis.points + ' unique ' + axis.min + '..' + axis.max + (axis.step === null || axis.step === undefined ? '' : ' step ' + axis.step)
+    : 'no numeric axis values'
+  const columns = Array.isArray(table.columns) ? table.columns.length : 0
+  return 'scan ' + table.scan + ': ' + table.n_points + ' shots, ' + unique + ', ' + columns + ' numeric columns' + (Array.isArray(table.rows) ? ' (' + table.rows.length + ' raw rows below)' : '')
+}
+
+function scanRowLines(value) {
+  const table = value.scan_table
+  if (!table || !Array.isArray(table.rows) || table.rows.length === 0) return []
+  const lines = ['rows (first ' + table.rows.length + '):']
+  for (const row of table.rows) {
+    const parts = []
+    if (row.shot !== undefined) parts.push('shot=' + row.shot)
+    if (row[table.scan] !== undefined) parts.push(table.scan + '=' + row[table.scan])
+    for (const key of Object.keys(row).sort()) {
+      if (key === 'shot' || key === table.scan) continue
+      parts.push(key + '=' + row[key])
+    }
+    const line = parts.join(' ')
+    lines.push('  ' + (line.length > 240 ? line.slice(0, 240) + '…' : line))
+  }
+  return lines
+}
+
 async function pendingProgress(runtime, exec) {
   const pending = runtime.state.pending
   if (!pending) return null
@@ -129,11 +192,11 @@ export function registerTools(ctx, runtime) {
 
   ctx.tools.register({
     name: 'labs_status',
-    description: 'Read the labscript runmanager panel: loaded labscript, globals (evaluated; raw expressions with raw=true), n_shots, error_in_globals, run_shots, shot output folder, pending run progress, and the panel snapshot directory. Read-only; call this before any write. Pass keys=[...] (leaf global names; UI group names are not globals) to print the selected panel values (240 chars each; raw=true prints the raw expression), missing keys are shown as <missing>. The shot output folder is the folder the NEXT run will use; it does not exist until engage. Paths in the output are already mapped for this machine through pathMap; do not map them again.',
+    description: 'Read the labscript runmanager panel: loaded labscript, globals (evaluated; raw expressions with raw=true), n_shots, error_in_globals, run_shots, shot output folder, pending run progress, and the panel snapshot directory. Read-only; call this before any write. Pass keys=[...] (leaf global names; UI group names are not globals) to print the selected panel values (240 chars each; raw=true prints the raw expression), missing keys are shown as <missing>; keys=["*"] prints every global. scan axes lists array-valued globals (their lengths multiply to n_shots_from_arrays) and a differing panel n_shots is flagged. The shot output folder is the folder the NEXT run will use; it does not exist until engage. Paths in the output are already mapped for this machine through pathMap; do not map them again.',
     parameters: {
       type: 'object',
       properties: {
-        keys: { type: 'array', items: { type: 'string' }, description: 'Only return these panel globals' },
+        keys: { type: 'array', items: { type: 'string' }, description: 'Only return these panel globals; ["*"] returns every global' },
         raw: { type: 'boolean', description: 'Also return raw (expression) values' },
       },
       additionalProperties: false,
@@ -146,11 +209,14 @@ export function registerTools(ctx, runtime) {
       'n_shots: ' + value.n_shots + '  error_in_globals: ' + value.error_in_globals + '  run_shots: ' + value.run_shots,
       progressLine(value.last_run),
       'panel history: ' + (value.panel_history ? value.panel_history.dir + ' (' + value.panel_history.count + ' snapshots)' : 'n/a'),
+      scanLine(value),
       ...globalLines(args, value),
     ]))),
     async execute(args, exec) {
+      const keys = Array.isArray(args.keys) ? args.keys : []
+      const all = keys.length === 1 && keys[0] === '*'
       const data = await runtime.runBridge('status', {
-        keys: Array.isArray(args.keys) ? args.keys : undefined,
+        keys: all || keys.length === 0 ? undefined : keys,
         raw: args.raw === true,
       }, { signal: exec.signal })
       if (runtime.state.pending) data.last_run = await pendingProgress(runtime, exec)
@@ -230,7 +296,7 @@ export function registerTools(ctx, runtime) {
 
   ctx.tools.register({
     name: 'labs_wait',
-    description: 'Wait for the pending run to finish (blocking, timeout in seconds). If the pending run came from labs_protocol run, this call also finalizes it: postprocess, finish write-back of panel and calibration history, and panel restore. Returns expected vs actual shots and, for protocols, the postprocess/finish/restore results.',
+    description: 'Wait for the pending run to finish (blocking, timeout in seconds). If the pending run came from labs_protocol run, this call also finalizes it: postprocess, finish write-back of panel and calibration history, and panel restore. Returns expected vs actual shots and, for protocols, the postprocess/finish/restore results. A run folder that does not exist and is not the pending run of this session fails immediately instead of waiting.',
     parameters: {
       type: 'object',
       properties: {
@@ -244,6 +310,7 @@ export function registerTools(ctx, runtime) {
     output: looseOutput((_args, value) => text(lines([
       'run: ' + (value.mapped_run_folder || value.run_folder),
       value.aborted ? 'wait cancelled by labs_abort (run aborted)' : '',
+      value.folder_exists === false ? 'folder: missing' : '',
       'shots: ' + value.n_h5 + '/' + (value.expected_n_shots || '?') + '  complete: ' + value.complete + (value.timed_out ? '  TIMED OUT' : '') + (value.diagnosis ? '  diagnosis: ' + value.diagnosis : ''),
       value.hint ? 'hint: ' + value.hint : '',
       value.finalize && value.finalize.postprocess ? 'postprocess: ' + JSON.stringify(value.finalize.postprocess).slice(0, 400) : '',
@@ -256,6 +323,20 @@ export function registerTools(ctx, runtime) {
         const runFolder = args.run_folder || (pending ? pending.run_folder : '')
         if (!runFolder) throw new LabError('NO_PENDING_RUN', 'no run folder: submit with labs_engage or labs_protocol run first, or pass run_folder')
         const expected = pending && pending.run_folder === runFolder ? pending.expected_n_shots : undefined
+        if (!(pending && pending.run_folder === runFolder)) {
+          let probe = null
+          try {
+            probe = await runtime.runBridge('progress', {
+              run_folder: runFolder,
+              expected_n_shots: expected,
+            }, { signal: exec.signal })
+          } catch (error) {
+            probe = null
+          }
+          if (probe && probe.folder_exists === false) {
+            throw new LabError('RUN_FOLDER_MISSING', 'run folder does not exist: ' + (probe.mapped_run_folder || runFolder) + '. Pass the folder of a run that exists (labs_engage / labs_protocol return one), or check pathMap.')
+          }
+        }
         const timeoutS = Number(args.timeout_s || runtime.config.engage.timeoutS || 900)
         let waited
         try {
@@ -339,23 +420,27 @@ export function registerTools(ctx, runtime) {
 
   ctx.tools.register({
     name: 'labs_data',
-    description: 'Read a run folder HDF5: globals of the first shot (execution facts) and, with scan: a per-shot table of scan values and numeric results. Read-only. run_folder defaults to the run this DSH session submitted; for a historical run pass its folder explicitly. A missing folder is an error: runmanager\'s shot output is the NEXT run folder and only exists after engage.',
+    description: 'Read a run folder HDF5: first-shot globals (pass keys=[...] to print selected values, ["*"] for all), the results groups with results: true, and with scan: a per-shot scan axis summary (unique points, min/max, median step) plus the numeric result columns; pass limit to print up to 20 raw rows. Read-only. run_folder defaults to the pending run, else the last run submitted through this plugin (persisted across restarts); for a historical run pass its folder explicitly. A missing folder is an error: runmanager\'s shot output is the NEXT run folder and only exists after engage.',
     parameters: {
       type: 'object',
       properties: {
         run_folder: { type: 'string', description: 'Run folder; defaults to the run submitted from this DSH session (pass a historical run folder explicitly)' },
-        scan: { type: 'string', description: 'Panel global name used as the scan axis; adds a per-shot table' },
+        scan: { type: 'string', description: 'Panel global name used as the scan axis; adds an axis summary and numeric columns (pass limit for raw rows)' },
         globals: { type: 'boolean', description: 'Include the first shot globals (default true)' },
+        keys: { type: 'array', items: { type: 'string' }, description: 'First-shot globals to print; ["*"] prints all. Missing keys show as <missing>.' },
+        limit: { type: 'number', minimum: 1, maximum: 20, description: 'How many raw scan rows to print (only with scan; default: none)' },
         results: { type: 'boolean', description: 'Include the results layout' },
       },
       additionalProperties: false,
     },
     isConcurrencySafe: () => true,
-    output: looseOutput((_args, value) => text(lines([
+    output: looseOutput((args, value) => text(lines([
       'run: ' + (value.mapped_run_folder || value.run_folder),
       'h5: ' + value.n_h5 + (value.empty ? ' (folder exists, no shots yet)' : ''),
-      value.scan_table ? 'scan ' + value.scan_table.scan + ': ' + value.scan_table.n_points + ' shots' : '',
-      value.globals ? 'globals: ' + Object.keys(value.globals).length + ' keys' : '',
+      resultsLine(value),
+      ...dataGlobalsLines(args, value),
+      scanSummaryLine(value),
+      ...scanRowLines(value),
     ]))),
     async execute(args, exec) {
       const runFolder = args.run_folder || (runtime.state.pending ? runtime.state.pending.run_folder : '') || (runtime.state.lastRun ? runtime.state.lastRun.run_folder : '')
@@ -363,6 +448,8 @@ export function registerTools(ctx, runtime) {
       return runtime.runBridge('data', {
         run_folder: runFolder,
         scan: args.scan,
+        keys: Array.isArray(args.keys) ? args.keys : undefined,
+        limit: typeof args.limit === 'number' ? args.limit : undefined,
         globals: args.globals !== false,
         results: args.results === true,
       }, { signal: exec.signal })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-labscript",
-  "version": "0.2.4",
+  "version": "0.2.6",
   "description": "Control labscript-suite experiments from DeepSeek Harness: runmanager globals and scans, submits and waits, YAML fixed-operation protocols with postprocess write-back, panel snapshots, a built-in mock backend and a read-only doctor.",
   "type": "module",
   "main": "lib/index.js",

package/python/bridge.py

@@ -919,10 +919,24 @@ def cmd_status(config, args):
     backend = make_backend(config)
     keys = args.get("keys")
     raw = bool(args.get("raw"))
-    values = backend.get_globals(raw=False)
+    panel_values = backend.get_globals(raw=False)
     raw_values = backend.get_globals(raw=True)
+    scan_axes = {}
+    n_shots_from_arrays = 1
+    for name, value in panel_values.items():
+        if isinstance(value, str) or not hasattr(value, "__len__"):
+            continue
+        try:
+            size = int(len(value))
+        except Exception:
+            continue
+        if size <= 1:
+            continue
+        scan_axes[str(name)] = size
+        n_shots_from_arrays *= size
+    values = dict(panel_values)
     if keys:
-        values = dict((k, values[k]) for k in keys if k in values)
+        values = dict((k, panel_values[k]) for k in keys if k in panel_values)
         raw_values = dict((k, raw_values[k]) for k in keys if k in raw_values)
     out = {
         "backend": config.get("backend"),
@@ -930,6 +944,8 @@ def cmd_status(config, args):
         "labscript": map_path(config, backend.labscript_file()),
         "shot_output_folder": map_path(config, backend.shot_output_folder()),
         "globals": jsonable(values),
+        "scan_axes": scan_axes,
+        "n_shots_from_arrays": n_shots_from_arrays,
         "n_shots": backend.n_shots(),
         "error_in_globals": backend.error_in_globals(),
         "run_shots": backend.run_shots(),
@@ -1127,7 +1143,7 @@ def cmd_progress(config, args):
     expected = int(args.get("expected_n_shots") or 0)
     folder = map_path(config, run_folder)
     shots = sorted(glob.glob(os.path.join(folder, "**", "*.h5"), recursive=True)) if folder else []
-    return {"run_folder": run_folder, "mapped_run_folder": folder, "n_h5": len(shots), "expected_n_shots": expected, "complete": bool(expected and len(shots) >= expected)}
+    return {"run_folder": run_folder, "mapped_run_folder": folder, "folder_exists": bool(folder) and os.path.isdir(folder), "n_h5": len(shots), "expected_n_shots": expected, "complete": bool(expected and len(shots) >= expected)}
 
 
 
@@ -1197,7 +1213,12 @@ def cmd_data(config, args):
     with h5py.File(shots[0], "r", locking=False) as handle:
         globals_attrs = dict(handle["globals"].attrs) if "globals" in handle else {}
         if args.get("globals", True):
-            out["globals"] = jsonable(globals_attrs)
+            selected = jsonable(globals_attrs)
+            keys = args.get("keys")
+            if keys and list(map(str, keys)) != ["*"]:
+                wanted = [str(key) for key in keys]
+                selected = dict((key, selected[key]) for key in wanted if key in selected)
+            out["globals"] = selected
         if args.get("results"):
             layout = {}
             if "results" in handle:
@@ -1236,7 +1257,30 @@ def cmd_data(config, args):
                                 if number is not None:
                                     row[group_name + "." + key] = number
                 rows.append(row)
-        out["scan_table"] = {"scan": scan, "n_points": len(rows), "rows": rows}
+        axis_values = [float(row[scan]) for row in rows if isinstance(row.get(scan), (int, float))]
+        unique = sorted(set(axis_values))
+        steps = [round(b - a, 9) for a, b in zip(unique, unique[1:])]
+        columns = sorted(set(key for row in rows for key in row) - {"shot", scan})
+        table = {
+            "scan": scan,
+            "n_points": len(rows),
+            "axis": {
+                "points": len(unique),
+                "min": unique[0] if unique else None,
+                "max": unique[-1] if unique else None,
+                "step": (sorted(steps)[len(steps) // 2] if steps else None),
+            },
+            "columns": columns,
+        }
+        limit = args.get("limit")
+        if limit:
+            try:
+                limit = max(1, min(20, int(limit)))
+            except Exception:
+                limit = 0
+            if limit:
+                table["rows"] = rows[:limit]
+        out["scan_table"] = table
     return out
 
 

package/python/tests/run_tests.py

@@ -232,7 +232,7 @@ def test_data_reading():
         cfg = make_config(tmp)
         started = bridge.cmd_protocol_start(cfg, {"protocol": protocol_raw()})
         run_folder = started["engage"]["run_folder"]
-        data = bridge.cmd_data(cfg, {"run_folder": run_folder, "scan": "duration_test_us"})
+        data = bridge.cmd_data(cfg, {"run_folder": run_folder, "scan": "duration_test_us", "limit": 1})
         assert data["n_h5"] == 6
         assert data["scan_table"]["n_points"] == 6
         assert data["globals"]["amplitude"] == 0.5
@@ -344,6 +344,7 @@ def test_progress_reports_mapped_run_folder():
     out = bridge.cmd_progress(cfg, {"run_folder": "E:\\Experiments\\nope", "expected_n_shots": 1})
     assert out["run_folder"] == "E:\\Experiments\\nope", out
     assert out["mapped_run_folder"] == "\\\\lab-pc\\Experiments\\nope", out
+    assert out["folder_exists"] is False, out
 
 
 def test_data_missing_run_folder():
@@ -388,6 +389,49 @@ def test_status_drops_missing_keys():
         shutil.rmtree(tmp, ignore_errors=True)
 
 
+def test_status_scan_axes():
+    tmp = tempfile.mkdtemp(prefix="dsh-labscript-scan-axes-")
+    try:
+        cfg = make_config(tmp)
+        bridge.cmd_set(cfg, {"values": {"duration_test_us": [0.0, 0.5, 1.0]}})
+        out = bridge.cmd_status(cfg, {})
+        assert out["scan_axes"].get("duration_test_us") == 3, out["scan_axes"]
+        assert out["n_shots_from_arrays"] == 3, out["n_shots_from_arrays"]
+        assert out["n_shots"] == 3, out["n_shots"]
+        partial = bridge.cmd_status(cfg, {"keys": ["amplitude"]})
+        assert "duration_test_us" not in partial["globals"], partial["globals"]
+        assert partial["scan_axes"].get("duration_test_us") == 3, partial["scan_axes"]
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+
+def test_data_keys_and_scan_summary():
+    tmp = tempfile.mkdtemp(prefix="dsh-labscript-data-keys-")
+    try:
+        cfg = make_config(tmp)
+        started = bridge.cmd_protocol_start(cfg, {"protocol": protocol_raw()})
+        run_folder = started["engage"]["run_folder"]
+        one = bridge.cmd_data(cfg, {"run_folder": run_folder, "keys": ["amplitude"]})
+        assert list(one["globals"].keys()) == ["amplitude"], one["globals"]
+        absent = bridge.cmd_data(cfg, {"run_folder": run_folder, "keys": ["nope"]})
+        assert absent["globals"] == {}, absent["globals"]
+        full = bridge.cmd_data(cfg, {"run_folder": run_folder, "keys": ["*"]})
+        assert len(full["globals"]) > 1, full["globals"]
+        summary = bridge.cmd_data(cfg, {"run_folder": run_folder, "scan": "duration_test_us"})
+        table = summary["scan_table"]
+        assert table["n_points"] == 6, table
+        assert table["axis"]["points"] == 3, table["axis"]
+        assert table["axis"]["min"] == 0.0 and table["axis"]["max"] == 2.0, table["axis"]
+        assert table["axis"]["step"] == 1.0, table["axis"]
+        assert "shot" not in table["columns"] and "duration_test_us" not in table["columns"], table["columns"]
+        assert "rows" not in table, list(table.keys())
+        limited = bridge.cmd_data(cfg, {"run_folder": run_folder, "scan": "duration_test_us", "limit": 2})
+        assert len(limited["scan_table"]["rows"]) == 2, limited["scan_table"]
+        assert limited["scan_table"]["rows"][0]["shot"], limited["scan_table"]["rows"][0]
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+
 def main():
     tests = [value for name, value in sorted(globals().items()) if name.startswith("test_")]
     failures = 0