dsh-origin-bridge 0.4.5 → 0.4.7

package/asciiio.py

@@ -10,6 +10,7 @@ This reader sniffs the delimiter, handles BOM and GBK, skips comment lines, and
 understands the common "long name / units" two-row header.
 """
 
+import codecs
 import csv
 import os
 import re
@@ -100,13 +101,72 @@ def read_metadata(head, k):
     return meta
 
 
+_BOMS = ((codecs.BOM_UTF32_LE, "utf-32-le"), (codecs.BOM_UTF32_BE, "utf-32-be"),
+         (codecs.BOM_UTF16_LE, "utf-16-le"), (codecs.BOM_UTF16_BE, "utf-16-be"),
+         (codecs.BOM_UTF8, "utf-8-sig"))
+
+# Some instrument software and Windows' "Unicode Text" save write UTF-16 with no
+# BOM at all. Half of every byte is then 0x00, which no other codec survives.
+_NUL_RATIO = 0.2
+
+
+def high_byte_runs(data):
+    """Lengths of maximal runs of bytes >= 0x80.
+
+    GBK-family text pairs them: one Chinese character is two high bytes. cp1252 /
+    latin-1 punctuation shows up as isolated singles — µ, °, ±, ·. That is the only
+    honest way to tell "温度/C" from "T/°C", because gb18030 will decode *both*
+    without raising and silently mojibake the second one into 祍/癈.
+    """
+    runs, cur = [], 0
+    for byte in data:
+        if byte >= 0x80:
+            cur += 1
+        elif cur:
+            runs.append(cur)
+            cur = 0
+    if cur:
+        runs.append(cur)
+    return runs
+
+
+def candidate_encodings(data):
+    order = list(ENCODINGS)
+    runs = high_byte_runs(data)
+    if runs and any(run % 2 for run in runs):
+        gb = order.index("gb18030")
+        singles = [enc for enc in ("cp1252", "latin-1") if enc in order]
+        for enc in singles:
+            order.remove(enc)
+        order[gb:gb] = singles
+    return order
+
+
 def read_text(path):
     """Decode a data file, trying the encodings that actually occur in labs."""
+    with open(path, "rb") as fh:
+        data = fh.read()
+    for bom, enc in _BOMS:
+        if data.startswith(bom):
+            body = data[len(bom):]
+            if enc == "utf-8-sig":
+                return body.decode("utf-8"), enc
+            try:
+                return body.decode(enc), enc
+            except UnicodeDecodeError as exc:
+                raise ValueError("文件带 %s 字节头但解不开：%s（%s）" % (enc, path, exc))
+    if len(data) >= 8 and data.count(b"\x00") / float(len(data)) > _NUL_RATIO:
+        for enc in ("utf-16-le", "utf-16-be"):
+            try:
+                text = data.decode(enc)
+            except UnicodeDecodeError:
+                continue
+            if "\x00" not in text:
+                return text, "%s（无 BOM，按 NUL 分布判断）" % enc
     last = None
-    for enc in ENCODINGS:
+    for enc in candidate_encodings(data):
         try:
-            with open(path, "r", encoding=enc, newline="") as fh:
-                return fh.read(), enc
+            return data.decode(enc), enc
         except (UnicodeDecodeError, LookupError) as exc:
             last = exc
     raise ValueError("无法解码 %s（试过 %s），最后错误：%s" % (path, "/".join(ENCODINGS), last))
@@ -126,13 +186,21 @@ def sniff_delimiter(rows):
         width = max(widths)
         if width < 2:
             continue
-        # score: how many columns are numeric in >80% of rows, tie-broken by width
+        # score: numeric columns first, then how many rows actually have that width,
+        # then FEWER columns. Ranking by width alone let a wrong comma split of a
+        # European "0,0;1,5" file beat the correct ';' because it made three columns.
+        consistent = sum(1 for g in grid if len(g) == width) / float(len(grid))
+        # The first row is a header candidate, so keep it out of the numeric ratio —
+        # otherwise a European "Zeit;Signal / 0,0;1,5" file scores every delimiter as
+        # non-numeric and the widest wrong split wins by default.
+        data = grid[1:] if len(grid) >= 2 else grid
         hits = 0
         for col in range(width):
-            vals = [g[col] for g in grid if len(g) > col]
-            if vals and sum(1 for v in vals if _is_number(v)) >= 0.8 * len(vals):
+            vals = [g[col] for g in data if len(g) > col]
+            if vals and sum(1 for v in vals if _is_number(v) or _CELL_DECCOMA.match(v.strip())) \
+                    >= 0.8 * len(vals):
                 hits += 1
-        score = (hits, width)
+        score = (hits, round(consistent, 3), -width)
         if score > best_score:
             best, best_score = delim, score
     if best is not None:
@@ -149,6 +217,92 @@ def sniff_delimiter(rows):
     raise ValueError("识别不出分隔符，也没有多列空白对齐的数据")
 
 
+_NUM_SRC = r"[+-]?(?:\d+\.?\d*|\.\d+)(?:[eEdD][+-]?\d+)?"
+# Lab exports decorate their numbers. Dropping a whole column because one cell is
+# decorated silently loses real data, so each shape below is recognised, unwrapped,
+# and reported as a warning instead of quietly skipped.
+_CELL_DL = re.compile(r"^(?:[<>]=?|≥|≤)\s*(%s)$" % _NUM_SRC)          # <0.001, >1e6
+_CELL_PM = re.compile(r"^(%s)\s*(?:±|\+/-|\+-)\s*\d*\.?\d+$" % _NUM_SRC)  # 2.5±0.1
+_CELL_UNIT = re.compile(r"^(%s)\s+\S+$" % _NUM_SRC)                   # "25 mV"
+# Two numbers sharing one cell (a space-aligned dump read with the wrong delimiter)
+# must not be dressed up as "number + unit", or the second one disappears quietly.
+_CELL_TWO = re.compile(r"^(%s)\s+(%s)\s*$" % (_NUM_SRC, _NUM_SRC))
+_CELL_FLAG = re.compile(r"^(%s)[*†‡°\s]+$" % _NUM_SRC)                # "2.5*"
+_CELL_THOUSANDS = re.compile(r"^[+-]?\d{1,3}(?:,\d{3})+(?:\.\d+)?$")  # "1,234.5"
+# European exports use the comma as the decimal mark, which only parses when the
+# field separator is something else (usually ';').
+_CELL_DECCOMA = re.compile(r"^[+-]?\d+,\d+$")                          # "0,5"
+_CELL_PAREN = re.compile(r"^\((%s)\)$" % _NUM_SRC)                     # "(500)"
+_MISSING_TOKENS = ("NA", "NAN", "-", "--", "NIL", "N/A", "NULL", "#N/A", "#DIV/0!")
+
+CELL_KIND_NOTES = {
+    "检测限": "第 %d 列 %r 有 %d 个检测限标记（例如 <0.001），已按边界值代入",
+    "不确定度": "第 %d 列 %r 有 %d 个 ± 写法（例如 2.5±0.1），已取中心值",
+    "带单位的数": "第 %d 列 %r 有 %d 个数值把单位写在同一格（例如 25 mV），已取数值",
+    "一格两个数": "第 %d 列 %r 有 %d 个格子里挤了两个数（空格分隔），已取第一个；"
+                  "若那本是两列，请把分隔符设为空格或改用制表符/CSV 再导",
+    "千分位": "第 %d 列 %r 有 %d 个千分位写法（例如 1,234.5），已去掉分隔逗号",
+    "小数逗号": "第 %d 列 %r 有 %d 个用逗号当小数点的写法（例如 0,5），已按小数解析",
+    "标记后缀": "第 %d 列 %r 有 %d 个带 * / † 标记的值，已按数值本身代入",
+    "括号负数": "第 %d 列 %r 有 %d 个用括号表示负数的写法（例如 (500)），已取负值",
+    "溢出": "第 %d 列 %r 有 %d 个数值超出浮点范围（例如 1e400），已按缺值处理",
+}
+
+
+def _to_float(text):
+    return float(text.replace("d", "E").replace("D", "E"))
+
+
+def _finite(value):
+    return value == value and value not in (float("inf"), float("-inf"))
+
+
+def cell_number(cell, decimal_comma=False):
+    """One cell -> (value or None, kind or None). kind is what we had to forgive.
+
+    Lab exports decorate their numbers: detection limits, uncertainties, a unit in
+    the same cell, thousands separators, outlier flags. Dropping the whole column
+    because one cell is decorated silently loses real data, so each shape is
+    recognised, unwrapped, and reported as a warning instead of quietly skipped.
+    """
+    text = str(cell).strip().strip('"').strip("'")
+    if not text or text.upper() in _MISSING_TOKENS:
+        return None, None
+    if decimal_comma and _CELL_DECCOMA.match(text):
+        value = _to_float(text.replace(",", "."))
+        return (value, "小数逗号") if _finite(value) else (None, "溢出")
+    try:
+        value = _to_float(text)
+    except ValueError:
+        pass
+    else:
+        return (value, None) if _finite(value) else (None, "溢出")
+    if _CELL_THOUSANDS.match(text):
+        value = _to_float(text.replace(",", ""))
+        return (value, "千分位") if _finite(value) else (None, "溢出")
+    m = _CELL_PAREN.match(text)
+    if m:
+        value = -abs(_to_float(m.group(1)))
+        return (value, "括号负数") if _finite(value) else (None, "溢出")
+    m = _CELL_TWO.match(text)
+    if m:
+        # Checked before "number + unit": "2.5 3.5" is a mis-split column, not a
+        # measurement with a unit, and calling it a unit loses the second number
+        # without ever telling anyone.
+        value = _to_float(m.group(1))
+        return (value, "一格两个数") if _finite(value) else (None, "溢出")
+    for rx, kind in ((_CELL_DL, "检测限"), (_CELL_PM, "不确定度"),
+                     (_CELL_UNIT, "带单位的数"), (_CELL_FLAG, "标记后缀")):
+        m = rx.match(text)
+        if m:
+            try:
+                value = _to_float(m.group(1))
+            except ValueError:
+                return None, None
+            return (value, kind) if _finite(value) else (None, "溢出")
+    return None, None
+
+
 def parse(path, comment=None):
     """Return {columns:[{name,units,values}], rows, delimiters, encoding, warnings}."""
     text, enc = read_text(path)
@@ -167,6 +321,10 @@ def parse(path, comment=None):
         grid = [next(csv.reader([ln], delimiter=delim)) for ln in kept]
 
     width = max(len(r) for r in grid)
+    # Deciding this here matters twice: the header/data split below and the cell
+    # reader both need to know whether a bare "0,5" is a European decimal number
+    # rather than two comma-separated fields.
+    decimal_comma = delim not in (",", None)
     if len({len(r) for r in grid}) > 1:
         warnings.append("行长度不一致，已按最宽 %d 列补齐" % width)
     grid = [r + [""] * (width - len(r)) for r in grid]
@@ -174,7 +332,8 @@ def parse(path, comment=None):
     # split leading header rows (rows that are mostly non-numeric) from data
     head, body = [], []
     for i, row in enumerate(grid):
-        numeric = sum(1 for c in row if _is_number(c))
+        numeric = sum(1 for c in row if _is_number(c)
+                      or (decimal_comma and _CELL_DECCOMA.match(c.strip())))
         if numeric >= max(1, int(0.6 * width)):
             body = grid[i:]
             break
@@ -205,22 +364,30 @@ def parse(path, comment=None):
     columns = []
     for c in range(width):
         vals = []
+        kinds = {}
         for row in body:
             cell = row[c].strip()
-            if not cell or cell.upper() in ("NA", "NAN", "-", "--"):
-                vals.append(None)
-                continue
-            try:
-                vals.append(float(cell.replace("d", "E").replace("D", "E")))
-            except ValueError:
-                vals.append(None)
+            value, kind = cell_number(cell, decimal_comma=decimal_comma)
+            if kind:
+                kinds[kind] = kinds.get(kind, 0) + 1
+            vals.append(value)
         live = sum(1 for v in vals if v is not None)
+        name = raw_names[c] or "col%d" % (c + 1)
+        # Report what we forgave even when the column ends up dropped: "全为空" alone
+        # hides that every cell was text (a row-label column) or overflowed.
+        for kind, count in sorted(kinds.items()):
+            warnings.append(CELL_KIND_NOTES[kind] % (c + 1, name, count))
         if live == 0:
-            warnings.append("第 %d 列 %r 解析后全为空，已跳过" % (c + 1, raw_names[c] or "col%d" % (c + 1)))
+            blanks = sum(1 for row in body if not row[c].strip())
+            if blanks == len(body):
+                continue  # a genuinely empty trailing column, nothing to explain
+            warnings.append("第 %d 列 %r 没有一个能读成数值（%d 行），已跳过；"
+                            "如果它是行标签/文本列属正常，要画它请选别的列"
+                            % (c + 1, name, len(body)))
             continue
         columns.append({
             "index": c + 1,
-            "name": raw_names[c] or "col%d" % (c + 1),
+            "name": name,
             "units": units[c] or "",
             "values": vals,
             "numeric_rows": live,

package/cordis.patch.yml

@@ -17,8 +17,12 @@
 #   manual install -> set ORIGIN_BRIDGE_HOME to the folder holding server.py
 #   interpreter    -> set ORIGIN_BRIDGE_PYTHON to your venv python.exe
 #
-# failOnStartupError: false keeps dsh bootable when Origin is missing or a modal
-# dialog stalls the COM server; the tools just do not register that session.
+# failOnStartupError: false keeps dsh bootable when this entry cannot start at all —
+# measured (2026-10-19) that means only a missing interpreter: the default command is
+# `python` from PATH. A missing originpro does NOT hide the tools: all 27 still
+# register and every call answers `missing_originpro` with the fix to run. Origin
+# itself being closed is irrelevant to startup — the COM connection is made lazily
+# inside the first tool call, not here.
 
 - insert:
     - id: mcp-origin-bridge

package/fixtures/l_cp1252_micro.dat

@@ -0,0 +1,3 @@
+T/V	s/�s
+1	2
+3	4

package/fixtures/m_cp1252_degree.dat

@@ -0,0 +1,3 @@
+T/�C	y
+25	1.2
+30	1.4

package/fixtures/make_fixtures.py

@@ -20,6 +20,21 @@ FILES = {
     "g_na_and_D.dat": ("utf-8", "X\tY\tZ\n0\t1.5D+00\tNA\n1\t2.5d+00\t-\n2\tNA\t3.1\n"),
     "h_gbk_units.dat": ("gbk", "温度\t电导\n25.0\t1.23\n30.0\t1.45\n"),
     "i_bom_utf8.csv": ("utf-8-sig", "波长,吸光度\n200,0.12\n210,0.34\n"),
+    # Encodings and decorated numbers that come out of real instruments.
+    "j_utf16_bom.dat": ("utf-16", "X\tY\n0\t1.5\n1\t2.5\n"),
+    "k_utf16_nobom.dat": ("utf-16-le", "X\tY\n0\t1.5\n1\t2.5\n"),
+    "l_cp1252_micro.dat": ("cp1252", "T/V\ts/µs\n1\t2\n3\t4\n"),
+    "m_cp1252_degree.dat": ("cp1252", "T/°C\ty\n25\t1.2\n30\t1.4\n"),
+    "n_detection_limit.dat": ("utf-8", "x\ty\n1\t<0.001\n2\t0.5\n3\t>1e6\n"),
+    "o_plusminus.dat": ("utf-8", "x\ty\n1\t2.5±0.1\n2\t3.5 +/- 0.2\n"),
+    "p_unit_in_cell.dat": ("utf-8", "x\ty\n1\t25 mV\n2\t30 mV\n"),
+    "q_thousands.csv": ("utf-8", "time,signal\n0,\"1,234.5\"\n1,\"2,345.6\"\n"),
+    "r_decimal_comma.txt": ("utf-8", "Zeit;Signal\n0,0;1,5\n1,0;2,5\n"),
+    "s_star_flag.dat": ("utf-8", "x\ty\n1\t2.5*\n2\t3.5\n"),
+    "t_paren_negative.dat": ("utf-8", "x\ty\n1\t(500)\n2\t250\n"),
+    "u_overflow.dat": ("utf-8", "x\ty\n1\t1e400\n2\t1.2e308\n"),
+    "v_row_labels.dat": ("utf-8", "ch\tV\tI\nCh1\t1.0\t2.0\nCh2\t3.0\t4.0\n"),
+    "w_two_numbers_cell.csv": ("utf-8", "x,y\n1,\"2.5 3.5\"\n2,\"4.0 5.0\"\n"),
 }
 
 

package/fixtures/n_detection_limit.dat

@@ -0,0 +1,4 @@
+x	y
+1	<0.001
+2	0.5
+3	>1e6

package/fixtures/o_plusminus.dat

@@ -0,0 +1,3 @@
+x	y
+1	2.5±0.1
+2	3.5 +/- 0.2

package/fixtures/p_unit_in_cell.dat

@@ -0,0 +1,3 @@
+x	y
+1	25 mV
+2	30 mV

package/fixtures/q_thousands.csv

@@ -0,0 +1,3 @@
+time,signal
+0,"1,234.5"
+1,"2,345.6"

package/fixtures/r_decimal_comma.txt

@@ -0,0 +1,3 @@
+Zeit;Signal
+0,0;1,5
+1,0;2,5

package/fixtures/s_star_flag.dat

@@ -0,0 +1,3 @@
+x	y
+1	2.5*
+2	3.5

package/fixtures/t_paren_negative.dat

@@ -0,0 +1,3 @@
+x	y
+1	(500)
+2	250

package/fixtures/u_overflow.dat

@@ -0,0 +1,3 @@
+x	y
+1	1e400
+2	1.2e308

package/fixtures/v_row_labels.dat

@@ -0,0 +1,3 @@
+ch	V	I
+Ch1	1.0	2.0
+Ch2	3.0	4.0

package/fixtures/w_two_numbers_cell.csv

@@ -0,0 +1,3 @@
+x,y
+1,"2.5 3.5"
+2,"4.0 5.0"