better-dsh 0.3.0 → 0.3.1-b

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +49 -177
  2. package/cordis.patch.yml +5 -36
  3. package/docs/20_specs/kernel-provisioning/spec.md +2 -0
  4. package/docs/adr/0015-independent-bundles-agent-remote.md +40 -0
  5. package/docs/remote-tools.md +56 -0
  6. package/docs/specs/kernel-provisioning/spec.md +12 -40
  7. package/docs/specs/omp-python-repl-port/runtime-guide.md +35 -0
  8. package/docs/specs/omp-python-repl-port/spec.md +68 -0
  9. package/docs/specs/remote-tool-enhancement/spec.md +203 -0
  10. package/dsh-docs/baseline.md +13 -0
  11. package/dsh-docs/subsystems/invariants.md +3 -0
  12. package/dsh-docs/subsystems/invariants.zh.md +3 -0
  13. package/eval-description.md +7 -3
  14. package/lib/LaunchOptions-0Mr6meJc.js +4 -0
  15. package/lib/{LaunchOptions-kGXcZOe0.js → LaunchOptions-iE9M-STS.js} +1 -1
  16. package/lib/{bidi-BalpKaJc.js → bidi-CaQeBZHY.js} +1 -1
  17. package/lib/client/index.js +30 -6
  18. package/lib/failover/index.js +1 -1
  19. package/lib/fs-aware/sandbox-plugin.js +3 -3
  20. package/lib/{index-browser-CysrjMyM.js → index-browser-CJzxT9Vi.js} +4 -4
  21. package/lib/index.d.ts +16 -50
  22. package/lib/index.js +482 -1720
  23. package/lib/kernel-env-x_WkG6Fr.js +76 -0
  24. package/lib/kernel-env.d.ts +5 -69
  25. package/lib/kernel-env.js +2 -2
  26. package/lib/{lsp-server-registry-BXvUTBY7.js → lsp-server-registry-BcdkHROD.js} +1 -1
  27. package/lib/{lsp-server-registry-CH9WNFFt.js → lsp-server-registry-D4VLs1Sm.js} +1 -1
  28. package/lib/{main-BQeFrBM1.js → main-taSMz1KX.js} +1 -1
  29. package/lib/{main-CupOhMp5.js → main-uqGqYYnA.js} +7 -5
  30. package/lib/omp-python/DILL-LICENSE +35 -0
  31. package/lib/omp-python/OMP-LICENSE +23 -0
  32. package/lib/omp-python/better_snapshot.py +391 -0
  33. package/lib/omp-python/better_support.py +425 -0
  34. package/lib/omp-python/dill-provenance.json +55 -0
  35. package/lib/omp-python/provenance.json +13 -0
  36. package/lib/omp-python/runner.py +2499 -0
  37. package/lib/omp-python/vendor/dill/__diff.py +234 -0
  38. package/lib/omp-python/vendor/dill/__info__.py +291 -0
  39. package/lib/omp-python/vendor/dill/__init__.py +119 -0
  40. package/lib/omp-python/vendor/dill/_dill.py +2265 -0
  41. package/lib/omp-python/vendor/dill/_objects.py +545 -0
  42. package/lib/omp-python/vendor/dill/_shims.py +193 -0
  43. package/lib/omp-python/vendor/dill/detect.py +287 -0
  44. package/lib/omp-python/vendor/dill/logger.py +285 -0
  45. package/lib/omp-python/vendor/dill/objtypes.py +24 -0
  46. package/lib/omp-python/vendor/dill/pointers.py +122 -0
  47. package/lib/omp-python/vendor/dill/session.py +612 -0
  48. package/lib/omp-python/vendor/dill/settings.py +25 -0
  49. package/lib/omp-python/vendor/dill/source.py +1025 -0
  50. package/lib/omp-python/vendor/dill/temp.py +252 -0
  51. package/lib/{puppeteer-core-CSk2PtSm.js → puppeteer-core-C-4NyOpY.js} +8 -8
  52. package/lib/remote/helper-bin/MANIFEST.json +35 -0
  53. package/lib/remote/helper-bin/aarch64-apple-darwin/BUILT-VER +1 -0
  54. package/lib/remote/helper-bin/aarch64-apple-darwin/superd-run +0 -0
  55. package/lib/remote/helper-bin/aarch64-unknown-linux-musl/BUILT-VER +1 -0
  56. package/lib/remote/helper-bin/aarch64-unknown-linux-musl/superd-run +0 -0
  57. package/lib/remote/helper-bin/x86_64-pc-windows-gnu/BUILT-VER +1 -0
  58. package/lib/remote/helper-bin/x86_64-pc-windows-gnu/superd-run.exe +0 -0
  59. package/lib/remote/helper-bin/x86_64-unknown-linux-musl/BUILT-VER +1 -0
  60. package/lib/remote/helper-bin/x86_64-unknown-linux-musl/superd-run +0 -0
  61. package/lib/remote/plugin.d.ts +487 -5
  62. package/lib/remote/plugin.js +2739 -213
  63. package/lib/url-schemes/index.js +11 -11
  64. package/lib/web-password.d.ts +1 -1
  65. package/lib/web-password.js +5 -1
  66. package/lib/welcome-notice.d.ts +5 -7
  67. package/lib/welcome-notice.js +9 -48
  68. package/lib/{wrap-T4I0nfPM.js → wrap-0VA0uFx8.js} +1 -1
  69. package/package.json +8 -25
  70. package/scripts/kernel-check.mjs +5 -0
  71. package/lib/LaunchOptions-B874eBu2.js +0 -4
  72. package/lib/bundle-delivery.d.ts +0 -20
  73. package/lib/bundle-delivery.js +0 -209
  74. package/lib/hmr.d.ts +0 -32
  75. package/lib/hmr.js +0 -49
  76. package/lib/kernel-bridge.py +0 -172
  77. package/lib/kernel-env-D8UOWeji.js +0 -195
  78. package/lib/webserver.d.ts +0 -10
  79. package/lib/webserver.js +0 -19
  80. package/scripts/kernel-provision.mjs +0 -35
  81. package/scripts/register-bundle.mjs +0 -80
@@ -0,0 +1,76 @@
1
+ import { existsSync } from "node:fs";
2
+ import { delimiter, dirname, isAbsolute, join, resolve } from "node:path";
3
+ import { execFile } from "node:child_process";
4
+ import { fileURLToPath } from "node:url";
5
+ import { homedir } from "node:os";
6
+ import { promisify } from "node:util";
7
+
8
+ //#region src/kernel-env.ts
9
+ const exec = promisify(execFile);
10
+ const PACKAGE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
11
+ function venvPython(dir) {
12
+ return join(dir, process.platform === "win32" ? "Scripts/python.exe" : "bin/python");
13
+ }
14
+ function activate(dir) {
15
+ const bin = dirname(venvPython(dir));
16
+ return {
17
+ ...process.env,
18
+ VIRTUAL_ENV: dir,
19
+ PATH: [bin, process.env.PATH].filter(Boolean).join(delimiter)
20
+ };
21
+ }
22
+ function explicitPath(python, cwd) {
23
+ const expanded = python === "~" ? homedir() : python.startsWith("~/") ? join(homedir(), python.slice(2)) : python;
24
+ return isAbsolute(expanded) || /[/\\]/.test(expanded) ? resolve(cwd, expanded) : expanded;
25
+ }
26
+ async function probe(python, env, cwd) {
27
+ const { stdout } = await exec(python, ["-c", "import ast, asyncio, json, sys; assert sys.version_info >= (3, 9), \"Packaged dill checkpoints require Python >=3.9\"; print(json.dumps({\"python\": sys.executable, \"version\": sys.version.split()[0]}))"], {
28
+ timeout: 15e3,
29
+ cwd,
30
+ env: {
31
+ ...env,
32
+ PYTHONDONTWRITEBYTECODE: "1"
33
+ }
34
+ });
35
+ const result = JSON.parse(stdout.trim());
36
+ if (!isAbsolute(result.python) || !result.version) throw new Error("Interpreter did not report an absolute executable and version");
37
+ return {
38
+ ...result,
39
+ env
40
+ };
41
+ }
42
+ async function resolveKernelEnv(options = {}) {
43
+ const cwd = resolve(options.cwd ?? process.cwd());
44
+ if (options.python) {
45
+ const python = explicitPath(options.python, cwd);
46
+ const venv$1 = dirname(dirname(python));
47
+ const env = isAbsolute(python) && existsSync(join(venv$1, "pyvenv.cfg")) ? activate(venv$1) : { ...process.env };
48
+ try {
49
+ return await probe(python, env, cwd);
50
+ } catch (error) {
51
+ throw new Error(`Could not use configured Python ${JSON.stringify(options.python)}: ${error instanceof Error ? error.message : String(error)}. Configure python with a working Python >=3.9 interpreter.`);
52
+ }
53
+ }
54
+ const active = process.env.VIRTUAL_ENV || process.env.CONDA_PREFIX;
55
+ const project = [join(cwd, ".venv"), join(cwd, "venv")].find(existsSync);
56
+ const venv = active || project;
57
+ const candidates = [];
58
+ if (venv && existsSync(venvPython(venv))) candidates.push({
59
+ python: venvPython(venv),
60
+ env: activate(venv)
61
+ });
62
+ for (const python of ["python3", "python"]) candidates.push({
63
+ python,
64
+ env: { ...process.env }
65
+ });
66
+ const failures = [];
67
+ for (const candidate of candidates) try {
68
+ return await probe(candidate.python, candidate.env, cwd);
69
+ } catch (error) {
70
+ failures.push(`${candidate.python}: ${error instanceof Error ? error.message : String(error)}`);
71
+ }
72
+ throw new Error(`No working Python interpreter found. Tried: ${failures.join("; ")}. Install Python >=3.9 on PATH (python3 or python), or configure python with an existing interpreter. Better does not install Python automatically.`);
73
+ }
74
+
75
+ //#endregion
76
+ export { resolveKernelEnv as n, PACKAGE_ROOT as t };
@@ -1,80 +1,16 @@
1
1
  //#region src/kernel-env.d.ts
2
- /**
3
- * Kernel environment resolution and provisioning.
4
- *
5
- * The runtime OWNS the interpreter it spawns: it never trusts a bare
6
- * `python3` on the PATH, because that interpreter may lack `ipykernel`
7
- * (hard failure — the kernel cannot start) or `dill` (snapshot capability —
8
- * silently degrading to no snapshots). Instead it resolves one of two
9
- * environments:
10
- *
11
- * - **Explicit** — `config.python` names a real interpreter (a path, or any
12
- * name other than the bare-`python3` sentinel the default/patch produce).
13
- * It is probed, and a missing `ipykernel` is a loud error while a missing
14
- * `dill` is a loud warning (snapshots disabled).
15
- * - **Managed** — no explicit interpreter: the runtime provisions a venv
16
- * UNDER THE PACKAGE (`<packageRoot>/.venv-kernel`, or `config.kernelEnvDir`)
17
- * with `ipykernel` + `dill`, installing it on first use when
18
- * `config.kernelAutoInstall` is on. The venv lives beside the package, NOT
19
- * in `/tmp` (the pre-v0.1.9 dangling-symlink failure mode), so it survives
20
- * the host's tmp reaper and the process restart.
21
- *
22
- * Provisioning uses `uv` when available (fast, can fetch the pinned CPython
23
- * version), falling back to `python3 -m venv` + `ensurepip`. It is idempotent:
24
- * a complete venv is reused untouched; a partial one is repaired.
25
- * @module dashr/kernel-env
26
- */
27
- /** The installed package root (where `lib/` and `src/` live): the default home of the managed venv. */
28
2
  declare const PACKAGE_ROOT: string;
29
- /** The config value meaning "no explicit interpreter — auto-manage a venv". */
30
- declare const AUTO_PYTHON_SENTINEL = "python3";
31
- /** Preferred CPython version for a managed venv. */
32
- declare const DEFAULT_KERNEL_PYTHON_VERSION = "3.11";
33
- /** Pinned, tested `ipykernel` version (2026-09-02 tested value; upgrade = explicit change + regression). */
34
- declare const IPYKERNEL_VERSION = "7.3.0";
35
- /** Pinned, tested `dill` version. */
36
- declare const DILL_VERSION = "0.4.1";
37
3
  interface KernelEnv {
38
- /** Absolute path to the interpreter to spawn (`-m ipykernel_launcher`). */
39
4
  python: string;
40
- /** True when this runtime provisioned a venv (vs an explicit interpreter). */
41
- managed: boolean;
42
- /** The managed venv root, when {@link managed}. */
43
- venvDir: string | undefined;
44
- /** True when `dill` is importable — snapshot capability is available. */
45
- dill: boolean;
46
- /** True when `ipykernel` is importable (a hard requirement). */
47
- ipykernel: boolean;
48
- /** The interpreter's reported version, e.g. `3.11.15`. */
5
+ env: NodeJS.ProcessEnv;
49
6
  version: string;
50
7
  }
51
8
  interface ResolveKernelEnvOptions {
52
- /** `config.python`; the bare sentinel (or absent) selects a managed venv. */
9
+ /** An explicit path or executable name bypasses automatic discovery. */
53
10
  python?: string;
54
- /** Managed venv directory; defaults to `<packageRoot>/.venv-kernel`. */
55
- venvDir?: string;
56
- /** Preferred CPython version for the managed venv. */
57
- pythonVersion?: string;
58
- /** Provision the managed venv when missing/incomplete (default true). */
59
- autoInstall?: boolean;
60
- /** Logger callback (info on provisioning, warn on degraded snapshot). */
61
- log?: (level: 'info' | 'warn', message: string) => void;
11
+ /** Session workspace, used for existing project environments and relative paths. */
12
+ cwd?: string;
62
13
  }
63
- /** Tool-cache env for uv: caches live NEXT TO the venv (plugin territory), so a read-only home can never block provisioning. */
64
- declare function uvToolEnv(venvDir: string): NodeJS.ProcessEnv;
65
- /** uv venv arguments (pure; unit-tested). */
66
- declare function uvVenvArgs(venvDir: string, pythonVersion: string): string[];
67
- /** python3 -m venv fallback arguments (pure; cache-free by construction). */
68
- declare function venvFallbackArgs(venvDir: string): string[];
69
- /** uv pip install arguments with pinned versions (pure; unit-tested). */
70
- declare function uvInstallArgs(python: string): string[];
71
- /** pip install arguments with pinned versions for the venv interpreter (pure). */
72
- declare function pipInstallArgs(): string[];
73
- /**
74
- * Resolve the kernel interpreter, provisioning a managed venv when no
75
- * explicit interpreter is configured. Throws when the resolved interpreter
76
- * cannot run `ipykernel`; a missing `dill` only disables snapshots (warned).
77
- */
78
14
  declare function resolveKernelEnv(options?: ResolveKernelEnvOptions): Promise<KernelEnv>;
79
15
  //#endregion
80
- export { AUTO_PYTHON_SENTINEL, DEFAULT_KERNEL_PYTHON_VERSION, DILL_VERSION, IPYKERNEL_VERSION, KernelEnv, PACKAGE_ROOT, ResolveKernelEnvOptions, pipInstallArgs, resolveKernelEnv, uvInstallArgs, uvToolEnv, uvVenvArgs, venvFallbackArgs };
16
+ export { KernelEnv, PACKAGE_ROOT, ResolveKernelEnvOptions, resolveKernelEnv };
package/lib/kernel-env.js CHANGED
@@ -1,3 +1,3 @@
1
- import { a as PACKAGE_ROOT, c as uvInstallArgs, d as venvFallbackArgs, i as IPYKERNEL_VERSION, l as uvToolEnv, n as DEFAULT_KERNEL_PYTHON_VERSION, o as pipInstallArgs, r as DILL_VERSION, s as resolveKernelEnv, t as AUTO_PYTHON_SENTINEL, u as uvVenvArgs } from "./kernel-env-D8UOWeji.js";
1
+ import { n as resolveKernelEnv, t as PACKAGE_ROOT } from "./kernel-env-x_WkG6Fr.js";
2
2
 
3
- export { AUTO_PYTHON_SENTINEL, DEFAULT_KERNEL_PYTHON_VERSION, DILL_VERSION, IPYKERNEL_VERSION, PACKAGE_ROOT, pipInstallArgs, resolveKernelEnv, uvInstallArgs, uvToolEnv, uvVenvArgs, venvFallbackArgs };
3
+ export { PACKAGE_ROOT, resolveKernelEnv };
@@ -1,3 +1,3 @@
1
- import { a as primaryServerForFile, c as serversForFile, i as installServer, n as findWorkspaceRoot, o as resolveCommandPath, r as installHintFor, s as serverByName, t as clearCommandProbeCache } from "./lsp-server-registry-CH9WNFFt.js";
1
+ import { a as primaryServerForFile, c as serversForFile, i as installServer, n as findWorkspaceRoot, o as resolveCommandPath, r as installHintFor, s as serverByName, t as clearCommandProbeCache } from "./lsp-server-registry-D4VLs1Sm.js";
2
2
 
3
3
  export { clearCommandProbeCache, findWorkspaceRoot, installHintFor, installServer, primaryServerForFile, resolveCommandPath, serverByName, serversForFile };
@@ -1,5 +1,5 @@
1
- import * as path$1 from "node:path";
2
1
  import { accessSync, constants, existsSync, readdirSync } from "node:fs";
2
+ import * as path$1 from "node:path";
3
3
 
4
4
  //#region src/devices/lsp/defaults.json
5
5
  var defaults_default = {
@@ -1,3 +1,3 @@
1
- import { _ as Browser, a as TimeoutError, b as ChromeReleaseChannel, c as computeSystemExecutablePath, d as InstalledBrowser, f as detectBrowserPlatform, g as resolveDefaultUserDataDir, h as resolveBuildId, i as Process, l as launch, m as getVersionComparator, n as uninstall, o as WEBDRIVER_BIDI_WEBSOCKET_ENDPOINT_REGEX, p as createProfile, r as CDP_WEBSOCKET_ENDPOINT_REGEX, s as computeExecutablePath, t as getInstalledBrowsers, u as Cache, v as BrowserPlatform, y as BrowserTag } from "./main-CupOhMp5.js";
1
+ import { _ as Browser, a as TimeoutError, b as ChromeReleaseChannel, c as computeSystemExecutablePath, d as InstalledBrowser, f as detectBrowserPlatform, g as resolveDefaultUserDataDir, h as resolveBuildId, i as Process, l as launch, m as getVersionComparator, n as uninstall, o as WEBDRIVER_BIDI_WEBSOCKET_ENDPOINT_REGEX, p as createProfile, r as CDP_WEBSOCKET_ENDPOINT_REGEX, s as computeExecutablePath, t as getInstalledBrowsers, u as Cache, v as BrowserPlatform, y as BrowserTag } from "./main-uqGqYYnA.js";
2
2
 
3
3
  export { Browser, BrowserPlatform, BrowserTag, CDP_WEBSOCKET_ENDPOINT_REGEX, Cache, ChromeReleaseChannel, InstalledBrowser, Process, TimeoutError, WEBDRIVER_BIDI_WEBSOCKET_ENDPOINT_REGEX, computeExecutablePath, computeSystemExecutablePath, createProfile, detectBrowserPlatform, getInstalledBrowsers, getVersionComparator, launch, resolveBuildId, resolveDefaultUserDataDir, uninstall };
@@ -1,14 +1,16 @@
1
1
  import { createHash } from "node:crypto";
2
- import childProcess, { execFile, execSync, spawn, spawnSync } from "node:child_process";
3
- import { mkdir, readdir, unlink } from "node:fs/promises";
2
+ import fs, { accessSync, constants, createReadStream, createWriteStream, existsSync, readFileSync } from "node:fs";
4
3
  import path from "node:path";
4
+ import childProcess, { execFile, execSync, spawn, spawnSync } from "node:child_process";
5
+ import * as http from "node:http";
6
+ import readline from "node:readline";
5
7
  import { URL as URL$1, urlToHttpOptions } from "node:url";
6
- import fs, { accessSync, constants, existsSync, readFileSync } from "node:fs";
7
8
  import os from "node:os";
8
- import readline from "node:readline";
9
9
  import { debuglog, promisify } from "node:util";
10
+ import { mkdir, readdir, symlink, unlink } from "node:fs/promises";
11
+ import { pipeline } from "node:stream/promises";
12
+ import { Writable } from "node:stream";
10
13
  import { EventEmitter } from "node:events";
11
- import * as http from "node:http";
12
14
  import * as https from "node:https";
13
15
 
14
16
  //#region node_modules/@puppeteer/browsers/lib/browser-data/types.js
@@ -0,0 +1,35 @@
1
+ Copyright (c) 2004-2016 California Institute of Technology.
2
+ Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
3
+ All rights reserved.
4
+
5
+ This software is available subject to the conditions and terms laid
6
+ out below. By downloading and using this software you are agreeing
7
+ to the following conditions.
8
+
9
+ Redistribution and use in source and binary forms, with or without
10
+ modification, are permitted provided that the following conditions
11
+ are met:
12
+
13
+ - Redistributions of source code must retain the above copyright
14
+ notice, this list of conditions and the following disclaimer.
15
+
16
+ - Redistributions in binary form must reproduce the above copyright
17
+ notice, this list of conditions and the following disclaimer in the
18
+ documentation and/or other materials provided with the distribution.
19
+
20
+ - Neither the names of the copyright holders nor the names of any of
21
+ the contributors may be used to endorse or promote products derived
22
+ from this software without specific prior written permission.
23
+
24
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
26
+ TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27
+ PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
28
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
29
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
30
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
31
+ OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32
+ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
33
+ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
34
+ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35
+
@@ -0,0 +1,23 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Mario Zechner
4
+ Copyright (c) 2025-2026 Can Bölük
5
+ Copyright (c) 2026 Stencil Labs, Inc.
6
+
7
+ Permission is hereby granted, free of charge, to any person obtaining a copy
8
+ of this software and associated documentation files (the "Software"), to deal
9
+ in the Software without restriction, including without limitation the rights
10
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11
+ copies of the Software, and to permit persons to whom the Software is
12
+ furnished to do so, subject to the following conditions:
13
+
14
+ The above copyright notice and this permission notice shall be included in all
15
+ copies or substantial portions of the Software.
16
+
17
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23
+ SOFTWARE.
@@ -0,0 +1,391 @@
1
+ """Best-effort checkpoints for the owned OMP namespace; dill 0.4.1 adapter.
2
+
3
+ No cell history is stored/replayed. Pickling user objects is not a sandbox:
4
+ reducers may execute arbitrary code. Host deadlines bound helper execution.
5
+ """
6
+ import _thread
7
+ import asyncio
8
+ import dis
9
+ import functools
10
+ import hashlib
11
+ import io
12
+ import json
13
+ import os
14
+ import platform
15
+ import re
16
+ import socket
17
+ import sqlite3
18
+ import subprocess
19
+ import sys
20
+ import threading
21
+ import time
22
+ import types
23
+ import uuid
24
+
25
+ # Select the unchanged packaged implementation before ANY user cell can import dill.
26
+ _VENDOR = os.path.join(os.path.dirname(__file__), 'vendor')
27
+ sys.path.insert(0, _VENDOR)
28
+ import dill
29
+ if dill.__version__ != '0.4.1' or os.path.realpath(dill.__file__) != os.path.realpath(os.path.join(_VENDOR, 'dill', '__init__.py')):
30
+ raise RuntimeError('Better requires its packaged dill 0.4.1')
31
+
32
+ _DIRECTORY = None
33
+ _CAP = None
34
+ _TIMEOUT = 30.0
35
+ _FORMAT = 2
36
+ _RESOURCES = (io.IOBase, _thread.LockType, type(threading.RLock()), socket.socket,
37
+ subprocess.Popen, threading.Thread, sqlite3.Connection, sqlite3.Cursor,
38
+ asyncio.Future, asyncio.AbstractEventLoop, types.GeneratorType,
39
+ types.CoroutineType, types.AsyncGeneratorType)
40
+ _PRIMITIVES = (str, bytes, int, float, bool, type(None))
41
+
42
+
43
+ class UnsupportedResource(Exception):
44
+ pass
45
+
46
+
47
+ class SizeCap(Exception):
48
+ pass
49
+
50
+
51
+ class NamespaceProxy:
52
+ pass
53
+
54
+
55
+ def _proxy(ns):
56
+ proxy = NamespaceProxy()
57
+ proxy.__dict__ = ns
58
+ return proxy
59
+
60
+
61
+ def configure(directory, cap, timeout_ms):
62
+ global _DIRECTORY, _CAP, _TIMEOUT
63
+ _DIRECTORY = os.path.abspath(directory) if directory else None
64
+ _CAP = cap
65
+ _TIMEOUT = timeout_ms / 1000.0 * 0.9
66
+
67
+
68
+ def clear():
69
+ """Delete only our fixed manifest and generation files, including orphans."""
70
+ if _DIRECTORY and os.path.islink(_DIRECTORY):
71
+ raise ValueError('checkpoint directory cannot be a symlink')
72
+ if not _DIRECTORY or not os.path.isdir(_DIRECTORY):
73
+ return
74
+ for name in os.listdir(_DIRECTORY):
75
+ if name == 'manifest.json' or re.fullmatch(r'(state-[0-9a-f]{32}\.dill|\.checkpoint-[0-9a-f]{32}\.tmp)', name):
76
+ os.unlink(os.path.join(_DIRECTORY, name))
77
+
78
+
79
+ def _paths(payload_path, manifest_path):
80
+ if not _DIRECTORY or os.path.abspath(manifest_path) != os.path.join(_DIRECTORY, 'manifest.json') or os.path.abspath(payload_path) != os.path.join(_DIRECTORY, 'state.dill'):
81
+ raise ValueError('checkpoint paths are not the configured session directory')
82
+ os.makedirs(_DIRECTORY, mode=0o700, exist_ok=True)
83
+ if os.path.islink(_DIRECTORY):
84
+ raise ValueError('checkpoint directory cannot be a symlink')
85
+ if os.path.islink(manifest_path):
86
+ raise ValueError('checkpoint manifest cannot be a symlink')
87
+ os.chmod(_DIRECTORY, 0o700)
88
+
89
+
90
+ def _environment():
91
+ return dict(pythonVersion=platform.python_version(), implementation=platform.python_implementation(),
92
+ interpreter=os.path.realpath(sys.executable), venvPath=sys.prefix,
93
+ environmentHash=hashlib.sha256(json.dumps([sys.prefix, sys.base_prefix, sys.path], sort_keys=True).encode()).hexdigest(),
94
+ dillVersion=dill.__version__, protocol=4, skills=[])
95
+
96
+
97
+ def _guard(ns):
98
+ import better_support
99
+ owned = [sys.modules['__main__'], sys.modules['better_support'], sys.modules[__name__]]
100
+ owned.extend(better_support.owned_objects())
101
+ denied = {id(value) for value in owned if not isinstance(value, _PRIMITIVES)}
102
+ def guard(value):
103
+ if id(value) in denied or (isinstance(value, (types.FunctionType, type)) and getattr(value, '__module__', None) in ('better_support', 'better_snapshot')) or type(value).__module__ == 'better_support':
104
+ raise UnsupportedResource('captured Better/runner binding')
105
+ if isinstance(value, _RESOURCES):
106
+ raise UnsupportedResource('unsupported live resource ' + type(value).__name__)
107
+ return None
108
+ return guard
109
+
110
+
111
+ class CappedBuffer(io.BytesIO):
112
+ def __init__(self, cap):
113
+ super().__init__()
114
+ self.cap = cap
115
+ def write(self, value):
116
+ if self.tell() + len(value) > self.cap:
117
+ raise SizeCap('serialized graph exceeds snapshotSizeCapBytes')
118
+ return super().write(value)
119
+
120
+
121
+ def _encode(ns, values, cap, guard):
122
+ output = CappedBuffer(cap)
123
+ # Exact Pickler: subclassing disables pinned dill's main-module logic.
124
+ pickler = dill.Pickler(output, protocol=4, recurse=False, byref=False)
125
+ pickler._main = _proxy(ns)
126
+ pickler._original_main = pickler._main
127
+ pickler._main_modified = True
128
+ pickler.persistent_id = guard
129
+ pickler.dump(values)
130
+ return output.getvalue()
131
+
132
+
133
+ def _decode(ns, payload):
134
+ unpickler = dill.Unpickler(io.BytesIO(payload))
135
+ unpickler._main = _proxy(ns)
136
+ unpickler._ignore = True
137
+ return unpickler.load()
138
+
139
+
140
+ def _dependencies(root, ns):
141
+ """Bounded static global scan; dynamic eval/globals lookups remain best effort."""
142
+ found, seen = set(), set()
143
+ def walk(value):
144
+ if id(value) in seen or isinstance(value, _PRIMITIVES):
145
+ return
146
+ seen.add(id(value))
147
+ if len(seen) > 10000:
148
+ raise UnsupportedResource('dependency graph exceeds inspection budget')
149
+ if isinstance(value, types.FunctionType):
150
+ if value.__globals__ is ns:
151
+ def code_names(code):
152
+ for op in dis.get_instructions(code):
153
+ if op.opname in ('LOAD_GLOBAL', 'LOAD_NAME'):
154
+ found.add(op.argval)
155
+ for item in code.co_consts:
156
+ if isinstance(item, types.CodeType): code_names(item)
157
+ code_names(value.__code__)
158
+ walk(value.__defaults__)
159
+ walk(value.__kwdefaults__)
160
+ for cell in value.__closure__ or ():
161
+ try: walk(cell.cell_contents)
162
+ except ValueError: pass
163
+ elif isinstance(value, functools.partial):
164
+ walk(value.func); walk(value.args); walk(value.keywords); walk(vars(value))
165
+ elif isinstance(value, (staticmethod, classmethod)):
166
+ walk(value.__func__)
167
+ elif isinstance(value, property):
168
+ walk(value.fget); walk(value.fset); walk(value.fdel)
169
+ elif isinstance(value, types.MethodType):
170
+ walk(value.__func__); walk(value.__self__)
171
+ elif isinstance(value, dict):
172
+ for key, item in value.items(): walk(key); walk(item)
173
+ elif isinstance(value, (tuple, list, set, frozenset)):
174
+ for item in value: walk(item)
175
+ elif isinstance(value, type):
176
+ if value.__module__ == ns.get('__name__'):
177
+ for item in vars(value).values(): walk(item)
178
+ for base in value.__bases__: walk(base)
179
+ elif not isinstance(value, types.ModuleType) and type(value).__module__ == ns.get('__name__'):
180
+ walk(type(value))
181
+ if hasattr(value, '__dict__'): walk(vars(value))
182
+ # Scan descriptors across the MRO, including inherited/mangled slots.
183
+ for owner in type(value).__mro__:
184
+ for descriptor in vars(owner).values():
185
+ if isinstance(descriptor, types.MemberDescriptorType):
186
+ try: walk(descriptor.__get__(value, type(value)))
187
+ except AttributeError: pass # uninitialized slot
188
+ walk(root)
189
+ return found
190
+
191
+
192
+ def _write_new(path, data):
193
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
194
+ with os.fdopen(fd, 'wb') as output:
195
+ output.write(data)
196
+ output.flush()
197
+ os.fsync(output.fileno())
198
+
199
+
200
+ def _emit(sentinel, result):
201
+ print(sentinel + json.dumps(result, ensure_ascii=True))
202
+
203
+
204
+ def _retained_payloads(manifest_path):
205
+ try:
206
+ with open(manifest_path, encoding='utf-8') as source:
207
+ manifest = json.load(source)
208
+ except (FileNotFoundError, ValueError):
209
+ return set(), None
210
+ if not isinstance(manifest, dict):
211
+ return set(), None
212
+ current = manifest.get('payload')
213
+ names = (current, manifest.get('previousPayload'))
214
+ retained = {name for name in names if isinstance(name, str) and re.fullmatch(r'state-[0-9a-f]{32}\.dill', name)}
215
+ return retained, current if isinstance(current, str) and current in retained else None
216
+
217
+
218
+ def _scavenge(retained):
219
+ for name in os.listdir(_DIRECTORY):
220
+ if name not in retained and re.fullmatch(r'(state-[0-9a-f]{32}\.dill|\.checkpoint-[0-9a-f]{32}\.tmp)', name):
221
+ os.unlink(os.path.join(_DIRECTORY, name))
222
+
223
+
224
+ def save(ns, payload_path, manifest_path, spec_json, sentinel):
225
+ attempt_paths, committed, result = [], False, None
226
+ payload_name, manifest, retained = None, None, set()
227
+ try:
228
+ _paths(payload_path, manifest_path)
229
+ retained, previous = _retained_payloads(manifest_path)
230
+ _scavenge(retained) # collect killed-writer artifacts even if this attempt fails
231
+ deadline = time.monotonic() + _TIMEOUT
232
+ def check_deadline():
233
+ if time.monotonic() >= deadline:
234
+ raise TimeoutError('checkpoint deadline exceeded; previous generation retained')
235
+ spec = json.loads(spec_json)
236
+ cap = min(spec['sizeCapBytes'], _CAP)
237
+ import better_support
238
+ candidates = better_support.user_namespace(ns)
239
+ guard = _guard(ns)
240
+ kept, omitted, dependencies = {}, {}, {}
241
+ for name, value in candidates.items():
242
+ check_deadline()
243
+ try:
244
+ _encode(ns, {name: value}, cap, guard)
245
+ dependencies[name] = _dependencies(value, ns)
246
+ kept[name] = value
247
+ except SizeCap:
248
+ raise
249
+ except Exception as error:
250
+ omitted[name] = type(error).__name__ + ': ' + str(error)
251
+ changed = True
252
+ while changed:
253
+ changed = False
254
+ for name in list(kept):
255
+ missing = dependencies[name].intersection(omitted)
256
+ if missing:
257
+ omitted[name] = 'depends on omitted globals: ' + ', '.join(sorted(missing))
258
+ del kept[name]
259
+ changed = True
260
+ check_deadline()
261
+ payload = _encode(ns, kept, cap, guard)
262
+ # Verify the FINAL shared graph, never independently reconstruct values.
263
+ trial_ns = dict(ns)
264
+ trial = _decode(trial_ns, payload)
265
+ if not isinstance(trial, dict) or set(trial) != set(kept):
266
+ raise ValueError('final graph round-trip changed admitted names')
267
+ trial_ns.update(trial)
268
+ groups = {}
269
+ for name, value in kept.items():
270
+ if id(value) in groups and trial[name] is not trial[groups[id(value)]]:
271
+ raise ValueError('final graph round-trip lost shared root identity')
272
+ groups[id(value)] = name
273
+ token = uuid.uuid4().hex
274
+ payload_name = 'state-' + token + '.dill'
275
+ manifest = dict(_environment(), snapshotFormat=_FORMAT, turn=spec['turn'],
276
+ payload=payload_name, previousPayload=previous, sizeBytes=len(payload), sha256=hashlib.sha256(payload).hexdigest(),
277
+ names=sorted(kept), dependencies={name: sorted(dependencies[name]) for name in kept},
278
+ bindingNames=sorted(better_support.injected_names()), omitted=omitted, coverage='partial' if omitted else 'best-effort',
279
+ skipped=False, limitations=['Dynamic globals/eval and arbitrary extension resources cannot be exhaustively inspected.'])
280
+ payload_file = os.path.join(_DIRECTORY, payload_name)
281
+ attempt_paths.append(payload_file)
282
+ _write_new(payload_file, payload)
283
+ temp = os.path.join(_DIRECTORY, '.checkpoint-' + token + '.tmp')
284
+ attempt_paths.append(temp)
285
+ _write_new(temp, json.dumps(manifest, sort_keys=True).encode())
286
+ # Sole commit point. Cleanup must never remove this payload afterwards.
287
+ check_deadline()
288
+ os.replace(temp, manifest_path)
289
+ committed = True
290
+ result = dict(kind='ok', sizeBytes=len(payload), names=manifest['names'], omitted=omitted, turn=spec['turn'])
291
+ warnings = []
292
+ if os.name == 'posix':
293
+ try:
294
+ fd = os.open(_DIRECTORY, os.O_RDONLY)
295
+ try: os.fsync(fd)
296
+ finally: os.close(fd)
297
+ except OSError as error:
298
+ warnings.append('directory fsync failed: ' + str(error))
299
+ try:
300
+ _scavenge({payload_name, previous})
301
+ except OSError as error:
302
+ warnings.append('stale artifact cleanup failed: ' + str(error))
303
+ if warnings:
304
+ result['warning'] = 'checkpoint committed; ' + '; '.join(warnings)
305
+ except SizeCap as error:
306
+ result = dict(kind='skipped', reason=str(error), capBytes=_CAP)
307
+ except Exception as error:
308
+ reason = type(error).__name__ + ': ' + str(error)
309
+ if payload_name is not None and not committed:
310
+ try:
311
+ _, pointed = _retained_payloads(manifest_path)
312
+ committed = pointed == payload_name
313
+ except OSError:
314
+ pass
315
+ if committed:
316
+ result = dict(kind='ok', sizeBytes=len(payload), names=manifest['names'], omitted=omitted, turn=spec['turn'])
317
+ result['warning'] = 'checkpoint committed; postcommit finalization failed: ' + reason
318
+ else:
319
+ result = dict(kind='failed', reason=reason)
320
+ finally:
321
+ if not committed and attempt_paths:
322
+ # A signal/error can arrive immediately after replace, before the
323
+ # local flag changes. The pointer is authoritative for deletion.
324
+ try:
325
+ latest, _ = _retained_payloads(manifest_path)
326
+ retained.update(latest)
327
+ except OSError:
328
+ # Cannot prove non-commit: leave recognized artifacts for the
329
+ # next save's pre-write scavenger rather than risk data loss.
330
+ retained.update(os.path.basename(path) for path in attempt_paths)
331
+ for path in attempt_paths:
332
+ if os.path.basename(path) in retained:
333
+ continue
334
+ try: os.unlink(path)
335
+ except FileNotFoundError: pass
336
+ except OSError as error:
337
+ if result is not None:
338
+ result['reason'] += '; uncommitted artifact cleanup failed: ' + str(error)
339
+ if result is not None:
340
+ _emit(sentinel, result)
341
+
342
+
343
+ def restore(ns, payload_path, manifest_path, sentinel, bindings_json):
344
+ try:
345
+ _paths(payload_path, manifest_path)
346
+ if os.path.getsize(manifest_path) > 1048576:
347
+ raise ValueError('manifest exceeds metadata cap')
348
+ with open(manifest_path, encoding='utf-8') as source:
349
+ manifest = json.load(source)
350
+ if not isinstance(manifest, dict) or manifest.get('snapshotFormat') != _FORMAT:
351
+ raise ValueError('unsupported snapshot format')
352
+ for key, value in _environment().items():
353
+ if manifest.get(key) != value: raise ValueError('checkpoint environment mismatch: ' + key)
354
+ name = manifest.get('payload', '')
355
+ if not isinstance(name, str) or not re.fullmatch(r'state-[0-9a-f]{32}\.dill', name):
356
+ raise ValueError('invalid owned payload name')
357
+ names = manifest.get('names')
358
+ import better_support
359
+ if not isinstance(names, list) or any(not isinstance(n, str) or n.startswith('__omp_') or n in ('__name__', '__builtins__', '__doc__') for n in names):
360
+ raise ValueError('invalid checkpoint namespace names')
361
+ turn = manifest.get('turn')
362
+ if not isinstance(turn, int) or isinstance(turn, bool) or turn < 0:
363
+ raise ValueError('invalid checkpoint turn')
364
+ path = os.path.join(_DIRECTORY, name)
365
+ size = os.path.getsize(path)
366
+ if os.path.islink(path) or size != manifest.get('sizeBytes') or size > _CAP:
367
+ raise ValueError('payload size/cap/path validation failed')
368
+ with open(path, 'rb') as source: payload = source.read(_CAP + 1)
369
+ if hashlib.sha256(payload).hexdigest() != manifest.get('sha256'):
370
+ raise ValueError('payload integrity hash mismatch')
371
+ values = _decode(ns, payload)
372
+ if not isinstance(values, dict) or sorted(values) != sorted(names):
373
+ raise ValueError('payload namespace differs from manifest')
374
+ omitted = dict(manifest.get('omitted', {}))
375
+ unavailable = set(manifest.get('bindingNames', [])) - set(json.loads(bindings_json))
376
+ dependencies = manifest.get('dependencies', {})
377
+ changed = True
378
+ while changed:
379
+ changed = False
380
+ for root in list(values):
381
+ missing_bindings = set(dependencies.get(root, [])).intersection(unavailable)
382
+ missing_state = set(dependencies.get(root, [])).intersection(omitted)
383
+ if missing_bindings or missing_state:
384
+ omitted[root] = ('missing current host binding: ' + ', '.join(sorted(missing_bindings))) if missing_bindings else ('depends on omitted globals: ' + ', '.join(sorted(missing_state)))
385
+ del values[root]
386
+ changed = True
387
+ ns.update(values)
388
+ _emit(sentinel, dict(kind='restored', count=len(values), turn=turn, names=names,
389
+ omitted=omitted))
390
+ except Exception as error:
391
+ _emit(sentinel, dict(kind='degraded', reason=type(error).__name__ + ': ' + str(error)))