@uns-kit/core 3.0.44 → 3.0.46

package/MIGRATIONS.md

@@ -9,6 +9,37 @@ Agents must inspect the application's existing ownership and shutdown flow
 before editing it. The examples below describe the intended behavior, not a
 mechanical search-and-replace operation.
 
+## 3.0.46 - Verified retained-store copy and recovery
+
+When crossing `<3.0.46` to `>=3.0.46`, existing rename and unbound consumers
+remain compatible. An explicit
+`transfer: "copy-verified"` adds stopped-source review and bounded asynchronous
+opaque copying between local mounts. Use the same operation/checksum after
+interruption; return current work and retain the original copy. Full Runtime
+recovery requires the matching verified offline retained root before RTT owners
+can be activated. See `docs/retained-store-adoption.md`, bundled as the core
+retained-store guide. No application-specific Controller logic, queue conversion,
+new default handover protocol or automatic disk persistence is introduced.
+
+## 3.0.45 - Opt-in retained-store storage protocol
+
+When crossing `<3.0.45` to `>=3.0.45`, ordinary applications keep their current
+behavior. Applications adopting bound
+retained stores must declare `storageProtocol: 1`, resolve only the shared resource
+paths, acquire `openRetainedStoreLease(binding)` before opening payload writers,
+and release it only after final writer cleanup. Do not infer support from a Kit
+version or default an old omitted capability to supported. Classify manifest
+resources explicitly as backup payload or metadata when needed.
+
+Read the [bundled guide](dist/runtime/retained-store-guide.md) and the two
+unrelated manifests in `dist/runtime/retained-store-examples/`. The source guide
+is `docs/retained-store-adoption.md` in the SDK repository.
+Managed use requires the Runtime's external retained-store root. Older local
+candidate bindings/layouts need reviewed initialization/relocation; the SDK never
+silently changes their absolute roots or converts application data. Snapshot
+requires stopped owners. Preserve original volumes and handle unknown outcomes
+through the same operation ID before restarting any owner.
+
 ## 3.0.44 - Bounded operator JWKS recovery
 
 When crossing `<3.0.44` to `>=3.0.44`, ordinary application handlers, standalone

package/README.md

@@ -29,6 +29,20 @@ npm install @uns-kit/core
 - **MQTT helpers** — resilient publishers, topic builders, throttled queues, and handover support.
 - **gRPC gateway** — infrastructure to bridge Python workers into the UNS message fabric.
 
+## Optional retained stores (3.0.45)
+
+Applications can explicitly declare stable opaque resources for reviewed
+same-host adoption, offline backup and recovery. Import the shared contract from
+`@uns-kit/core/runtime/retained-store.js`; hold its store lease before opening
+payload writers. Ordinary applications retain their existing behavior.
+
+Read the [bundled guide](dist/runtime/retained-store-guide.md) and manifests in
+`dist/runtime/retained-store-examples/`, or the
+[source guide](https://github.com/uns-openhub/uns-kit/blob/master/docs/retained-store-adoption.md).
+Runtime mounts, Controller orchestration and helper releases must also support
+the declared capability. The SDK version alone grants no permissions and does
+not move data, enable queues or change the handover protocol.
+
 ## Basic usage
 
 ```ts

package/dist/runtime/retained-store-copy.d.mts

@@ -0,0 +1,3 @@
+export type RetainedCopyFingerprint = { digest: string; bytes: number; entries: number };
+export function inspectRetainedCopy(root: string, kind: "file" | "directory", limits: unknown): RetainedCopyFingerprint;
+export function applyRetainedCopy(plan: unknown, direction: "adopt" | "return", limits: unknown, afterMove?: (id: string) => void, onFile?: (file: string) => void): void;

package/dist/runtime/retained-store-copy.mjs

@@ -0,0 +1,133 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { safePath, owned, sha, sync, entry, relative, checkLimit } from "./retained-store-storage.mjs";
+
+function inventory(root, kind, limits) {
+  checkLimit(limits);
+  const entries = [];
+  let bytes = 0;
+  function visit(file, rel, depth) {
+    if (depth > limits.maximumDepth || (rel && !relative(rel)) || entries.length >= limits.maximumEntries) throw Error("Retained copy path/count limit");
+    const stat = entry(file);
+    if (!stat) throw Error("Retained copy source is missing");
+    if (stat.isDirectory()) {
+      owned(file, true);
+      entries.push({ path: rel, type: "directory" });
+      for (const name of fs.readdirSync(file).sort()) visit(path.join(file, name), rel ? rel + "/" + name : name, depth + 1);
+    } else {
+      const value = owned(file);
+      bytes += value.size;
+      if (bytes > limits.maximumBytes || value.size > limits.maximumFileBytes) throw Error("Retained copy byte limit");
+      entries.push({ path: rel, type: "file", size: value.size, sha256: sha(file) });
+    }
+  }
+  safePath(root);
+  if ((kind === "directory") !== !!entry(root)?.isDirectory()) throw Error("Retained copy resource kind differs");
+  visit(root, "", 0);
+  return { entries, fingerprint: { digest: createHash("sha256").update(JSON.stringify(entries)).digest("hex"), bytes, entries: entries.length } };
+}
+export function inspectRetainedCopy(root, kind, limits) { return inventory(root, kind, limits).fingerprint; }
+function same(a, b) { return a?.digest === b?.digest && a?.bytes === b?.bytes && a?.entries === b?.entries; }
+function assertFingerprint(file, move, limits) {
+  const stat = entry(file);
+  if (!stat || stat.dev !== move.device || stat.ino !== move.inode || !same(inspectRetainedCopy(file, move.kind, limits), move.content)) throw Error("Retained copy source changed after review");
+}
+function privateStage(stage) {
+  if (!entry(stage)) { fs.mkdirSync(stage, { mode: 0o700 }); sync(path.dirname(stage)); }
+  if (owned(stage, true).mode & 0o077) throw Error("Retained copy staging must be private");
+}
+function checkStage(stage, entries) {
+  const allowed = new Set(["payload", ...entries.map(e => e.path ? "payload/" + e.path : "payload")]);
+  function visit(dir, prefix) {
+    for (const name of fs.readdirSync(dir)) {
+      const rel = prefix ? prefix + "/" + name : name, file = path.join(dir, name);
+      if (!allowed.has(rel)) throw Error("Unexpected retained copy staging entry");
+      const stat = entry(file);
+      owned(file, !!stat?.isDirectory());
+      if (stat.isDirectory()) visit(file, rel);
+    }
+  }
+  visit(stage, "");
+}
+function copyFile(source, destination, expected) {
+  if (entry(destination)) owned(destination);
+  const input = fs.openSync(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
+  let output;
+  try {
+    output = fs.openSync(destination, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_TRUNC | fs.constants.O_NOFOLLOW, 0o600);
+    const h = createHash("sha256"), buffer = Buffer.alloc(65536);
+    let count = 0, size;
+    while ((size = fs.readSync(input, buffer, 0, buffer.length, null))) {
+      count += size;
+      if (count > expected.size) throw Error("Retained copy source grew");
+      h.update(buffer.subarray(0, size));
+      let offset = 0;
+      while (offset < size) offset += fs.writeSync(output, buffer, offset, size - offset);
+    }
+    fs.fchmodSync(output, 0o600); fs.fsyncSync(output);
+    if (count !== expected.size || h.digest("hex") !== expected.sha256) throw Error("Retained copy source changed while reading");
+  } finally { fs.closeSync(input); if(output!==undefined)fs.closeSync(output); }
+}
+function copyClosed(source, destination, stage, move, limits, onFile) {
+  assertFingerprint(source, move, limits);
+  const snapshot = inventory(source, move.kind, limits);
+  if (entry(destination)) {
+    if (!same(inspectRetainedCopy(destination, move.kind, limits), move.content)) throw Error("Published retained copy differs; preserve for review");
+    if (entry(stage)) { checkStage(stage, snapshot.entries); fs.rmSync(stage, { recursive: true }); sync(path.dirname(stage)); }
+    return;
+  }
+  privateStage(stage); checkStage(stage, snapshot.entries);
+  const payload = path.join(stage, "payload");
+  for (const item of snapshot.entries) {
+    const from = item.path ? path.join(source, item.path) : source, to = item.path ? path.join(payload, item.path) : payload;
+    if (item.type === "directory") {
+      if (!entry(to)) fs.mkdirSync(to, { mode: 0o700 });
+      if (owned(to, true).mode & 0o077) throw Error("Retained stage directory must be private");
+    } else {
+      if (entry(to) && owned(to).size === item.size && sha(to) === item.sha256) continue;
+      copyFile(from, to, item); onFile?.(item.path);
+    }
+  }
+  assertFingerprint(source, move, limits);
+  if (!same(inspectRetainedCopy(payload, move.kind, limits), move.content)) throw Error("Retained staged copy differs");
+  for (const item of [...snapshot.entries].reverse()) if (item.type === "directory") sync(item.path ? path.join(payload, item.path) : payload);
+  fs.renameSync(payload, destination); sync(stage); sync(path.dirname(destination));
+  fs.rmdirSync(stage); sync(path.dirname(stage));
+}
+export function applyRetainedCopy(plan, direction, limits, afterMove, onFile) {
+  checkLimit(limits); safePath(plan.source); safePath(plan.directory);
+  for (const move of plan.moves) {
+    const legacy = path.join(plan.source, move.relative), stable = path.join(plan.directory, move.id);
+    const stage = direction === "adopt" ? path.join(plan.directory, ".retained-copy-" + move.id) : path.join(plan.source, ".retained-return-" + plan.storeId + "-" + move.id);
+    if (direction === "adopt") copyClosed(legacy, stable, stage, move, limits, onFile);
+    else {
+      assertFingerprint(stable, move, limits);
+      const original = plan.originalMoves?.find(m => m.id === move.id), saved = path.join(plan.source, ".retained-original-" + plan.storeId + "-" + move.id);
+      if (entry(legacy) && !same(inspectRetainedCopy(legacy, move.kind, limits), move.content)) {
+        if (!original || entry(saved)) throw Error("Return would overwrite unreviewed legacy work");
+        assertFingerprint(legacy, original, limits);
+        fs.renameSync(legacy, saved); sync(path.dirname(legacy));
+      }
+      if (entry(saved)) {
+        if (!original) throw Error("Unreviewed original retained copy");
+        assertFingerprint(saved, original, limits);
+      }
+      copyClosed(stable, legacy, stage, move, limits, onFile);
+    }
+    afterMove?.(move.id);
+  }
+  if (direction === "return") {
+    for (const original of plan.originalMoves ?? []) {
+      const stage = path.join(plan.directory, ".retained-copy-" + original.id);
+      if (!entry(stage)) continue;
+      const legacy = path.join(plan.source, original.relative);
+      if (!plan.moves.some(m => m.id === original.id)) assertFingerprint(legacy, original, limits);
+      const reference = plan.moves.some(m => m.id === original.id) ? path.join(plan.directory, original.id) : legacy;
+      const snapshot = inventory(reference, original.kind, limits);
+      checkStage(stage, snapshot.entries);
+      // Only incomplete scratch is discarded; all original bytes remain proven.
+      fs.rmSync(stage, { recursive: true }); sync(plan.directory);
+    }
+  }
+}

package/dist/runtime/retained-store-examples/index-worker.manifest.json

@@ -0,0 +1,22 @@
+{
+  "protocol": 1,
+  "format": "text-index-v1",
+  "resources": [
+    {
+      "id": "queue",
+      "path": "index-data",
+      "kind": "directory",
+      "required": true,
+      "backup": "payload"
+    },
+    {
+      "id": "replay",
+      "path": "maintenance.json",
+      "kind": "file",
+      "required": false,
+      "backup": "metadata"
+    }
+  ],
+  "legacyVersions": [],
+  "storageProtocol": 1
+}

package/dist/runtime/retained-store-examples/measurement-worker.manifest.json

@@ -0,0 +1,22 @@
+{
+  "protocol": 1,
+  "format": "measurement-journal-v1",
+  "resources": [
+    {
+      "id": "queue",
+      "path": "pending-events",
+      "kind": "directory",
+      "required": true,
+      "backup": "payload"
+    },
+    {
+      "id": "policy",
+      "path": "destination-policy.json",
+      "kind": "file",
+      "required": false,
+      "backup": "metadata"
+    }
+  ],
+  "legacyVersions": ["1.0.0"],
+  "storageProtocol": 1
+}

package/dist/runtime/retained-store-guide.md

@@ -0,0 +1,164 @@
+# Retained stores: opt-in same-host adoption
+
+Retained-store adoption starts in UNS Kit 3.0.45. The 3.0.46 extension adds
+explicit copying between local mounts and verification of recovered stores.
+Managed activation requires coordinated Controller and Runtime releases. Existing applications do not change behavior.
+This is a filesystem capability, separate from the small JSON
+handover-state contract and MQTT output persistence. It does not change the
+default handover protocol, enable disk queues, or transfer data between hosts.
+
+## Ownership
+
+The SDK owns the versioned manifest, binding, safe path validation and resumable
+same-filesystem rename helpers. Controller owns authenticated review, launch
+checks, reservations, durable operation state and managed lifecycle fences.
+The application owns data schema compatibility, its exclusive writer/lease,
+maintenance semantics, replay and database acknowledgements. Runtime/tools own
+durable mounts and backup/restore coverage. No Controller package-name branches
+or execution of application code are required.
+
+Publish `retained-store.manifest.json` at the installed package root. Give each
+opaque resource a stable ID, a relative legacy path, kind and required flag.
+Declare a format only when both versions can safely open the same data. An
+explicit legacy version list permits sources without a manifest; it is a
+compatibility promise by the application, not an automatic schema conversion.
+Keep keys, tokens and startup configuration outside resource declarations.
+
+The two example manifests in `examples/retained-store/` deliberately describe
+different applications/formats. Whole data directories include SQLite WAL/SHM
+sidecars without the Controller reading SQLite or interpreting payloads.
+The npm core package includes this guide at `dist/runtime/retained-store-guide.md`
+and the same manifests at `dist/runtime/retained-store-examples/`.
+
+```ts
+import {
+  readRetainedStoreManifest,
+  resolveRetainedStoreResources,
+  openRetainedStoreLease,
+} from '@uns-kit/core/runtime/retained-store.js';
+
+const resolved = resolveRetainedStoreResources(instanceDirectory,
+  readRetainedStoreManifest(packageDirectory),
+  { rttNode: packageName, version: packageVersion, instanceId });
+const lease = resolved.binding ? openRetainedStoreLease(resolved.binding) : undefined;
+// Now open resolved payload resources and acquire application database leases.
+// Keep the shared lease until writers close; call lease?.close() on shutdown.
+// An attached store enters application maintenance before replay.
+const dataDirectory = resolved.resources.queue;
+```
+
+A binding is private and names an opaque store UUID, generation, operation,
+owner, reviewed configuration digest and stable directory. The shared store
+metadata must match it exactly. A stale owner or fenced instance fails closed.
+The helper does not acquire the application's database lock or authorize an
+operator. Do not bypass either check with an application-specific fallback.
+
+## Host procedure
+
+1. Resolve installed package/instance IDs server-side. Require matching portable
+   configuration, compatible declarations, exact compiled artifact fingerprints,
+   an unoccupied target and one filesystem. Preview never creates storage.
+2. Persist the reviewed selection/digest and, before moving, commit operation
+   intent. Require the reviewed source launch's clean physical exit according to
+   the supervisor; target must also be stopped. Reserve both managed instances.
+3. Use `prepareRetainedStoreMove` and `applyRetainedStoreMove(plan, 'adopt')`.
+   Persist the complete plan before applying it. Moves are atomic per resource,
+   fsynced, inode/device pinned and resumable; there is no copy or overwrite
+   fallback. Several resources do not form one filesystem transaction.
+4. Commit the binding/current owner and durable outcome before releasing the
+   receiving instance fence. Start only the new owner, initially in maintenance.
+5. After stopping that owner, capture `prepareRetainedStoreReturn(plan)` and
+   persist return intent. Return the *current* resources, including new optional
+   or atomically replaced control files. Never restore an old queue snapshot.
+
+Reconcile a timeout by reading/repeating the same operation UUID. A different
+request is not a substitute for recovery. Keep the old-side fence after success.
+A direct manual launch of an older binary outside Controller can ignore a fence;
+this protocol protects managed lifecycle and participating SDK consumers.
+
+## Deliberate limits
+
+POSIX same-user, same-filesystem storage only. No cross-host transport, resource
+merging, schema conversion, automatic deletion, retention finalization, or
+exactly-once destination guarantee. Unknown write acknowledgements keep their
+existing application semantics. Launches, configurations and artifacts changed
+after review are rejected. An absent target requires the existing installation
+workflow first.
+
+Before production release, Runtime and backup tools must explicitly cover the
+stable root, including a queue-excluding backup policy and full restore proof.
+Moving a directory outside the previous archive layout is not backup coverage.
+Consumers must be checked again against published, versioned registry packages;
+a local file tarball is development evidence only.
+
+## Runtime storage and backup extension
+
+A bound application must now explicitly declare `storageProtocol: 1` in its
+manifest and acquire `openRetainedStoreLease(binding)` before opening payloads.
+Keep that handle until final writer shutdown and close it then; the SDK also pins
+live leases process-wide, bounded to32. This is a participation contract, not an
+inference from an installed SDK version. Older unbound applications keep their
+legacy behavior. Bound sources/targets lacking the declaration are rejected by
+managed startup/adoption instead of being guessed safe for offline capture.
+The storage program has been tested on Node 24 with POSIX local filesystems;
+application payload formats remain opaque to it.
+
+The trusted Runtime root is `UNS_RETAINED_STORE_ROOT`; managed adoption requires
+it, and an existing binding must match it exactly. No automatic rebasing of old
+absolute bindings is performed. `store-resources.json` records the declaration,
+including optional per-resource `backup: "payload" | "metadata"` (default payload).
+An incomplete store operation or root restore blocks startup. A returned legacy
+layout leaves a validated empty-store retirement marker, rather than making old
+bindings appear live.
+
+The shared `retained-store-storage.mjs` program supplies leases, offline capture
+and retryable restoration without importing application modules. Runtime tools
+vendor its exact bytes/limits with provenance. Capture requires all bound owners
+stopped; output/input journals retain their existing distinct online protocols.
+The full RTT backup includes the extra capsule inside its existing signed,
+age-encrypted payload. Queue-excluding capture reports excluded payloads and
+cannot be used as a complete queue restore. Restore retains store/owner/generation
+identity at the same logical root on another offline volume, and leaves an applying
+fence until complete. Known private partial staging files may be re-created only
+from the verified snapshot; differing published work is never overwritten.
+
+## Cross-mount transfer and recovery verification in 3.0.46
+
+Same-filesystem rename remains the default.
+For initial relocation to a separately mounted Runtime volume, the host may
+explicitly select `transfer: "copy-verified"`. Stop the source cleanly **before**
+reviewing this transport. Review pins content hashes, aggregate byte/file counts,
+resource identities, both installed artifacts, configuration and supervisor epoch.
+There is no automatic `EXDEV` fallback or caller-selected filesystem path.
+
+The host uses `prepareRetainedStoreMoveAsync`, `applyRetainedStoreMoveAsync` and
+`prepareRetainedStoreReturnAsync`. Bounded worker jobs perform opaque hashing and
+copying while the host event loop can serve health and recovery reads. Persist the
+complete plan before mutation and hold the existing maintenance/lifecycle fences
+through the durable outcome. Jobs have a 128MiB old-generation heap and a 300s
+bound; uncertainty always requires reconciliation of the same operation.
+
+Copy stages and fsyncs each declared resource on its receiving mount, verifies
+its exact contents and atomically publishes it. Source files remain unchanged and
+fenced; allow space for both copies. Known incomplete scratch can be resumed from
+the unchanged reviewed source. Unknown entries, links, changed source bytes and
+differing published files are preserved and refused. The existing limits remain
+32 stores, 100,000 entries, 64GiB aggregate files and 32GiB per file.
+
+Return copies the **current** stopped store back. If current work differs from
+the original, the original is first preserved as
+`.retained-original-<store UUID>-<resource ID>` beside its legacy path. The retired
+stable copy remains inert and is included by full backup; it cannot acquire a
+runtime lease. No cleanup, resource merge, conversion or cross-host transfer is
+implied. Runtime disk capacity and the application's destination acknowledgement
+semantics remain independent operational requirements.
+
+Offline restore now retains a verified local snapshot manifest. The data-only
+`verify-restored` / `verify-root` actions rehash exact declared payloads under an
+exclusive root lease before Runtime activation. A completed restore retry never
+replaces newer application data. Full RTT stage/activation/completion binds the
+verified root, signed backup digest and local manifest digest into its protected
+receipts. Missing, excluded, unfinished or altered storage cannot enable bound
+owners. After application work advances, do not activate an old snapshot: preserve
+current storage and use a fresh reviewed owner transition. Historic operation
+plans pin original physical files and are not rebased after a full restore.

package/dist/runtime/retained-store-job.d.mts

@@ -0,0 +1 @@
+export function retainedStoreJob(job: unknown, onProgress?: (id: string) => void): Promise<unknown>;

package/dist/runtime/retained-store-job.mjs

@@ -0,0 +1,28 @@
+import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads";
+import { inspectRetainedCopy, applyRetainedCopy } from "./retained-store-copy.mjs";
+if (!isMainThread) {
+  const job = workerData;
+  if (job.action === "inspect") parentPort.postMessage({ done: true, result: inspectRetainedCopy(job.path, job.kind, job.limits) });
+  else if (job.action === "apply") {
+    applyRetainedCopy(job.plan, job.direction, job.limits, id => parentPort.postMessage({ progress: id }));
+    parentPort.postMessage({ done: true });
+  } else throw Error("Unknown retained storage job");
+}
+export function retainedStoreJob(job, onProgress) {
+  if (!isMainThread) throw Error("Nested retained storage job");
+  return new Promise((resolve, reject) => {
+    const worker = new Worker(new URL(import.meta.url), { workerData: job, execArgv: [], env: { NODE_NO_WARNINGS: "1" }, resourceLimits: { maxOldGenerationSizeMb: 128 } });
+    let done = false;
+    const timer = setTimeout(() => finish(Error("Retained storage job timed out; reconcile the same operation")), 300000);
+    const finish = (error, result) => {
+      if (done) return; done = true; clearTimeout(timer);
+      void worker.terminate().finally(() => error ? reject(error) : resolve(result));
+    };
+    worker.on("message", message => {
+      if (message.progress) { try { onProgress?.(message.progress); } catch (error) { finish(error); } }
+      if (message.done) finish(undefined, message.result);
+    });
+    worker.once("error", error => finish(error));
+    worker.once("exit", () => { if (!done) finish(Error("Retained storage job exited without confirmation; reconcile the same operation")); });
+  });
+}

package/dist/runtime/retained-store-storage-contract.json

@@ -0,0 +1,9 @@
+{
+  "schemaVersion": 1,
+  "maximumStores": 32,
+  "maximumEntries": 100000,
+  "maximumBytes": 68719476736,
+  "maximumManifestBytes": 33554432,
+  "maximumFileBytes": 34359738368,
+  "maximumDepth": 32
+}

package/dist/runtime/retained-store-storage.d.mts

@@ -0,0 +1,24 @@
+export function rootLease(root: string, exclusive?: boolean, restoring?: boolean): () => void;
+export function maintenanceLease(directory: string): () => void;
+export function runtimeLease(binding: unknown): { close: () => void };
+export function captureRetained(
+  root: string,
+  destination: string,
+  mode: string,
+  limits: unknown,
+): { state: string; stores: number; files: number; payloadBytes: number };
+export function restoreRetained(
+  source: string,
+  root: string,
+  backupDigest: string,
+  limits: unknown,
+  afterPublish?: (id: string) => void,
+): { state: string; stores: number };
+export function available(root: string): void;
+export function owned(path: string, directory?: boolean): import("node:fs").Stats;
+export function sha(path: string): string;
+export function sync(path: string): void;
+export function relative(path: string): boolean;
+export function checkLimit(limits: unknown): void;
+export function verifyRestoredRetained(source:string,root:string,backupDigest:string,limits:unknown):{state:string;stores:number;backupDigest:string;manifestDigest:string;root:string};
+export function verifyRetainedRecoveryRoot(root:string,backupDigest:string,manifestDigest:string,limits:unknown):{state:string;stores:number;backupDigest:string;manifestDigest:string;root:string};