@remnic/core 1.1.30 → 9.3.515

package/src/network/webdav.ts

@@ -1,5 +1,6 @@
-import { createReadStream } from "node:fs";
-import { mkdir, readdir, realpath, stat } from "node:fs/promises";
+import { constants } from "node:fs";
+import type { FileHandle } from "node:fs/promises";
+import { mkdir, open, readdir, realpath, stat } from "node:fs/promises";
 import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
 import { timingSafeEqual } from "node:crypto";
 import path from "node:path";
@@ -38,10 +39,49 @@ interface AllowedRoot {
   name: string;
 }
 
+type WebDavReadOpenResult =
+  | { ok: true; handle: FileHandle; size: number }
+  | { ok: false; code: number; message: string };
+
+function validateWebDavAuth(auth: WebDavAuth): WebDavAuth {
+  if (typeof auth.username !== "string" || auth.username.trim().length === 0) {
+    throw new Error("webdav auth.username must be a non-empty string");
+  }
+  if (typeof auth.password !== "string" || auth.password.trim().length === 0) {
+    throw new Error("webdav auth.password must be a non-empty string");
+  }
+  return auth;
+}
+
+export async function openWebDavFileForRead(absolutePath: string): Promise<WebDavReadOpenResult> {
+  let handle: FileHandle | null = null;
+  try {
+    handle = await open(absolutePath, constants.O_RDONLY | constants.O_NOFOLLOW);
+    const info = await handle.stat();
+    if (!info.isFile()) {
+      await handle.close().catch(() => {});
+      return { ok: false, code: 403, message: "path is not a file" };
+    }
+    return { ok: true, handle, size: info.size };
+  } catch (err) {
+    await handle?.close().catch(() => {});
+    const code = (err as NodeJS.ErrnoException).code;
+    if (code === "ELOOP") {
+      return { ok: false, code: 403, message: "path escaped allowlist via symlink" };
+    }
+    if (code === "ENOENT" || code === "ENOTDIR") {
+      return { ok: false, code: 404, message: "not found" };
+    }
+    return { ok: false, code: 404, message: "not found" };
+  }
+}
+
 export class WebDavServer {
   private readonly options: Required<Omit<WebDavServerOptions, "auth">> & Pick<WebDavServerOptions, "auth">;
   private readonly allowedRoots: AllowedRoot[];
   private server: Server | null = null;
+  private startPromise: Promise<WebDavServerStatus> | null = null;
+  private listening = false;
   private boundPort: number;
 
   private constructor(
@@ -59,7 +99,7 @@ export class WebDavServer {
       host: input.host ?? "127.0.0.1",
       port: input.port,
       allowlistDirs: input.allowlistDirs,
-      auth: input.auth,
+      auth: input.auth ? validateWebDavAuth(input.auth) : undefined,
     };
 
     if (!Array.isArray(options.allowlistDirs) || options.allowlistDirs.length === 0) {
@@ -90,9 +130,10 @@ export class WebDavServer {
     if (!this.options.enabled) {
       throw new Error("webdav server is disabled; set enabled=true to start");
     }
-    if (this.server) {
+    if (this.server && this.listening) {
       return this.status();
     }
+    if (this.startPromise) return this.startPromise;
 
     const server = createServer((req, res) => {
       this.handle(req, res).catch((err) => {
@@ -105,48 +146,80 @@ export class WebDavServer {
       });
     });
     this.server = server;
+    this.listening = false;
+
+    this.startPromise = (async () => {
+      try {
+        await new Promise<void>((resolve, reject) => {
+          const onError = (err: Error) => {
+            server.removeListener("listening", onListening);
+            server.removeListener("close", onClose);
+            reject(err);
+          };
+          const onListening = () => {
+            server.removeListener("error", onError);
+            server.removeListener("close", onClose);
+            resolve();
+          };
+          const onClose = () => {
+            server.removeListener("error", onError);
+            server.removeListener("listening", onListening);
+            reject(new Error("webdav server closed before listening"));
+          };
+          server.once("error", onError);
+          server.once("listening", onListening);
+          server.once("close", onClose);
+          server.listen(this.options.port, this.options.host);
+        });
+      } catch (err) {
+        if (this.server === server) {
+          this.server = null;
+        }
+        this.listening = false;
+        server.close();
+        throw err;
+      }
 
-    try {
-      await new Promise<void>((resolve, reject) => {
-        const onError = (err: Error) => {
-          server.removeListener("listening", onListening);
-          reject(err);
-        };
-        const onListening = () => {
-          server.removeListener("error", onError);
-          resolve();
-        };
-        server.once("error", onError);
-        server.once("listening", onListening);
-        server.listen(this.options.port, this.options.host);
-      });
-    } catch (err) {
-      this.server = null;
-      server.close();
-      throw err;
-    }
+      const address = server.address();
+      if (address && typeof address !== "string") {
+        this.boundPort = address.port;
+      }
+      this.listening = true;
 
-    const address = server.address();
-    if (address && typeof address !== "string") {
-      this.boundPort = address.port;
-    }
+      return this.status();
+    })();
 
-    return this.status();
+    try {
+      return await this.startPromise;
+    } finally {
+      this.startPromise = null;
+    }
   }
 
   async stop(): Promise<void> {
     if (!this.server) return;
     const server = this.server;
-    this.server = null;
-    this.boundPort = this.options.port;
+    const pendingStart = this.startPromise;
     await new Promise<void>((resolve, reject) => {
-      server.close((err) => (err ? reject(err) : resolve()));
+      server.close((err) => {
+        if (err && (err as NodeJS.ErrnoException).code !== "ERR_SERVER_NOT_RUNNING") {
+          reject(err);
+          return;
+        }
+        resolve();
+      });
     });
+    await pendingStart?.catch(() => undefined);
+    if (this.server === server) {
+      this.server = null;
+    }
+    this.listening = false;
+    this.boundPort = this.options.port;
   }
 
   status(): WebDavServerStatus {
     return {
-      running: this.server !== null,
+      running: this.server !== null && this.listening,
       host: this.options.host,
       port: this.boundPort,
       rootCount: this.allowedRoots.length,
@@ -182,12 +255,12 @@ export class WebDavServer {
     }
 
     if (method === "PROPFIND") {
-      await this.handlePropfind(resolved.absolutePath, resolved.displayPath, res);
+      await this.handlePropfind(resolved.absolutePath, resolved.rootAbsolute, resolved.displayPath, res);
       return;
     }
 
     if (method === "GET" || method === "HEAD") {
-      await this.handleRead(method, resolved.absolutePath, res);
+      await this.handleRead(method, resolved.absolutePath, resolved.rootAbsolute, res);
       return;
     }
 
@@ -241,7 +314,7 @@ export class WebDavServer {
   }
 
   private async resolvePath(requestPathname: string): Promise<
-    | { ok: true; absolutePath: string; displayPath: string }
+    | { ok: true; absolutePath: string; displayPath: string; rootAbsolute: string }
     | { ok: false; code: number; message: string }
   > {
     let decodedPath: string;
@@ -282,11 +355,11 @@ export class WebDavServer {
       if (!this.isPathInside(root.absolute, canonicalCandidate)) {
         return { ok: false, code: 403, message: "path escaped allowlist via symlink" };
       }
-      return { ok: true, absolutePath: canonicalCandidate, displayPath: `/${segments.join("/")}` };
+      return { ok: true, absolutePath: canonicalCandidate, displayPath: `/${segments.join("/")}`, rootAbsolute: root.absolute };
     } catch (err) {
       const code = (err as NodeJS.ErrnoException).code;
       if (code === "ENOENT") {
-        return { ok: true, absolutePath: candidate, displayPath: `/${segments.join("/")}` };
+        return { ok: true, absolutePath: candidate, displayPath: `/${segments.join("/")}`, rootAbsolute: root.absolute };
       }
       if (code === "ENOTDIR" || code === "ELOOP") {
         return { ok: false, code: 400, message: "invalid path" };
@@ -295,36 +368,57 @@ export class WebDavServer {
     }
   }
 
-  private async handleRead(method: "GET" | "HEAD", absolutePath: string, res: ServerResponse): Promise<void> {
-    let info;
-    try {
-      info = await stat(absolutePath);
-    } catch {
-      res.writeHead(404, { "Content-Type": "text/plain; charset=utf-8" });
-      res.end("not found");
+  private async handleRead(
+    method: "GET" | "HEAD",
+    absolutePath: string,
+    rootAbsolute: string,
+    res: ServerResponse,
+  ): Promise<void> {
+    const revalidated = await this.revalidatePathInsideRoot(absolutePath, rootAbsolute);
+    if (!revalidated.ok) {
+      res.writeHead(revalidated.code, { "Content-Type": "text/plain; charset=utf-8" });
+      res.end(revalidated.message);
       return;
     }
-
-    if (!info.isFile()) {
-      res.writeHead(403, { "Content-Type": "text/plain; charset=utf-8" });
-      res.end("path is not a file");
+    const opened = await openWebDavFileForRead(revalidated.absolutePath);
+    if (!opened.ok) {
+      res.writeHead(opened.code, { "Content-Type": "text/plain; charset=utf-8" });
+      res.end(opened.message);
       return;
     }
 
-    res.writeHead(200, {
-      "Content-Length": String(info.size),
-      "Content-Type": "application/octet-stream",
-    });
+    const { handle, size } = opened;
 
-    if (method === "HEAD") {
-      res.end();
-      return;
-    }
+    try {
+      res.writeHead(200, {
+        "Content-Length": String(size),
+        "Content-Type": "application/octet-stream",
+      });
+
+      if (method === "HEAD") {
+        res.end();
+        return;
+      }
 
-    await pipeline(createReadStream(absolutePath), res);
+      await pipeline(handle.createReadStream({ autoClose: false }), res);
+    } finally {
+      await handle.close().catch(() => {});
+    }
   }
 
-  private async handlePropfind(absolutePath: string, displayPath: string, res: ServerResponse): Promise<void> {
+  private async handlePropfind(
+    absolutePath: string,
+    rootAbsolute: string,
+    displayPath: string,
+    res: ServerResponse,
+  ): Promise<void> {
+    const revalidated = await this.revalidatePathInsideRoot(absolutePath, rootAbsolute);
+    if (!revalidated.ok) {
+      res.writeHead(revalidated.code, { "Content-Type": "text/plain; charset=utf-8" });
+      res.end(revalidated.message);
+      return;
+    }
+    absolutePath = revalidated.absolutePath;
     let info;
     try {
       info = await stat(absolutePath);
@@ -366,6 +460,31 @@ export class WebDavServer {
     }
     return target.startsWith(`${root}${path.sep}`);
   }
+
+  private async revalidatePathInsideRoot(
+    absolutePath: string,
+    rootAbsolute: string,
+  ): Promise<
+    | { ok: true; absolutePath: string }
+    | { ok: false; code: number; message: string }
+  > {
+    try {
+      const canonical = await realpath(absolutePath);
+      if (!this.isPathInside(rootAbsolute, canonical)) {
+        return { ok: false, code: 403, message: "path escaped allowlist via symlink" };
+      }
+      return { ok: true, absolutePath: canonical };
+    } catch (err) {
+      const code = (err as NodeJS.ErrnoException).code;
+      if (code === "ENOENT" || code === "ENOTDIR") {
+        return { ok: false, code: 404, message: "not found" };
+      }
+      if (code === "ELOOP") {
+        return { ok: false, code: 403, message: "path escaped allowlist via symlink" };
+      }
+      throw err;
+    }
+  }
 }
 
 function xmlEscape(value: string): string {

package/src/offline-sync.ts

@@ -594,21 +594,18 @@ async function readPlainFileContentChunk(options: {
   }
 }
 
-export async function buildOfflineSyncSnapshot(options: {
+export async function* iterateOfflineSyncSnapshotFileRecords(options: {
   root: string;
-  sourceId: string;
   includeContent?: boolean;
   includeTranscripts?: boolean;
-  now?: Date;
   readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
   readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
-}): Promise<OfflineSyncSnapshot> {
+}): AsyncIterable<OfflineSyncFileRecord> {
   const rootAbs = path.resolve(options.root);
-  const root = await prepareSafeArchiveRoot(rootAbs, "buildOfflineSyncSnapshot", "root");
+  const root = await prepareSafeArchiveRoot(rootAbs, "iterateOfflineSyncSnapshotFileRecords", "root");
   const includeTranscripts = options.includeTranscripts !== false;
-  const files: OfflineSyncFileRecord[] = [];
 
-  async function walk(dirAbs: string): Promise<void> {
+  async function* walk(dirAbs: string): AsyncIterable<OfflineSyncFileRecord> {
     let entries = await readdir(dirAbs, { withFileTypes: true });
     entries = entries.sort((left, right) => left.name.localeCompare(right.name));
     for (const entry of entries) {
@@ -617,22 +614,36 @@ export async function buildOfflineSyncSnapshot(options: {
       if (shouldExcludeRelPath(relPosix, includeTranscripts)) continue;
       if (entry.isSymbolicLink()) continue;
       if (entry.isDirectory()) {
-        await walk(abs);
+        yield* walk(abs);
         continue;
       }
       if (!entry.isFile()) continue;
-      files.push(await readOfflineSyncFileRecord({
+      yield await readOfflineSyncFileRecord({
         root,
         relPath: relPosix,
         filePath: abs,
         includeContent: options.includeContent === true,
         readFile: options.readFile,
         readFileDigest: options.readFileDigest,
-      }));
+      });
     }
   }
 
-  await walk(root.abs);
+  yield* walk(root.abs);
+}
+
+export async function buildOfflineSyncSnapshot(options: {
+  root: string;
+  sourceId: string;
+  includeContent?: boolean;
+  includeTranscripts?: boolean;
+  now?: Date;
+  readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
+  readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
+}): Promise<OfflineSyncSnapshot> {
+  const includeTranscripts = options.includeTranscripts !== false;
+  const files: OfflineSyncFileRecord[] = [];
+  for await (const file of iterateOfflineSyncSnapshotFileRecords(options)) files.push(file);
 
   return {
     format: OFFLINE_SYNC_SNAPSHOT_FORMAT,

package/src/onboarding/index.test.ts

@@ -0,0 +1,105 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { onboard } from "./index.js";
+
+test("onboard normalizes trailing separators before root-level shape detection", async () => {
+  const projectDir = await mkdtemp(path.join(os.tmpdir(), "remnic-onboard-library-"));
+  try {
+    await writeFile(
+      path.join(projectDir, "package.json"),
+      JSON.stringify({ name: "library", exports: "./index.js" }),
+      "utf8",
+    );
+    await writeFile(path.join(projectDir, "README.md"), "# Library\n", "utf8");
+
+    const result = onboard({ directory: `${projectDir}${path.sep}` });
+
+    assert.equal(result.directory, projectDir);
+    assert.equal(result.shape, "library");
+    assert.deepEqual(result.shapeEvidence, ["package.json has exports/main"]);
+  } finally {
+    await rm(projectDir, { recursive: true, force: true });
+  }
+});
+
+test("onboard resolves relative directory input before returning docs", async () => {
+  const projectDir = await mkdtemp(path.join(os.tmpdir(), "remnic-onboard-relative-"));
+  try {
+    await writeFile(path.join(projectDir, "README.md"), "# Relative\n", "utf8");
+    const relativeDir = path.relative(process.cwd(), projectDir);
+
+    const result = onboard({ directory: relativeDir });
+
+    assert.equal(result.directory, path.resolve(relativeDir));
+    assert.equal(result.docs.length, 1);
+    assert.equal(path.isAbsolute(result.docs[0]!.path), true);
+    assert.equal(result.docs[0]!.path, path.join(projectDir, "README.md"));
+    assert.equal(result.docs[0]!.relativePath, "README.md");
+  } finally {
+    await rm(projectDir, { recursive: true, force: true });
+  }
+});
+
+test("onboard does not treat package.json alone as TypeScript or JavaScript", async () => {
+  const projectDir = await mkdtemp(path.join(os.tmpdir(), "remnic-onboard-package-only-"));
+  try {
+    await writeFile(path.join(projectDir, "package.json"), JSON.stringify({ name: "pkg" }), "utf8");
+
+    const result = onboard({ directory: projectDir });
+
+    assert.deepEqual(result.languages.map((entry) => entry.language), []);
+  } finally {
+    await rm(projectDir, { recursive: true, force: true });
+  }
+});
+
+test("onboard detects JavaScript from source files without adding TypeScript from package.json", async () => {
+  const projectDir = await mkdtemp(path.join(os.tmpdir(), "remnic-onboard-js-"));
+  try {
+    await writeFile(path.join(projectDir, "package.json"), JSON.stringify({ name: "pkg" }), "utf8");
+    await writeFile(path.join(projectDir, "index.js"), "export const value = 1;\n", "utf8");
+
+    const result = onboard({ directory: projectDir });
+
+    assert.deepEqual(result.languages.map((entry) => entry.language), ["JavaScript"]);
+  } finally {
+    await rm(projectDir, { recursive: true, force: true });
+  }
+});
+
+test("onboard detects TypeScript from TypeScript-specific evidence", async () => {
+  const projectDir = await mkdtemp(path.join(os.tmpdir(), "remnic-onboard-ts-"));
+  try {
+    await writeFile(path.join(projectDir, "package.json"), JSON.stringify({ name: "pkg" }), "utf8");
+    await writeFile(path.join(projectDir, "index.ts"), "export const value: number = 1;\n", "utf8");
+
+    const result = onboard({ directory: projectDir });
+
+    assert.deepEqual(result.languages.map((entry) => entry.language), ["TypeScript"]);
+  } finally {
+    await rm(projectDir, { recursive: true, force: true });
+  }
+});
+
+test("onboard rejects missing and file-backed scan roots", async () => {
+  const projectDir = await mkdtemp(path.join(os.tmpdir(), "remnic-onboard-invalid-root-"));
+  try {
+    assert.throws(
+      () => onboard({ directory: path.join(projectDir, "missing") }),
+      /Cannot scan onboarding directory/,
+    );
+
+    const filePath = path.join(projectDir, "README.md");
+    await writeFile(filePath, "# Not a directory\n", "utf8");
+    assert.throws(
+      () => onboard({ directory: filePath }),
+      /not a directory/,
+    );
+  } finally {
+    await rm(projectDir, { recursive: true, force: true });
+  }
+});

package/src/onboarding/index.ts

@@ -12,7 +12,7 @@ import path from "node:path";
 
 export interface OnboardOptions {
   /** Directory to scan (defaults to cwd) */
-  directory: string;
+  directory?: string;
   /** Max depth to walk (default: 6) */
   maxDepth?: number;
   /** Directories to skip */
@@ -84,13 +84,13 @@ const LANGUAGE_RULES: LanguageRule[] = [
   {
     language: "TypeScript",
     extensions: [".ts", ".tsx"],
-    manifests: ["package.json"],
+    manifests: [],
     configFiles: ["tsconfig.json", "tsup.config.ts"],
   },
   {
     language: "JavaScript",
     extensions: [".js", ".jsx", ".mjs", ".cjs"],
-    manifests: ["package.json"],
+    manifests: [],
     configFiles: [".eslintrc", ".prettierrc"],
   },
   {
@@ -182,10 +182,19 @@ const DEFAULT_EXCLUDE = new Set([
 export function onboard(options: OnboardOptions): OnboardResult {
   const startTime = Date.now();
   const {
-    directory,
     maxDepth = 6,
     excludeDirs = [],
   } = options;
+  const directory = path.resolve(options.directory ?? process.cwd());
+  let rootStat: fs.Stats;
+  try {
+    rootStat = fs.statSync(directory);
+  } catch (err) {
+    throw new Error(`Cannot scan onboarding directory ${directory}: ${err instanceof Error ? err.message : String(err)}`);
+  }
+  if (!rootStat.isDirectory()) {
+    throw new Error(`Cannot scan onboarding directory ${directory}: not a directory`);
+  }
 
   const exclude = new Set([...DEFAULT_EXCLUDE, ...excludeDirs]);
 
@@ -230,7 +239,10 @@ function walkDir(
     let entries: fs.Dirent[];
     try {
       entries = fs.readdirSync(dir, { withFileTypes: true });
-    } catch {
+    } catch (err) {
+      if (depth === 0) {
+        throw new Error(`Cannot scan onboarding directory ${root}: ${err instanceof Error ? err.message : String(err)}`);
+      }
       return;
     }
 

package/src/operator-toolkit.ts

@@ -4,7 +4,7 @@ import { access, mkdir, readFile, readdir, stat, unlink, writeFile } from "node:
 import { lintWorkspaceFiles } from "./hygiene.js";
 import { parseConfig } from "./config.js";
 import { readEnvVar, resolveHomeDir } from "./runtime/env.js";
-import { resolveRemnicPluginEntry } from "./plugin-id.js";
+import { resolvePluginEntry } from "./plugin-entry-resolver.js";
 import {
   resolveCuratedIncludeFilesStatePath,
   resolveNativeKnowledgeStatePath,
@@ -49,6 +49,41 @@ import type {
 import { reportBufferSurpriseDistribution } from "./buffer-surprise-report.js";
 import { readJudgeVerdictStats } from "./extraction-judge-telemetry.js";
 
+const OPENCLAW_REMNIC_PLUGIN_IDS = ["openclaw-remnic", "openclaw-engram"] as const;
+
+function getOpenClawPluginEntries(raw: Record<string, unknown>): Record<string, unknown> | undefined {
+  const plugins =
+    raw["plugins"] && typeof raw["plugins"] === "object" && !Array.isArray(raw["plugins"])
+      ? (raw["plugins"] as Record<string, unknown>)
+      : undefined;
+  const entries =
+    plugins && plugins["entries"] && typeof plugins["entries"] === "object" && !Array.isArray(plugins["entries"])
+      ? (plugins["entries"] as Record<string, unknown>)
+      : undefined;
+  return entries;
+}
+
+function getOpenClawMemorySlotId(raw: Record<string, unknown>): string | undefined {
+  const plugins =
+    raw["plugins"] && typeof raw["plugins"] === "object" && !Array.isArray(raw["plugins"])
+      ? (raw["plugins"] as Record<string, unknown>)
+      : undefined;
+  const slots =
+    plugins && plugins["slots"] && typeof plugins["slots"] === "object" && !Array.isArray(plugins["slots"])
+      ? (plugins["slots"] as Record<string, unknown>)
+      : undefined;
+  const slotId = slots?.["memory"];
+  return typeof slotId === "string" ? slotId : undefined;
+}
+
+function resolveOpenClawRemnicPluginEntry(raw: unknown): Record<string, unknown> | undefined {
+  return resolvePluginEntry(raw, {
+    candidateIds: OPENCLAW_REMNIC_PLUGIN_IDS,
+    getEntries: getOpenClawPluginEntries,
+    getSlotId: getOpenClawMemorySlotId,
+  });
+}
+
 interface QmdRuntimeLike {
   probe(): Promise<boolean>;
   isAvailable(): boolean;
@@ -394,9 +429,9 @@ async function loadCliPluginConfig(configPath?: string): Promise<OperatorConfigL
   const resolvedPath = resolveConfigPath(configPath);
   try {
     const raw = JSON.parse(await readFile(resolvedPath, "utf-8")) as Record<string, unknown>;
-    // Delegate slot → PLUGIN_ID → LEGACY_PLUGIN_ID resolution to the shared
-    // helper so all config loaders stay in sync (#403).
-    const entry = resolveRemnicPluginEntry(raw);
+    // Delegate slot → canonical → legacy resolution to the generic helper so
+    // all config loaders stay in sync (#403).
+    const entry = resolveOpenClawRemnicPluginEntry(raw);
     const parsedConfig = parseConfig(
       entry && typeof entry === "object"
         ? ((entry["config"] as Record<string, unknown> | undefined) ?? {})