@remnic/core 1.1.31 → 9.3.516

package/src/memory-extension-host/host-discovery.ts

@@ -16,6 +16,13 @@ import type { DiscoveredExtension, ExtensionSchema } from "./types.js";
 /** Total token budget for all discovered extension instructions combined. */
 export const REMNIC_EXTENSIONS_TOTAL_TOKEN_LIMIT = 5_000;
 
+/** Per-extension instructions read cap, using the same 4 chars/token budget heuristic. */
+export const REMNIC_EXTENSION_INSTRUCTIONS_BYTE_LIMIT =
+  REMNIC_EXTENSIONS_TOTAL_TOKEN_LIMIT * 4;
+
+/** Optional schemas are metadata only; refuse unusually large schema files. */
+export const REMNIC_EXTENSION_SCHEMA_BYTE_LIMIT = 64 * 1024;
+
 /** Maximum number of example files collected per extension. */
 const MAX_EXAMPLES_PER_EXTENSION = 10;
 
@@ -131,11 +138,15 @@ export async function discoverMemoryExtensions(
     }
     let instructions: string;
     try {
-      instructions = await readFile(instructionsPath, "utf-8");
-    } catch {
-      log.warn?.(
-        `[memory-extensions] skipping "${entry}": missing instructions.md`,
+      instructions = await readUtf8FileWithinLimit(
+        instructionsPath,
+        REMNIC_EXTENSION_INSTRUCTIONS_BYTE_LIMIT,
       );
+    } catch (err) {
+      const reason = err instanceof FileTooLargeError
+        ? `instructions.md exceeds ${REMNIC_EXTENSION_INSTRUCTIONS_BYTE_LIMIT} bytes`
+        : "missing instructions.md";
+      log.warn?.(`[memory-extensions] skipping "${entry}": ${reason}`);
       continue;
     }
 
@@ -148,7 +159,10 @@ export async function discoverMemoryExtensions(
       );
     } else {
       try {
-        const schemaRaw = await readFile(schemaPath, "utf-8");
+        const schemaRaw = await readUtf8FileWithinLimit(
+          schemaPath,
+          REMNIC_EXTENSION_SCHEMA_BYTE_LIMIT,
+        );
         const parsed = JSON.parse(schemaRaw);
         if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) {
           schema = validateSchema(parsed);
@@ -161,6 +175,10 @@ export async function discoverMemoryExtensions(
         // File doesn't exist → fine, no warning needed
         if (isFileNotFoundError(err)) {
           // schema remains undefined
+        } else if (err instanceof FileTooLargeError) {
+          log.warn?.(
+            `[memory-extensions] "${entry}": schema.json exceeds ${REMNIC_EXTENSION_SCHEMA_BYTE_LIMIT} bytes, ignoring schema`,
+          );
         } else {
           log.warn?.(
             `[memory-extensions] "${entry}": malformed schema.json, ignoring schema`,
@@ -174,13 +192,34 @@ export async function discoverMemoryExtensions(
     const examplesPaths: string[] = [];
     const examplesDir = path.join(entryPath, "examples");
     try {
+      const examplesStat = await lstat(examplesDir);
+      if (examplesStat.isSymbolicLink()) {
+        log.warn?.(
+          `[memory-extensions] "${entry}": examples/ is a symlink, ignoring examples`,
+        );
+        throw new Error("skip symlinked examples directory");
+      }
+      if (!examplesStat.isDirectory()) {
+        throw new Error("skip non-directory examples path");
+      }
       const exampleEntries = await readdir(examplesDir);
       const mdFiles = exampleEntries
         .filter((f) => f.endsWith(".md"))
         .sort()
         .slice(0, MAX_EXAMPLES_PER_EXTENSION);
       for (const f of mdFiles) {
-        examplesPaths.push(path.join(examplesDir, f));
+        const examplePath = path.join(examplesDir, f);
+        const exampleStat = await lstat(examplePath);
+        if (exampleStat.isSymbolicLink()) {
+          log.warn?.(
+            `[memory-extensions] "${entry}": examples/${f} is a symlink, ignoring example`,
+          );
+          continue;
+        }
+        if (!exampleStat.isFile()) {
+          continue;
+        }
+        examplesPaths.push(examplePath);
       }
     } catch {
       // No examples dir — fine
@@ -206,6 +245,24 @@ export async function discoverMemoryExtensions(
   return extensions;
 }
 
+class FileTooLargeError extends Error {
+  constructor(filePath: string, limitBytes: number, actualBytes: number) {
+    super(`${filePath} is ${actualBytes} bytes; limit is ${limitBytes} bytes`);
+    this.name = "FileTooLargeError";
+  }
+}
+
+async function readUtf8FileWithinLimit(filePath: string, limitBytes: number): Promise<string> {
+  const stat = await lstat(filePath);
+  if (!stat.isFile()) {
+    throw new Error(`${filePath} is not a file`);
+  }
+  if (stat.size > limitBytes) {
+    throw new FileTooLargeError(filePath, limitBytes, stat.size);
+  }
+  return readFile(filePath, "utf-8");
+}
+
 function validateSchema(raw: Record<string, unknown>): ExtensionSchema {
   const memoryTypes: ExtensionSchema["memoryTypes"] = (() => {
     if (!Array.isArray(raw.memoryTypes)) return undefined;

package/src/memory-projection-store.ts

@@ -350,6 +350,70 @@ function parseJsonObject<T>(value: unknown): T | undefined {
   }
 }
 
+function resolveMemoryDirRoot(memoryDir: string): string {
+  try {
+    return fs.realpathSync(memoryDir);
+  } catch {
+    return path.resolve(memoryDir);
+  }
+}
+
+function isPathInsideRoot(candidatePath: string, rootPath: string): boolean {
+  const relative = path.relative(rootPath, candidatePath);
+  return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
+}
+
+function resolveProjectedMemoryPath(memoryDir: string, pathRel: string): string | null {
+  if (path.isAbsolute(pathRel)) return null;
+  const root = resolveMemoryDirRoot(memoryDir);
+  const candidate = path.resolve(root, pathRel);
+  if (!isPathInsideRoot(candidate, root)) return null;
+  try {
+    const realCandidate = fs.realpathSync(candidate);
+    if (!isPathInsideRoot(realCandidate, root)) return null;
+  } catch {
+    // Missing files are handled by callers. Still keep the lexical guard above.
+  }
+  return candidate;
+}
+
+function projectedBrowseRowFromCurrentRow(
+  memoryDir: string,
+  row: Record<string, unknown>,
+): ProjectedMemoryBrowseRow | null {
+  if (
+    typeof row.memory_id !== "string" ||
+    typeof row.path_rel !== "string" ||
+    typeof row.category !== "string" ||
+    typeof row.status !== "string"
+  ) {
+    return null;
+  }
+  const filePath = resolveProjectedMemoryPath(memoryDir, row.path_rel);
+  if (!filePath) return null;
+  return {
+    id: row.memory_id,
+    path: filePath,
+    category: row.category as MemoryCategory,
+    status: row.status as MemoryStatus,
+    created: typeof row.created_at === "string" ? row.created_at : undefined,
+    updated: typeof row.updated_at === "string" ? row.updated_at : undefined,
+    tags: parseStringArray(row.tags_json),
+    entityRef: typeof row.entity_ref === "string" ? row.entity_ref : undefined,
+    preview: typeof row.preview_text === "string" ? row.preview_text : "",
+  };
+}
+
+function projectedBrowsePathSqlClauses(): string[] {
+  return [
+    "path_rel <> ''",
+    "path_rel NOT LIKE '/%'",
+    "path_rel <> '..'",
+    "path_rel NOT LIKE '../%'",
+    "path_rel NOT LIKE '%/../%'",
+  ];
+}
+
 export function parseCurrentRow(
   memoryDir: string,
   row: Record<string, unknown> | undefined,
@@ -368,6 +432,8 @@ export function parseCurrentRow(
   ) {
     return null;
   }
+  const filePath = resolveProjectedMemoryPath(memoryDir, row.path_rel);
+  if (!filePath) return null;
 
   return {
     memoryId: row.memory_id,
@@ -377,7 +443,7 @@ export function parseCurrentRow(
       typeof row.lifecycle_state === "string"
         ? (row.lifecycle_state as MemoryProjectionCurrentState["lifecycleState"])
         : undefined,
-    path: path.join(memoryDir, row.path_rel),
+    path: filePath,
     pathRel: row.path_rel,
     created: row.created_at,
     updated: row.updated_at,
@@ -568,6 +634,8 @@ export function readProjectedMemoryBrowse(
 
       const filtered = allRows.filter((row) => {
         if (typeof row.memory_id !== "string" || typeof row.path_rel !== "string") return false;
+        const filePath = resolveProjectedMemoryPath(memoryDir, row.path_rel);
+        if (!filePath) return false;
         // Check preview, category, entity_ref, tags first (fast)
         const preview = typeof row.preview_text === "string" ? row.preview_text.toLowerCase() : "";
         const category = typeof row.category === "string" ? row.category.toLowerCase() : "";
@@ -579,7 +647,6 @@ export function readProjectedMemoryBrowse(
         }
         // Fall back to reading full file content from disk
         try {
-          const filePath = path.join(memoryDir, row.path_rel as string);
           const content = fs.readFileSync(filePath, "utf-8").toLowerCase();
           return content.includes(normalizedQuery);
         } catch {
@@ -591,71 +658,56 @@ export function readProjectedMemoryBrowse(
       return {
         total: filtered.length,
         memories: pageRows
-          .filter(
-            (row) =>
-              typeof row.memory_id === "string" &&
-              typeof row.path_rel === "string" &&
-              typeof row.category === "string" &&
-              typeof row.status === "string",
-          )
-          .map((row) => ({
-            id: row.memory_id as string,
-            path: path.join(memoryDir, row.path_rel as string),
-            category: row.category as MemoryCategory,
-            status: row.status as MemoryStatus,
-            created: typeof row.created_at === "string" ? row.created_at : undefined,
-            updated: typeof row.updated_at === "string" ? row.updated_at : undefined,
-            tags: parseStringArray(row.tags_json),
-            entityRef: typeof row.entity_ref === "string" ? row.entity_ref : undefined,
-            preview: typeof row.preview_text === "string" ? row.preview_text : "",
-          })),
+          .map((row) => projectedBrowseRowFromCurrentRow(memoryDir, row))
+          .filter((row): row is ProjectedMemoryBrowseRow => row !== null),
       };
     }
 
-    // No query: use SQL pagination directly
-    const totalRow = db
-      .prepare(`SELECT COUNT(*) AS total FROM memory_current ${whereSql}`)
-      .get(...params) as { total?: number } | undefined;
-    const rows = db
-      .prepare(`
-        SELECT
-          memory_id,
-          path_rel,
-          category,
-          status,
-          created_at,
-          updated_at,
-          entity_ref,
-          ${currentSelect.tagsJson},
-          ${currentSelect.previewText}
-        FROM memory_current
-        ${whereSql}
-        ORDER BY ${orderBySql}
-        LIMIT ? OFFSET ?
-      `)
-      .all(...params, options.limit, options.offset) as Array<Record<string, unknown>>;
+    // No query: push lexical path safety into SQL, then count through the same
+    // realpath-aware row parser used for returned rows so symlink escapes do not
+    // inflate totals.
+    const browseWhereClauses = [...whereClauses, ...projectedBrowsePathSqlClauses()];
+    const browseWhereSql = `WHERE ${browseWhereClauses.join(" AND ")}`;
+    const pageRows: ProjectedMemoryBrowseRow[] = [];
+    const fetchSize = Math.max(options.limit * 2, 50);
+    let validRowsSeen = 0;
+    let scanOffset = 0;
+
+    while (true) {
+      const rows = db
+        .prepare(`
+          SELECT
+            memory_id,
+            path_rel,
+            category,
+            status,
+            created_at,
+            updated_at,
+            entity_ref,
+            ${currentSelect.tagsJson},
+            ${currentSelect.previewText}
+          FROM memory_current
+          ${browseWhereSql}
+          ORDER BY ${orderBySql}
+          LIMIT ? OFFSET ?
+        `)
+        .all(...params, fetchSize, scanOffset) as Array<Record<string, unknown>>;
+      if (rows.length === 0) break;
+      scanOffset += rows.length;
+      for (const row of rows) {
+        const browseRow = projectedBrowseRowFromCurrentRow(memoryDir, row);
+        if (!browseRow) continue;
+        if (validRowsSeen >= options.offset && pageRows.length < options.limit) {
+          pageRows.push(browseRow);
+        }
+        validRowsSeen += 1;
+      }
+      if (rows.length < fetchSize) break;
+    }
 
     return {
-      total: typeof totalRow?.total === "number" ? totalRow.total : 0,
-      memories: rows
-        .filter(
-          (row) =>
-            typeof row.memory_id === "string" &&
-            typeof row.path_rel === "string" &&
-            typeof row.category === "string" &&
-            typeof row.status === "string",
-        )
-        .map((row) => ({
-          id: row.memory_id as string,
-          path: path.join(memoryDir, row.path_rel as string),
-          category: row.category as MemoryCategory,
-          status: row.status as MemoryStatus,
-          created: typeof row.created_at === "string" ? row.created_at : undefined,
-          updated: typeof row.updated_at === "string" ? row.updated_at : undefined,
-          tags: parseStringArray(row.tags_json),
-          entityRef: typeof row.entity_ref === "string" ? row.entity_ref : undefined,
-          preview: typeof row.preview_text === "string" ? row.preview_text : "",
-        })),
+      total: validRowsSeen,
+      memories: pageRows,
     };
   });
 }

package/src/message-parts/index.ts

@@ -245,6 +245,7 @@ export function parseOpenClawMessageParts(
 ): LcmMessagePartInput[] {
   const explicit = normalizeExplicitParts(input);
   if (explicit.length > 0) return explicit;
+  if (Array.isArray(input)) return parseOpenClawContentArray(input, options);
   if (!input || typeof input !== "object") return [];
   const obj = input as Record<string, unknown>;
 
@@ -281,46 +282,53 @@ export function parsePiMessageParts(
 ): LcmMessagePartInput[] {
   const explicit = normalizeExplicitParts(input);
   if (explicit.length > 0) return explicit;
-  if (!isRecord(input)) return renderedFallbackParts(options);
+  const inputRecord = isRecord(input) ? input : null;
+  if (!inputRecord && !Array.isArray(input)) return renderedFallbackParts(options);
 
-  if (input.role === "bashExecution") {
-    const command = asNonEmptyString(input.command);
-    const output = asNonEmptyString(input.output);
+  if (inputRecord?.role === "bashExecution") {
+    const command = asNonEmptyString(inputRecord.command);
+    const output = asNonEmptyString(inputRecord.output);
     const rendered = [command ? `Ran ${command}` : null, output].filter(Boolean).join("\n");
     return withOrdinals([
       makePart("tool_result", {
         role: "bashExecution",
         command,
         output,
-        exitCode: input.exitCode ?? input.exit_code,
+        exitCode: inputRecord.exitCode ?? inputRecord.exit_code,
       }, {
         toolName: "bashExecution",
-        filePath: firstFilePathFromObject(input) ?? firstFilePath(rendered),
+        filePath: firstFilePathFromObject(inputRecord) ?? firstFilePath(rendered),
       }),
     ]);
   }
 
-  const topLevelType = asNonEmptyString(input.type ?? input.kind ?? input.role);
+  const topLevelType = inputRecord
+    ? asNonEmptyString(inputRecord.type ?? inputRecord.kind ?? inputRecord.role)
+    : null;
   if (topLevelType === "toolResult" || topLevelType === "tool_result") {
-    const toolName = asNonEmptyString(input.name ?? input.toolName ?? input.tool_name);
-    const output = input.output ?? input.result ?? input.content;
-    const rendered = renderUnknownContent(output ?? input);
+    const toolName = asNonEmptyString(inputRecord?.name ?? inputRecord?.toolName ?? inputRecord?.tool_name);
+    const output = inputRecord?.output ?? inputRecord?.result ?? inputRecord?.content;
+    const rendered = renderUnknownContent(output ?? inputRecord);
     return withOrdinals([
       makePart("tool_result", {
-        id: input.id ?? input.toolCallId ?? input.tool_call_id,
+        id: inputRecord?.id ?? inputRecord?.toolCallId ?? inputRecord?.tool_call_id,
         name: toolName,
         output: sanitizePayload(output),
-        ...(typeof input.isError === "boolean" ? { isError: input.isError } : {}),
-        ...(typeof input.is_error === "boolean" ? { is_error: input.is_error } : {}),
+        ...(typeof inputRecord?.isError === "boolean" ? { isError: inputRecord.isError } : {}),
+        ...(typeof inputRecord?.is_error === "boolean" ? { is_error: inputRecord.is_error } : {}),
       }, {
         toolName,
-        filePath: firstFilePathFromObject(input) ?? firstFilePath(rendered),
+        filePath: firstFilePathFromObject(inputRecord) ?? firstFilePath(rendered),
       }),
     ]);
   }
 
-  const content = input.content;
-  const blocks = Array.isArray(content) ? content.filter(isRecord) : [];
+  const content = inputRecord?.content;
+  const blocks = Array.isArray(input)
+    ? input.filter(isRecord)
+    : Array.isArray(content)
+      ? content.filter(isRecord)
+      : [];
   const parts: LcmMessagePartInput[] = [];
 
   if (typeof content === "string") {
@@ -362,12 +370,12 @@ export function parsePiMessageParts(
     }
   });
 
-  const toolName = asNonEmptyString(input.toolName ?? input.tool_name ?? input.name);
+  const toolName = asNonEmptyString(inputRecord?.toolName ?? inputRecord?.tool_name ?? inputRecord?.name);
   if (parts.length === 0 && toolName) {
     parts.push(classifyToolPart(toolName, {
       name: toolName,
-      arguments: sanitizePayload(input.arguments ?? input.args ?? input.input ?? input.params),
-      output: sanitizePayload(input.output ?? input.result),
+      arguments: sanitizePayload(inputRecord?.arguments ?? inputRecord?.args ?? inputRecord?.input ?? inputRecord?.params),
+      output: sanitizePayload(inputRecord?.output ?? inputRecord?.result),
     }));
   }
 
@@ -398,24 +406,31 @@ function inferSourceFormat(input: unknown): MessagePartSourceFormat | undefined
     }
     if (Array.isArray(obj.output)) return "openai";
     if (isOpenAiResponseItem(obj)) return "openai";
-    if (
-      Array.isArray(obj.content) &&
-      obj.content.some(isOpenAiContentBlock) &&
-      obj.content.some(isPiToolContentBlock)
-    ) {
-      return "openclaw";
+    if (Array.isArray(obj.content)) {
+      const hasOpenAiBlocks = obj.content.some(isOpenAiContentBlock);
+      const hasAnthropicBlocks = obj.content.some(isAnthropicContentBlock);
+      const hasPiToolBlocks = obj.content.some(isPiToolContentBlock);
+      if (hasOpenAiBlocks && (hasAnthropicBlocks || hasPiToolBlocks)) {
+        return "openclaw";
+      }
+      if (hasOpenAiBlocks) return "openai";
     }
-    if (Array.isArray(obj.content) && obj.content.some(isOpenAiContentBlock)) return "openai";
     if (isAnthropicMessageObject(obj)) return "anthropic";
     if (isPiMessageObject(obj)) return "pi";
     if (Array.isArray(obj.content)) return "anthropic";
   }
   if (Array.isArray(input)) {
-    return input.some((item) =>
+    const records = input.filter(isRecord);
+    const hasOpenAiBlocks = records.some((item) =>
       isRecord(item) && (isOpenAiResponseItem(item) || isOpenAiContentBlock(item))
-    )
-      ? "openai"
-      : "anthropic";
+    );
+    const hasAnthropicBlocks = records.some(isAnthropicContentBlock);
+    const hasPiToolBlocks = records.some(isPiToolContentBlock);
+    if (hasOpenAiBlocks && (hasAnthropicBlocks || hasPiToolBlocks)) return "openclaw";
+    if (hasOpenAiBlocks) return "openai";
+    if (hasAnthropicBlocks) return "anthropic";
+    if (hasPiToolBlocks) return "pi";
+    return "anthropic";
   }
   return undefined;
 }
@@ -640,7 +655,7 @@ function renderUnknownContent(value: unknown): string {
 
 function firstFilePathFromObject(value: unknown): string | null {
   if (!isRecord(value)) return null;
-  const keys = ["file_path", "filePath", "path", "filename", "cwd"];
+  const keys = ["file_path", "filePath", "path", "filename"];
   for (const key of keys) {
     const candidate = asNonEmptyString(value[key]);
     if (candidate) return candidate;

package/src/message-parts/message-parts.test.ts

@@ -227,6 +227,27 @@ describe("message-parts parsers", () => {
     assert.equal(parts[1]!.filePath, "src/openclaw.ts");
   });
 
+  it("infers mixed OpenAI text and Anthropic tool blocks as OpenClaw", () => {
+    const parts = parseMessageParts({
+      content: [
+        { type: "output_text", text: "Updated src/mixed-provider.ts." },
+        {
+          type: "tool_use",
+          id: "toolu_mixed",
+          name: "Edit",
+          input: { path: "src/mixed-provider.ts", old_string: "a", new_string: "b" },
+        },
+      ],
+    });
+
+    assert.equal(parts.length, 2);
+    assert.equal(parts[0]!.kind, "text");
+    assert.equal(parts[0]!.filePath, "src/mixed-provider.ts");
+    assert.equal(parts[1]!.kind, "file_write");
+    assert.equal(parts[1]!.toolName, "Edit");
+    assert.equal(parts[1]!.filePath, "src/mixed-provider.ts");
+  });
+
   it("extracts Pi tool-call content blocks as structured file writes", () => {
     const parts = parsePiMessageParts({
       role: "assistant",
@@ -244,6 +265,49 @@ describe("message-parts parsers", () => {
     assert.equal(parts[1]!.filePath, "packages/plugin-pi/src/index.ts");
   });
 
+  it("extracts top-level Pi tool-call arrays without rendered fallback", () => {
+    const parts = parseMessageParts(
+      [{ type: "toolCall", name: "edit", arguments: { path: "src/a.ts" } }],
+      { sourceFormat: "pi" },
+    );
+
+    assert.equal(parts.length, 1);
+    assert.equal(parts[0]!.kind, "file_write");
+    assert.equal(parts[0]!.toolName, "edit");
+    assert.equal(parts[0]!.filePath, "src/a.ts");
+  });
+
+  it("infers top-level Pi mixed arrays without dropping tool-call blocks", () => {
+    const parts = parseMessageParts([
+      { type: "text", text: "Updated src/a.ts" },
+      { type: "toolCall", name: "edit", arguments: { path: "src/a.ts" } },
+    ]);
+
+    assert.equal(parts.length, 2);
+    assert.equal(parts[0]!.kind, "text");
+    assert.equal(parts[0]!.filePath, "src/a.ts");
+    assert.equal(parts[1]!.kind, "file_write");
+    assert.equal(parts[1]!.toolName, "edit");
+    assert.equal(parts[1]!.filePath, "src/a.ts");
+  });
+
+  it("extracts top-level OpenClaw arrays through mixed block parsing", () => {
+    const parts = parseMessageParts(
+      [
+        { type: "output_text", text: "Updated src/openclaw.ts" },
+        { type: "toolCall", name: "edit", arguments: { path: "src/openclaw.ts" } },
+      ],
+      { sourceFormat: "openclaw" },
+    );
+
+    assert.equal(parts.length, 2);
+    assert.equal(parts[0]!.kind, "text");
+    assert.equal(parts[0]!.filePath, "src/openclaw.ts");
+    assert.equal(parts[1]!.kind, "file_write");
+    assert.equal(parts[1]!.toolName, "edit");
+    assert.equal(parts[1]!.filePath, "src/openclaw.ts");
+  });
+
   it("infers Pi source format for Pi-shaped raw content", () => {
     const parts = parseMessageParts({
       role: "assistant",
@@ -278,6 +342,19 @@ describe("message-parts parsers", () => {
     });
   });
 
+  it("prefers Pi tool-result content paths over cwd metadata", () => {
+    const parts = parseMessageParts({
+      role: "toolResult",
+      toolName: "read",
+      cwd: "/repo",
+      content: "Read failed for src/config.ts",
+    });
+
+    assert.equal(parts.length, 1);
+    assert.equal(parts[0]!.kind, "tool_result");
+    assert.equal(parts[0]!.filePath, "src/config.ts");
+  });
+
   it("uses rendered fallback inside the Pi parser when structured parsing finds no parts", () => {
     const parts = parsePiMessageParts(
       { role: "assistant", content: [{ type: "unknown", value: "ignored" }] },