@remnic/core 1.1.12 → 1.1.14

package/src/transfer/fs-utils.ts

@@ -0,0 +1,269 @@
+import { createHash } from "node:crypto";
+import {
+  lstat,
+  mkdir,
+  readdir,
+  readFile,
+  realpath,
+  stat,
+  writeFile,
+} from "node:fs/promises";
+import path from "node:path";
+
+const fatalUtf8Decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+
+export async function sha256File(filePath: string): Promise<{ sha256: string; bytes: number }> {
+  const buf = await readFile(filePath);
+  const sha256 = createHash("sha256").update(buf).digest("hex");
+  return { sha256, bytes: buf.byteLength };
+}
+
+export function sha256Bytes(content: Uint8Array): { sha256: string; bytes: number } {
+  const buf = Buffer.from(content);
+  const sha256 = createHash("sha256").update(buf).digest("hex");
+  return { sha256, bytes: buf.byteLength };
+}
+
+export function sha256String(content: string): { sha256: string; bytes: number } {
+  return sha256Bytes(Buffer.from(content, "utf-8"));
+}
+
+export async function readUtf8FileStrict(filePath: string): Promise<{ content: string; sha256: string; bytes: number }> {
+  const buf = await readFile(filePath);
+  let content: string;
+  try {
+    content = fatalUtf8Decoder.decode(buf);
+  } catch {
+    throw new Error(`transfer export requires UTF-8 text files: ${filePath}`);
+  }
+  return { content, ...sha256Bytes(buf) };
+}
+
+export async function writeJsonFile(filePath: string, value: unknown): Promise<void> {
+  await mkdir(path.dirname(filePath), { recursive: true });
+  await writeFile(filePath, JSON.stringify(value, null, 2) + "\n", "utf-8");
+}
+
+export async function readJsonFile<T>(filePath: string): Promise<T> {
+  const raw = await readFile(filePath, "utf-8");
+  return JSON.parse(raw) as T;
+}
+
+export async function listFilesRecursive(rootDir: string): Promise<string[]> {
+  const out: string[] = [];
+  async function walk(dir: string): Promise<void> {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const ent of entries) {
+      const fp = path.join(dir, ent.name);
+      if (ent.isDirectory()) {
+        await walk(fp);
+      } else if (ent.isFile()) {
+        out.push(fp);
+      }
+    }
+  }
+  await walk(rootDir);
+  return out.sort();
+}
+
+export async function ensureDirExists(dirPath: string): Promise<void> {
+  await mkdir(dirPath, { recursive: true });
+}
+
+export async function fileExists(filePath: string): Promise<boolean> {
+  try {
+    await stat(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function toPosixRelPath(absPath: string, rootDir: string): string {
+  const rel = path.relative(rootDir, absPath);
+  // normalize to posix for portability across platforms
+  return rel.split(path.sep).join("/");
+}
+
+export function fromPosixRelPath(relPath: string): string {
+  return relPath.split("/").join(path.sep);
+}
+
+export interface SafeArchiveRoot {
+  abs: string;
+  real: string;
+  errorPrefix: string;
+  argName: string;
+}
+
+export async function prepareSafeArchiveRoot(
+  absPath: string,
+  errorPrefix: string,
+  argName: string,
+): Promise<SafeArchiveRoot> {
+  const rootAbs = path.resolve(absPath);
+  await assertIsDirectoryNotSymlink(rootAbs, errorPrefix, argName);
+  return {
+    abs: rootAbs,
+    real: await realpath(rootAbs),
+    errorPrefix,
+    argName,
+  };
+}
+
+export function validateArchiveRelativePath(
+  relPath: string,
+  errorPrefix: string,
+): string {
+  if (relPath.length === 0) {
+    throw new Error(`${errorPrefix}: record path must not be empty`);
+  }
+  if (relPath.includes("\\")) {
+    throw new Error(
+      `${errorPrefix}: record path must use POSIX separators: ${relPath}`,
+    );
+  }
+  if (path.posix.isAbsolute(relPath)) {
+    throw new Error(
+      `${errorPrefix}: record path must be relative: ${relPath}`,
+    );
+  }
+
+  const segments = relPath.split("/");
+  if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) {
+    throw new Error(
+      `${errorPrefix}: record path contains unsafe segments: ${relPath}`,
+    );
+  }
+
+  const normalized = path.posix.normalize(relPath);
+  if (
+    normalized === "." ||
+    normalized === ".." ||
+    normalized.startsWith("../") ||
+    path.posix.isAbsolute(normalized)
+  ) {
+    throw new Error(
+      `${errorPrefix}: record path escapes target root: ${relPath}`,
+    );
+  }
+
+  return normalized;
+}
+
+export async function resolveSafeArchiveTarget(
+  root: SafeArchiveRoot,
+  relPath: string,
+): Promise<string> {
+  const safeRelPath = validateArchiveRelativePath(relPath, root.errorPrefix);
+  const targetAbs = path.resolve(root.abs, fromPosixRelPath(safeRelPath));
+  if (!isPathInsideRoot(root.abs, targetAbs)) {
+    throw new Error(
+      `${root.errorPrefix}: record path escapes target root: ${relPath}`,
+    );
+  }
+
+  const targetStat = await lstat(targetAbs).catch((error: unknown) => {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw error;
+  });
+  if (targetStat?.isSymbolicLink()) {
+    throw new Error(
+      `${root.errorPrefix}: record path targets a symlink: ${relPath}`,
+    );
+  }
+
+  await assertRealpathInsideRoot(
+    root.real,
+    targetAbs,
+    relPath,
+    root.errorPrefix,
+  );
+  return targetAbs;
+}
+
+// ---------------------------------------------------------------------------
+// Shared path-safety helpers (used by capsule-import, capsule-merge, and
+// capsule-fork).
+//
+// These helpers are security-critical (path-traversal and symlink-bypass
+// guards), so any future fix must apply uniformly. The `errorPrefix` and
+// `argName` arguments let each caller surface module-specific error messages
+// ("importCapsule:" vs "mergeCapsule:" vs "forkCapsule:") without forking the
+// implementation.
+// ---------------------------------------------------------------------------
+
+/**
+ * Return true when {@link absPath} is the same as {@link rootReal} or a
+ * descendant. {@link rootReal} should be the value returned by
+ * `realpath(rootAbs)` so that symlinked subdirectories are detected.
+ */
+export function isPathInsideRoot(rootReal: string, absPath: string): boolean {
+  const rel = path.relative(rootReal, absPath);
+  if (rel === "") return true;
+  if (rel === "..") return false;
+  if (rel.startsWith(`..${path.sep}`)) return false;
+  if (path.isAbsolute(rel)) return false;
+  return true;
+}
+
+/**
+ * Assert that {@link absPath} is an existing directory and is not itself a
+ * symlink. `existsSync` returns true for files (gotcha #24); a stat-based
+ * check is required. Symlinked roots are rejected up-front so an attacker
+ * cannot hand the importer a `~/import-target` link → `/etc` and have writes
+ * silently follow the link (Codex P1 round 5 thread on PR #741).
+ */
+export async function assertIsDirectoryNotSymlink(
+  absPath: string,
+  errorPrefix: string,
+  argName: string,
+): Promise<void> {
+  const st = await stat(absPath).catch(() => null);
+  if (!st || !st.isDirectory()) {
+    throw new Error(
+      `${errorPrefix}: '${argName}' must be an existing directory: ${absPath}`,
+    );
+  }
+  const lst = await lstat(absPath).catch(() => null);
+  if (lst && lst.isSymbolicLink()) {
+    throw new Error(
+      `${errorPrefix}: '${argName}' must not be a symlink — resolve it to its real path first: ${absPath}`,
+    );
+  }
+}
+
+/**
+ * Walk upward from {@link targetAbs} to find the nearest existing ancestor,
+ * resolve it via `fs.realpath` (which follows symlinks), then re-append the
+ * remaining suffix and verify the result is inside {@link rootReal}.
+ *
+ * This catches the case where an existing subdirectory at any point in the
+ * path is a symlink that points outside the intended root. Because the file
+ * does not exist yet we cannot realpath it directly; we resolve the deepest
+ * existing prefix and re-apply the non-existent suffix. Callers must ensure
+ * {@link rootReal} was already resolved via `realpath`.
+ */
+export async function assertRealpathInsideRoot(
+  rootReal: string,
+  targetAbs: string,
+  sourcePath: string,
+  errorPrefix: string,
+): Promise<void> {
+  let existing = targetAbs;
+  const suffix: string[] = [];
+  while (existing !== path.dirname(existing)) {
+    const st = await lstat(existing).catch(() => null);
+    if (st !== null) break;
+    suffix.unshift(path.basename(existing));
+    existing = path.dirname(existing);
+  }
+  const existingReal = await realpath(existing).catch(() => existing);
+  const targetReal =
+    suffix.length > 0 ? path.join(existingReal, ...suffix) : existingReal;
+  if (!isPathInsideRoot(rootReal, targetReal)) {
+    throw new Error(
+      `${errorPrefix}: record path escapes target root via symlink: ${sourcePath}`,
+    );
+  }
+}

package/src/transfer/import-json.ts

@@ -0,0 +1,108 @@
+import path from "node:path";
+import { mkdir, writeFile } from "node:fs/promises";
+import { ExportBundleV1Schema } from "./types.js";
+import {
+  fileExists,
+  prepareSafeArchiveRoot,
+  readJsonFile,
+  resolveSafeArchiveTarget,
+  type SafeArchiveRoot,
+} from "./fs-utils.js";
+import { parseConflictPolicy, type ConflictPolicy } from "./conflict-policy.js";
+import { validateExportBundleRecords } from "./integrity.js";
+
+export type { ConflictPolicy };
+
+export interface ImportJsonOptions {
+  targetMemoryDir: string;
+  fromDir: string;
+  conflict?: ConflictPolicy;
+  dryRun?: boolean;
+  workspaceDir?: string;
+}
+
+function normalizeForDedupe(s: string): string {
+  return s.replace(/\s+/g, " ").trim();
+}
+
+export async function importJsonBundle(opts: ImportJsonOptions): Promise<{ written: number; skipped: number }> {
+  const conflict = parseConflictPolicy(opts.conflict, "importJsonBundle");
+  const fromDirAbs = path.resolve(opts.fromDir);
+  const bundlePath = path.join(fromDirAbs, "bundle.json");
+  const bundle = ExportBundleV1Schema.parse(await readJsonFile(bundlePath));
+  validateExportBundleRecords(bundle.manifest, bundle.records, "importJsonBundle");
+
+  const memDirAbs = path.resolve(opts.targetMemoryDir);
+  const memoryRoot = await prepareSafeArchiveRoot(
+    memDirAbs,
+    "importJsonBundle",
+    "targetMemoryDir",
+  );
+  const written: Array<{ abs: string; content: string }> = [];
+
+  let skipped = 0;
+  let workspaceRoot: SafeArchiveRoot | null = null;
+
+  for (const rec of bundle.records) {
+    const isWorkspace = rec.path.startsWith("workspace/");
+    let absTarget: string;
+    if (isWorkspace) {
+      if (!opts.workspaceDir) {
+        skipped += 1;
+        continue;
+      }
+      workspaceRoot ??= await prepareSafeArchiveRoot(
+        path.resolve(opts.workspaceDir),
+        "importJsonBundle",
+        "workspaceDir",
+      );
+      absTarget = await resolveSafeArchiveTarget(
+        workspaceRoot,
+        rec.path.slice("workspace/".length),
+      );
+    } else {
+      absTarget = await resolveSafeArchiveTarget(memoryRoot, rec.path);
+    }
+
+    const exists = await fileExists(absTarget);
+    if (exists) {
+      if (conflict === "skip") {
+        skipped += 1;
+        continue;
+      }
+      if (conflict === "dedupe") {
+        try {
+          const existing = await (await import("node:fs/promises")).readFile(absTarget, "utf-8");
+          if (normalizeForDedupe(existing) === normalizeForDedupe(rec.content)) {
+            skipped += 1;
+            continue;
+          }
+        } catch {
+          // if can't read, fall through to overwrite
+        }
+      }
+      // overwrite: proceed
+    }
+
+    written.push({ abs: absTarget, content: rec.content });
+  }
+
+  if (opts.dryRun) {
+    return { written: 0, skipped };
+  }
+
+  for (const w of written) {
+    await mkdir(path.dirname(w.abs), { recursive: true });
+    await writeFile(w.abs, w.content, "utf-8");
+  }
+
+  return { written: written.length, skipped };
+}
+
+export function looksLikeEngramJsonExport(fromDir: string): Promise<boolean> {
+  const dir = path.resolve(fromDir);
+  return Promise.all([
+    fileExists(path.join(dir, "manifest.json")),
+    fileExists(path.join(dir, "bundle.json")),
+  ]).then(([m, b]) => m && b);
+}

package/src/transfer/import-md.ts

@@ -0,0 +1,84 @@
+import path from "node:path";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import {
+  fileExists,
+  listFilesRecursive,
+  prepareSafeArchiveRoot,
+  readJsonFile,
+  resolveSafeArchiveTarget,
+  toPosixRelPath,
+} from "./fs-utils.js";
+import { parseConflictPolicy, type ConflictPolicy } from "./conflict-policy.js";
+import { validateManifestRecords } from "./integrity.js";
+import { ExportManifestV1Schema } from "./types.js";
+
+export type { ConflictPolicy };
+
+export interface ImportMdOptions {
+  targetMemoryDir: string;
+  fromDir: string;
+  conflict?: ConflictPolicy;
+  dryRun?: boolean;
+}
+
+export async function importMarkdownBundle(opts: ImportMdOptions): Promise<{ written: number; skipped: number }> {
+  const conflict = parseConflictPolicy(opts.conflict, "importMarkdownBundle");
+  const fromAbs = path.resolve(opts.fromDir);
+  const targetAbs = path.resolve(opts.targetMemoryDir);
+  const targetRoot = await prepareSafeArchiveRoot(
+    targetAbs,
+    "importMarkdownBundle",
+    "targetMemoryDir",
+  );
+  const manifest = ExportManifestV1Schema.parse(
+    await readJsonFile(path.join(fromAbs, "manifest.json")),
+  );
+
+  const filesAbs = await listFilesRecursive(fromAbs);
+  const records: Array<{ path: string; content: Uint8Array }> = [];
+  for (const abs of filesAbs) {
+    const relPosix = toPosixRelPath(abs, fromAbs);
+    if (relPosix === "manifest.json") continue;
+    records.push({ path: relPosix, content: await readFile(abs) });
+  }
+  validateManifestRecords(manifest, records, "importMarkdownBundle");
+
+  const writes: Array<{ abs: string; content: Uint8Array }> = [];
+  let skipped = 0;
+
+  for (const record of records) {
+    const relPosix = record.path;
+    const dstAbs = await resolveSafeArchiveTarget(targetRoot, relPosix);
+    const content = record.content;
+
+    const exists = await fileExists(dstAbs);
+    if (exists) {
+      if (conflict === "skip") {
+        skipped += 1;
+        continue;
+      }
+      if (conflict === "dedupe") {
+        try {
+          const existing = await readFile(dstAbs);
+          if (Buffer.compare(existing, Buffer.from(content)) === 0) {
+            skipped += 1;
+            continue;
+          }
+        } catch {
+          // fall through to overwrite
+        }
+      }
+    }
+
+    writes.push({ abs: dstAbs, content });
+  }
+
+  if (opts.dryRun) return { written: 0, skipped };
+
+  for (const w of writes) {
+    await mkdir(path.dirname(w.abs), { recursive: true });
+    await writeFile(w.abs, w.content);
+  }
+
+  return { written: writes.length, skipped };
+}

package/src/transfer/import-sqlite.ts

@@ -0,0 +1,100 @@
+import path from "node:path";
+import { mkdir, writeFile } from "node:fs/promises";
+import { SQLITE_SCHEMA_VERSION } from "./sqlite-schema.js";
+import {
+  fileExists,
+  prepareSafeArchiveRoot,
+  resolveSafeArchiveTarget,
+} from "./fs-utils.js";
+import { parseConflictPolicy, type ConflictPolicy } from "./conflict-policy.js";
+import { openBetterSqlite3 } from "../runtime/better-sqlite.js";
+import { validateManifestRecords } from "./integrity.js";
+
+export type { ConflictPolicy };
+
+export interface ImportSqliteOptions {
+  targetMemoryDir: string;
+  fromFile: string;
+  conflict?: ConflictPolicy;
+  dryRun?: boolean;
+}
+
+function normalizeForDedupe(s: string): string {
+  return s.replace(/\s+/g, " ").trim();
+}
+
+export async function importSqlite(opts: ImportSqliteOptions): Promise<{ written: number; skipped: number }> {
+  const conflict = parseConflictPolicy(opts.conflict, "importSqlite");
+  const memDirAbs = path.resolve(opts.targetMemoryDir);
+  const memoryRoot = await prepareSafeArchiveRoot(
+    memDirAbs,
+    "importSqlite",
+    "targetMemoryDir",
+  );
+  const fromAbs = path.resolve(opts.fromFile);
+  const db = openBetterSqlite3(fromAbs, { readonly: true });
+
+  const written: Array<{ abs: string; content: string }> = [];
+  let skipped = 0;
+
+  try {
+    const metaRows = db.prepare("SELECT key,value FROM meta").all() as Array<{ key: string; value: string }>;
+    const meta = Object.fromEntries(metaRows.map((r) => [r.key, r.value]));
+    if (String(meta.schemaVersion) !== String(SQLITE_SCHEMA_VERSION)) {
+      throw new Error(`unsupported sqlite schemaVersion: ${meta.schemaVersion}`);
+    }
+
+    const rows = db.prepare("SELECT path_rel, bytes, sha256, content FROM files").all() as Array<{
+      path_rel: string;
+      bytes: number;
+      sha256: string;
+      content: string;
+    }>;
+    validateManifestRecords(
+      {
+        files: rows.map((row) => ({
+          path: row.path_rel,
+          bytes: row.bytes,
+          sha256: row.sha256,
+        })),
+      },
+      rows.map((row) => ({ path: row.path_rel, content: row.content })),
+      "importSqlite",
+    );
+
+    for (const r of rows) {
+      const absTarget = await resolveSafeArchiveTarget(memoryRoot, r.path_rel);
+
+      const exists = await fileExists(absTarget);
+      if (exists) {
+        if (conflict === "skip") {
+          skipped += 1;
+          continue;
+        }
+        if (conflict === "dedupe") {
+          try {
+            const existing = await (await import("node:fs/promises")).readFile(absTarget, "utf-8");
+            if (normalizeForDedupe(existing) === normalizeForDedupe(r.content)) {
+              skipped += 1;
+              continue;
+            }
+          } catch {
+            // fall through
+          }
+        }
+      }
+      written.push({ abs: absTarget, content: r.content });
+    }
+  } finally {
+    db.close();
+  }
+
+  if (opts.dryRun) return { written: 0, skipped };
+
+  for (const w of written) {
+    await mkdir(path.dirname(w.abs), { recursive: true });
+    await writeFile(w.abs, w.content, "utf-8");
+  }
+
+  return { written: written.length, skipped };
+}

package/src/transfer/integrity.ts

@@ -0,0 +1,71 @@
+import type { ExportManifestV1, ExportMemoryRecordV1 } from "./types.js";
+import { sha256Bytes, sha256String } from "./fs-utils.js";
+
+export interface TransferIntegrityRecord {
+  path: string;
+  content: string | Uint8Array;
+}
+
+export function validateManifestRecords(
+  manifest: Pick<ExportManifestV1, "files">,
+  records: readonly TransferIntegrityRecord[],
+  errorPrefix: string,
+): void {
+  const manifestByPath = new Map<string, ExportManifestV1["files"][number]>();
+  for (const file of manifest.files) {
+    if (manifestByPath.has(file.path)) {
+      throw new Error(`${errorPrefix}: duplicate manifest path: ${file.path}`);
+    }
+    manifestByPath.set(file.path, file);
+  }
+
+  const recordByPath = new Map<string, TransferIntegrityRecord>();
+  for (const record of records) {
+    if (recordByPath.has(record.path)) {
+      throw new Error(`${errorPrefix}: duplicate record path: ${record.path}`);
+    }
+    recordByPath.set(record.path, record);
+  }
+
+  for (const record of records) {
+    const expected = manifestByPath.get(record.path);
+    if (!expected) {
+      throw new Error(`${errorPrefix}: record missing from manifest: ${record.path}`);
+    }
+    assertContentIntegrity(record, expected, errorPrefix);
+  }
+
+  for (const file of manifest.files) {
+    if (!recordByPath.has(file.path)) {
+      throw new Error(`${errorPrefix}: manifest file missing from records: ${file.path}`);
+    }
+  }
+}
+
+export function validateExportBundleRecords(
+  manifest: Pick<ExportManifestV1, "files">,
+  records: readonly ExportMemoryRecordV1[],
+  errorPrefix: string,
+): void {
+  validateManifestRecords(manifest, records, errorPrefix);
+}
+
+function assertContentIntegrity(
+  record: TransferIntegrityRecord,
+  expected: ExportManifestV1["files"][number],
+  errorPrefix: string,
+): void {
+  const actual = typeof record.content === "string"
+    ? sha256String(record.content)
+    : sha256Bytes(record.content);
+  if (actual.bytes !== expected.bytes) {
+    throw new Error(
+      `${errorPrefix}: byte count mismatch for ${record.path}`,
+    );
+  }
+  if (actual.sha256 !== expected.sha256) {
+    throw new Error(
+      `${errorPrefix}: checksum mismatch for ${record.path}`,
+    );
+  }
+}

package/src/transfer/sqlite-schema.ts

@@ -0,0 +1,16 @@
+export const SQLITE_SCHEMA_VERSION = 1 as const;
+
+export const SQLITE_TABLES_SQL = `
+CREATE TABLE IF NOT EXISTS meta (
+  key TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS files (
+  path_rel TEXT PRIMARY KEY,
+  bytes INTEGER NOT NULL,
+  sha256 TEXT NOT NULL,
+  content TEXT NOT NULL
+);
+`;
+