@relayflows/sdk 2.0.29 → 2.0.31

package/src/helper-reference.ts

@@ -0,0 +1,130 @@
+import { parse, parseExpressionAt } from 'acorn';
+
+/**
+ * Which `f.<namespace>` helpers a flow body actually USES.
+ *
+ * Read from a parse, not from the text. A helper merely NAMED in a string, a
+ * comment, a template quasi or a regex is not used, and declaring a
+ * requirement from one refuses a flow for a mount it never touches:
+ * `f.run("echo f.gitlab")`, an agent prompt naming `f.slack`, or a PR title in
+ * a commit message were each enough to demand a GitLab mount.
+ *
+ * This was first written as a lexer that blanked comments, strings and
+ * regexes. That is the wrong tool: `/` is a regex or a division depending on
+ * whether the preceding token ends an expression, which is not decidable
+ * without parse context, and the two shapes that ambiguity broke —
+ * interpolations (`${await f.gitlab…}`) and a quote inside a regex (`/'/`) —
+ * both HID real helper use, deploying a flow without its mount to fail at
+ * runtime. A parser settles every one of those by construction.
+ *
+ * What a parser still cannot see is indirection: `const p = 'gitlab';
+ * f[p].issues…` is invisible to any static reading, because it needs value
+ * tracking. This scan is therefore a convenience for the literal case, never
+ * the authority — `header.tools[namespace] === true` is the declaration that
+ * is, and it short-circuits this entirely.
+ */
+export function helperNamespacesUsed(body: string, root: string): ReadonlySet<string> {
+  const program = parseFlowBody(body);
+  if (program === null) return textFallback(body, root);
+  const used = new Set<string>();
+  walk(program, (node) => {
+    if (node.type !== 'MemberExpression') return;
+    const object = node.object as AstNode | undefined;
+    if (object?.type !== 'Identifier' || object.name !== root) return;
+    const namespace = memberName(node);
+    if (namespace !== undefined) used.add(namespace);
+  });
+  return used;
+}
+
+/** `f.slack`, `f["slack"]`, `f?.slack` — but not `f[variable]`, which is unknowable. */
+function memberName(node: AstNode): string | undefined {
+  const property = node.property as AstNode | undefined;
+  if (property === undefined) return undefined;
+  if (node.computed !== true) return property.type === 'Identifier' ? property.name : undefined;
+  return property.type === 'Literal' && typeof property.value === 'string' ? property.value : undefined;
+}
+
+/**
+ * Parse whatever `Function.prototype.toString()` produced.
+ *
+ * It can be an arrow, a function expression, an async method shorthand from an
+ * object literal, or a declaration — only some of which are expressions, so
+ * each shape gets a try. Types are already stripped by the time a body reaches
+ * here (Node strips before evaluating, and bundlers transpile), so this is
+ * plain JavaScript.
+ */
+function parseFlowBody(body: string): AstNode | null {
+  const options = { ecmaVersion: 'latest' as const, allowAwaitOutsideFunction: true, allowReturnOutsideFunction: true };
+  // `parseExpressionAt` stops at the end of the first expression and does not
+  // object to what follows, so `async post(f) { … }` parses as the identifier
+  // `async` and reports success having read three characters. Every attempt
+  // must therefore consume the whole source, or a method body would be walked
+  // as its own name and declare no helpers at all.
+  const whole = (source: string, node: AstNode): AstNode => {
+    const end = typeof node.end === 'number' ? node.end : -1;
+    if (end < 0 || source.slice(end).trim() !== '') throw new SyntaxError('unconsumed input');
+    return node;
+  };
+  for (const attempt of [
+    () => whole(body, parseExpressionAt(body, 0, options) as unknown as AstNode),
+    () => whole(`(${body})`, parseExpressionAt(`(${body})`, 0, options) as unknown as AstNode),
+    () => parse(body, options) as unknown as AstNode,
+    // An object-literal method (`async post(f) { … }`) is neither expression
+    // nor statement on its own; it only parses inside an object.
+    () => whole(`({${body}})`, parseExpressionAt(`({${body}})`, 0, options) as unknown as AstNode),
+  ]) {
+    try {
+      return attempt();
+    } catch {
+      continue;
+    }
+  }
+  return null;
+}
+
+/**
+ * Last resort when nothing parses: match the text.
+ *
+ * Deliberately the permissive direction. An unparseable body is a shape this
+ * code does not understand, and under-reporting would deploy a flow without a
+ * mount it needs and fail at the call; over-reporting only asks for a mount
+ * that may go unused. Reaching here at all is a bug worth hearing about.
+ */
+function textFallback(body: string, root: string): ReadonlySet<string> {
+  const used = new Set<string>();
+  const escaped = root.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
+  const pattern = new RegExp(`(?:^|[^\\w$.])${escaped}\\s*(?:\\.\\s*([\\w$]+)|\\[\\s*['"]([^'"]+)['"]\\s*\\])`, 'gu');
+  for (const match of body.matchAll(pattern)) {
+    const namespace = match[1] ?? match[2];
+    if (namespace !== undefined) used.add(namespace);
+  }
+  return used;
+}
+
+type AstNode = {
+  type: string;
+  end?: number;
+  name?: string;
+  value?: unknown;
+  computed?: boolean;
+  [key: string]: unknown;
+};
+
+/** Depth-first over every child node, without pulling in a second package. */
+function walk(node: AstNode, visit: (node: AstNode) => void): void {
+  visit(node);
+  for (const key of Object.keys(node)) {
+    if (key === 'type' || key === 'start' || key === 'end' || key === 'loc') continue;
+    const child = node[key];
+    if (Array.isArray(child)) {
+      for (const entry of child) if (isNode(entry)) walk(entry, visit);
+    } else if (isNode(child)) {
+      walk(child, visit);
+    }
+  }
+}
+
+function isNode(value: unknown): value is AstNode {
+  return typeof value === 'object' && value !== null && typeof (value as { type?: unknown }).type === 'string';
+}

package/src/hosted-base-snapshot.ts

@@ -0,0 +1,351 @@
+import { constants } from 'node:fs';
+import { chmod, mkdir, mkdtemp, realpath, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
+import { canonicalize } from './canonical.js';
+import { sha256 } from './bundle.js';
+import {
+  closeDescriptor,
+  closeDirectory,
+  descriptorIsDirectory,
+  descriptorIsFile,
+  directoryEntryIsDirectory,
+  directoryEntryIsFile,
+  openDirectory,
+  openDescriptor,
+  readDescriptor,
+  readDirectory,
+  readDirectoryEntry,
+  statDescriptor,
+} from './fs-descriptor.js';
+import { findHostedProject } from './hosted-project.js';
+import { appendIntrinsicArray } from './intrinsic-array.js';
+import {
+  assertHostedPromiseSafety,
+  frozenHostedPromiseValue,
+  hostedPromiseValue,
+} from './hosted-promise-safety.js';
+import { PluginError } from './plugin-manifest.js';
+
+const EXCLUDED_DIRECTORIES = new Set(['.flows', '.git', 'node_modules']);
+const CHMOD = chmod;
+const MKDIR = mkdir;
+const MKDTEMP = mkdtemp;
+const REALPATH = realpath;
+const RM = rm;
+const WRITE_FILE = writeFile;
+const TMPDIR = tmpdir;
+const PATH_DIRNAME = dirname;
+const PATH_IS_ABSOLUTE = isAbsolute;
+const PATH_JOIN = join;
+const PATH_RELATIVE = relative;
+const PATH_RESOLVE = resolve;
+const PATH_SEPARATOR = sep;
+const PROCESS_PLATFORM = process.platform;
+const ARRAY_SORT = Function.prototype.call.bind(Array.prototype.sort) as <T>(
+  array: T[],
+  compare?: (left: T, right: T) => number,
+) => T[];
+const OBJECT_CREATE = Object.create;
+const OBJECT_FREEZE = Object.freeze;
+const OBJECT_GET_OWN_PROPERTY_DESCRIPTOR = Object.getOwnPropertyDescriptor;
+const BIG_INT = BigInt;
+const BUFFER_ALLOC_UNSAFE = Buffer.allocUnsafe;
+const NUMBER = Number;
+const SET_HAS = Function.prototype.call.bind(Set.prototype.has) as <T>(set: Set<T>, value: T) => boolean;
+const STRING_LOCALE_COMPARE = Function.prototype.call.bind(String.prototype.localeCompare) as (
+  value: string,
+  other: string,
+) => number;
+const STRING_STARTS_WITH = Function.prototype.call.bind(String.prototype.startsWith) as (
+  value: string,
+  search: string,
+) => boolean;
+const MAX_ENTRIES = 10_000;
+const MAX_BYTES = 64 * 1024 * 1024;
+const MAX_DEPTH = 64;
+const READ_FLAGS = constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK;
+
+interface SourceFile {
+  readonly path: string;
+  readonly bytes: Uint8Array;
+  readonly sha256: string;
+}
+
+interface SourceBudget {
+  entries: number;
+  bytes: number;
+}
+
+/** @internal Deterministic race seams for this security boundary's tests. */
+export interface HostedBaseSnapshotTestHooks {
+  readonly beforeOpen?: (path: string) => Promise<void>;
+  readonly afterStat?: (path: string) => Promise<void>;
+}
+
+const EMPTY_HOSTED_BASE_HOOKS = OBJECT_FREEZE(
+  OBJECT_CREATE(null),
+) as HostedBaseSnapshotTestHooks;
+
+export interface HostedBaseSourceRoot {
+  readonly root: string;
+  readonly prefix: string;
+}
+
+export interface HostedBaseSnapshot {
+  readonly liveSources: readonly HostedBaseSourceRoot[];
+  readonly liveDigest: string;
+  readonly snapshotRoot: string;
+  readonly snapshotFlowPath: string;
+  readonly snapshotFlowSha256: string;
+}
+
+/**
+ * Materialize project source without importing tenant code. Project
+ * node_modules is never linked or read. Every admitted source byte is
+ * represented in liveDigest and copied to the private identity directory.
+ */
+export async function createHostedBaseSnapshot(
+  flowPath: string,
+  projectRootOverride?: string,
+): Promise<HostedBaseSnapshot> {
+  assertHostedPromiseSafety('plugin_source_invalid');
+  const origin = await REALPATH(PATH_RESOLVE(flowPath));
+  const discovered = projectRootOverride ?? findHostedProject(PATH_DIRNAME(origin)) ?? PATH_DIRNAME(origin);
+  const projectRoot = await REALPATH(discovered);
+  const flowRelative = PATH_RELATIVE(projectRoot, origin);
+  if (
+    flowRelative === '' ||
+    PATH_IS_ABSOLUTE(flowRelative) ||
+    flowRelative === '..' ||
+    STRING_STARTS_WITH(flowRelative, `..${PATH_SEPARATOR}`)
+  ) {
+    throw invalid('Hosted base flow must be a file inside its project root.');
+  }
+  const liveSources = frozenHostedPromiseValue([OBJECT_FREEZE({ root: projectRoot, prefix: '' })]);
+  const files = await readAuthorityFiles(liveSources);
+  const liveDigest = sourceDigest(files);
+  const liveFlow = sourceFileAt(files, flowRelative);
+  if (liveFlow === undefined) throw invalid('Hosted base flow is missing from its admitted source.');
+  const snapshotRoot = await MKDTEMP(PATH_JOIN(TMPDIR(), 'flows-hosted-base-'));
+  await CHMOD(snapshotRoot, 0o700);
+  try {
+    for (let index = 0; index < files.length; index += 1) {
+      const file = files[index]!;
+      const target = PATH_JOIN(snapshotRoot, file.path);
+      await MKDIR(PATH_DIRNAME(target), { recursive: true, mode: 0o700 });
+      await WRITE_FILE(target, file.bytes, { flag: 'wx', mode: 0o400 });
+    }
+    const snapshotFiles = await readSnapshotTree(snapshotRoot);
+    const snapshotDigest = sourceDigest(snapshotFiles);
+    if (snapshotDigest !== liveDigest) {
+      throw invalid('Hosted base private snapshot does not match its buffered source.');
+    }
+    const snapshotFlow = sourceFileAt(snapshotFiles, flowRelative);
+    if (snapshotFlow === undefined || snapshotFlow.sha256 !== liveFlow.sha256) {
+      throw invalid('Hosted base private snapshot does not contain its buffered flow source.');
+    }
+    return frozenHostedPromiseValue({
+      liveSources,
+      liveDigest,
+      snapshotRoot,
+      snapshotFlowPath: PATH_JOIN(snapshotRoot, flowRelative),
+      snapshotFlowSha256: snapshotFlow.sha256,
+    });
+  } catch (error) {
+    await RM(snapshotRoot, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+export async function hostedBaseSourceDigest(
+  sources: readonly HostedBaseSourceRoot[],
+  hooks: HostedBaseSnapshotTestHooks = EMPTY_HOSTED_BASE_HOOKS,
+): Promise<string> {
+  assertHostedPromiseSafety('plugin_source_invalid');
+  return sourceDigest(await readAuthorityFiles(sources, hooks));
+}
+
+export async function removeHostedBaseSnapshot(snapshot: HostedBaseSnapshot): Promise<void> {
+  await RM(snapshot.snapshotRoot, { recursive: true, force: true });
+}
+
+async function readAuthorityFiles(
+  sources: readonly HostedBaseSourceRoot[],
+  hooks: HostedBaseSnapshotTestHooks = EMPTY_HOSTED_BASE_HOOKS,
+): Promise<readonly SourceFile[]> {
+  const budget: SourceBudget = { entries: 0, bytes: 0 };
+  const files: SourceFile[] = [];
+  for (let sourceIndex = 0; sourceIndex < sources.length; sourceIndex += 1) {
+    const source = sources[sourceIndex]!;
+    const sourceFiles = await readTree(source.root, source.prefix, budget, hooks);
+    for (let fileIndex = 0; fileIndex < sourceFiles.length; fileIndex += 1) {
+      appendIntrinsicArray(files, sourceFiles[fileIndex]!);
+    }
+  }
+  ARRAY_SORT(files, (left, right) => STRING_LOCALE_COMPARE(left.path, right.path));
+  return frozenHostedPromiseValue(files);
+}
+
+async function readSnapshotTree(root: string): Promise<readonly SourceFile[]> {
+  const files: SourceFile[] = [];
+  const budget: SourceBudget = { entries: 0, bytes: 0 };
+  async function visit(directory: string, prefix: string): Promise<void> {
+    const entries = await readDirectory(directory);
+    ARRAY_SORT(entries, (left, right) => STRING_LOCALE_COMPARE(left.name, right.name));
+    for (let index = 0; index < entries.length; index += 1) {
+      const entry = entries[index]!;
+      budget.entries += 1;
+      if (budget.entries > MAX_ENTRIES) throw tooLarge();
+      const path = prefix === '' ? entry.name : PATH_JOIN(prefix, entry.name);
+      const absolute = PATH_JOIN(directory, entry.name);
+      if (directoryEntryIsDirectory(entry)) await visit(absolute, path);
+      else if (directoryEntryIsFile(entry)) {
+        const descriptor = await openDescriptor(absolute, READ_FLAGS);
+        try {
+          const before = await statDescriptor(descriptor, { bigint: true });
+          if (!descriptorIsFile(before) || before.size < 0n || before.size > BIG_INT(MAX_BYTES - budget.bytes))
+            throw tooLarge();
+          const expectedBytes = NUMBER(before.size);
+          const bytes = await readBounded(descriptor, expectedBytes, path);
+          const after = await statDescriptor(descriptor, { bigint: true });
+          if (after.size !== before.size || after.mtimeNs !== before.mtimeNs || after.ctimeNs !== before.ctimeNs) {
+            throw invalid(`Hosted base snapshot changed while reading "${path}".`);
+          }
+          budget.bytes += expectedBytes;
+          appendIntrinsicArray(files, OBJECT_FREEZE({ path, bytes, sha256: sha256(bytes) }));
+        } finally {
+          await closeDescriptor(descriptor);
+        }
+      } else throw invalid(`Hosted base snapshot contains unsupported entry "${path}".`);
+    }
+  }
+  await visit(root, '');
+  ARRAY_SORT(files, (left, right) => STRING_LOCALE_COMPARE(left.path, right.path));
+  return frozenHostedPromiseValue(files);
+}
+
+async function readTree(
+  root: string,
+  prefix: string,
+  budget: SourceBudget,
+  hooks: HostedBaseSnapshotTestHooks,
+): Promise<readonly SourceFile[]> {
+  if (PROCESS_PLATFORM !== 'linux') {
+    throw invalid('Hosted base source snapshotting requires Linux.');
+  }
+  const files: SourceFile[] = [];
+  async function visit(directory: number, relativeDirectory: string, depth: number): Promise<void> {
+    if (depth > MAX_DEPTH) throw tooLarge();
+    const entries = await openDirectory(`/proc/self/fd/${directory}`);
+    try {
+      for (;;) {
+        const entry = await readDirectoryEntry(entries);
+        if (entry === null) break;
+        if (SET_HAS(EXCLUDED_DIRECTORIES, entry.name)) continue;
+        budget.entries += 1;
+        if (budget.entries > MAX_ENTRIES) throw tooLarge();
+        const relativePath = relativeDirectory === '' ? entry.name : PATH_JOIN(relativeDirectory, entry.name);
+        const absolutePath = PATH_JOIN(root, relativePath);
+        await ownHook(hooks, 'beforeOpen')?.(absolutePath);
+        const descriptor = await openDescriptor(`/proc/self/fd/${directory}/${entry.name}`, READ_FLAGS);
+        try {
+          const before = await statDescriptor(descriptor, { bigint: true });
+          if (descriptorIsDirectory(before)) {
+            await visit(descriptor, relativePath, depth + 1);
+          } else if (descriptorIsFile(before)) {
+            if (before.size < 0n || before.size > BIG_INT(MAX_BYTES - budget.bytes)) throw tooLarge();
+            const expectedBytes = NUMBER(before.size);
+            await ownHook(hooks, 'afterStat')?.(absolutePath);
+            const bytes = await readBounded(descriptor, expectedBytes, relativePath);
+            const after = await statDescriptor(descriptor, { bigint: true });
+            if (after.size !== before.size || after.mtimeNs !== before.mtimeNs || after.ctimeNs !== before.ctimeNs) {
+              throw invalid(`Hosted base source changed while reading "${relativePath}".`);
+            }
+            budget.bytes += expectedBytes;
+            appendIntrinsicArray(
+              files,
+              OBJECT_FREEZE({
+                path: prefix === '' ? relativePath : PATH_JOIN(prefix, relativePath),
+                bytes,
+                sha256: sha256(bytes),
+              }),
+            );
+          } else {
+            throw invalid(`Hosted base authority contains unsupported entry "${relativePath}".`);
+          }
+        } finally {
+          await closeDescriptor(descriptor);
+        }
+      }
+    } finally {
+      await closeDirectory(entries);
+    }
+  }
+  try {
+    const rootDescriptor = await openDescriptor(root, READ_FLAGS);
+    try {
+      if (!descriptorIsDirectory(await statDescriptor(rootDescriptor, { bigint: true }))) {
+        throw invalid('Hosted base source root is not a directory.');
+      }
+      await visit(rootDescriptor, '', 0);
+    } finally {
+      await closeDescriptor(rootDescriptor);
+    }
+  } catch (error) {
+    if (error instanceof PluginError) throw error;
+    throw invalid('Hosted base source or trusted dependency is unreadable.');
+  }
+  return frozenHostedPromiseValue(files);
+}
+
+function ownHook(
+  hooks: HostedBaseSnapshotTestHooks,
+  name: keyof HostedBaseSnapshotTestHooks,
+): ((path: string) => Promise<void>) | undefined {
+  const descriptor = OBJECT_GET_OWN_PROPERTY_DESCRIPTOR(hooks, name);
+  return descriptor !== undefined && 'value' in descriptor && typeof descriptor.value === 'function'
+    ? descriptor.value as (path: string) => Promise<void>
+    : undefined;
+}
+
+async function readBounded(descriptor: number, expectedBytes: number, relativePath: string): Promise<Buffer> {
+  const bytes = BUFFER_ALLOC_UNSAFE(expectedBytes);
+  let offset = 0;
+  while (offset < expectedBytes) {
+    const bytesRead = await readDescriptor(descriptor, bytes, offset, expectedBytes - offset, offset);
+    if (bytesRead === 0) {
+      throw invalid(`Hosted base source changed while reading "${relativePath}".`);
+    }
+    offset += bytesRead;
+  }
+  const extra = BUFFER_ALLOC_UNSAFE(1);
+  if ((await readDescriptor(descriptor, extra, 0, 1, expectedBytes)) !== 0) {
+    throw invalid(`Hosted base source changed while reading "${relativePath}".`);
+  }
+  return hostedPromiseValue(bytes);
+}
+
+function tooLarge(): PluginError {
+  return invalid('Hosted base source exceeds the snapshot entry or byte limit.');
+}
+
+function sourceDigest(files: readonly SourceFile[]): string {
+  const records: Array<{ path: string; sha256: string }> = [];
+  for (let index = 0; index < files.length; index += 1) {
+    const file = files[index]!;
+    appendIntrinsicArray(records, { path: file.path, sha256: file.sha256 });
+  }
+  return sha256(canonicalize(records));
+}
+
+function sourceFileAt(files: readonly SourceFile[], path: string): SourceFile | undefined {
+  for (let index = 0; index < files.length; index += 1) {
+    if (files[index]!.path === path) return files[index];
+  }
+  return undefined;
+}
+
+function invalid(message: string): PluginError {
+  return new PluginError('plugin_source_invalid', message);
+}