@venn-lang/runtime 0.1.0

package/src/run/resolve-imports.ts

@@ -0,0 +1,166 @@
+import {
+  type Document,
+  type FragmentDecl,
+  type ImportedDeco,
+  isDecoDecl,
+  isFragmentDecl,
+  isPackageSpecifier,
+  isUseDecl,
+  isValueImport,
+  parse,
+} from "@venn-lang/core";
+
+/** Read source by URI and resolve a specifier relative to a base URI. */
+export interface ModuleIo {
+  read: (uri: string) => Promise<string>;
+  resolve: (base: string, spec: string) => string;
+}
+
+/**
+ * Loading a package that was installed rather than written here.
+ *
+ * Kept apart from {@link ModuleIo} because the two answer different questions:
+ * one reads `.vn` source to be parsed, the other hands back a module that is
+ * already a value. A host without one simply has no packages, which is the state
+ * of every run in a Web Worker and not an error.
+ */
+export interface NpmModules {
+  load(spec: string): Promise<Record<string, unknown> | undefined>;
+}
+
+/** What a run needs from the files it reaches: their fragments, decos and plugins. */
+export interface ResolvedImports {
+  fragments: Map<string, FragmentDecl>;
+  /** The `pub deco`s reachable through the import graph, with the file each came from. */
+  decos: Map<string, ImportedDeco>;
+  /** Every package `use`d anywhere in the graph, so only those need loading. */
+  packages: Set<string>;
+  /**
+   * What each npm specifier loaded to, by the name that was written.
+   *
+   * Loaded here, before anything runs, because binding a scope cannot wait: by
+   * the time a name is looked up the module has to already be a value.
+   */
+  npm: Map<string, Record<string, unknown>>;
+  /**
+   * Every module reached, parsed, keyed by its resolved URI.
+   *
+   * Carried whole rather than reduced to a list of exported names, because a
+   * `pub fn` is a closure over the file it was written in: it calls that file's
+   * private helpers and reads that file's globals. Handing over the function
+   * without the place it came from produces something that resolves and then
+   * fails on its first line.
+   */
+  modules: Map<string, Document>;
+}
+
+/**
+ * Walk the import graph from one document, parsing every `.vn` file it reaches
+ * and loading every package it names. A file already seen is skipped, so a cycle
+ * ends rather than loops. A file that cannot be read is skipped in silence:
+ * whoever tried to read it reports the failure.
+ *
+ * @param args The entry document, its URI, the source reader and the optional
+ * package loader.
+ * @returns The exported fragments and decos, the packages, the loaded npm
+ * modules and every parsed document, keyed by resolved URI.
+ */
+export async function resolveImports(args: {
+  document: Document;
+  uri: string;
+  io: ModuleIo;
+  /** Absent when the host has no way to load one: a Worker, most tests. */
+  npm?: NpmModules;
+}): Promise<ResolvedImports> {
+  const found: Exports = { fragments: new Map(), decos: new Map() };
+  const packages = new Set<string>();
+  const modules = new Map<string, Document>();
+  const npm = new Map<string, Record<string, unknown>>();
+  collectUses(args.document, packages);
+  await loadInto({
+    document: args.document,
+    uri: args.uri,
+    io: args.io,
+    npmLoader: args.npm,
+    found,
+    packages,
+    modules,
+    npm,
+    seen: new Set([args.uri]),
+  });
+  return { ...found, packages, modules, npm };
+}
+
+/**
+ * Add the packages a file declares with `use` to `into`. Collected across the
+ * whole graph: a fragment imported from elsewhere calls the verbs *its* file
+ * asked for.
+ *
+ * @param document The file to read `use` declarations from.
+ * @param into The accumulating set of package specifiers, mutated in place.
+ */
+export function collectUses(document: Document, into: Set<string>): void {
+  for (const decl of document.imports) {
+    if (isUseDecl(decl)) into.add(decl.pkg);
+  }
+}
+
+/** What `pub` has handed over so far, filled as the graph is walked. */
+interface Exports {
+  fragments: Map<string, FragmentDecl>;
+  decos: Map<string, ImportedDeco>;
+}
+
+interface LoadState {
+  document: Document;
+  uri: string;
+  io: ModuleIo;
+  npmLoader?: NpmModules;
+  found: Exports;
+  packages: Set<string>;
+  modules: Map<string, Document>;
+  npm: Map<string, Record<string, unknown>>;
+  seen: Set<string>;
+}
+
+async function loadInto(state: LoadState): Promise<void> {
+  for (const decl of state.document.imports) {
+    if (!isValueImport(decl)) continue;
+    if (isPackageSpecifier(decl.path)) await loadPackage(state, decl.path);
+    else await loadModule(state, decl.path);
+  }
+}
+
+/**
+ * A package that was installed, loaded once however many files name it.
+ *
+ * A host with no loader has no packages, which is the state of every run in a
+ * Web Worker. The import is left unbound and the name reports itself as unknown,
+ * rather than the load failing somewhere nobody can see.
+ */
+async function loadPackage(state: LoadState, spec: string): Promise<void> {
+  if (state.npm.has(spec) || !state.npmLoader) return;
+  const found = await state.npmLoader.load(spec).catch(() => undefined);
+  if (found) state.npm.set(spec, found);
+}
+
+async function loadModule(state: LoadState, spec: string): Promise<void> {
+  const target = state.io.resolve(state.uri, spec);
+  if (state.seen.has(target)) return;
+  state.seen.add(target);
+  const source = await state.io.read(target).catch(() => undefined);
+  if (source === undefined) return;
+  const { ast } = parse(source, { uri: target });
+  state.modules.set(target, ast);
+  collectExports({ document: ast, uri: target, found: state.found });
+  collectUses(ast, state.packages);
+  await loadInto({ ...state, document: ast, uri: target });
+}
+
+/** Everything a file marked `pub`. A `deco` keeps its file: its faults are its own. */
+function collectExports(args: { document: Document; uri: string; found: Exports }): void {
+  for (const decl of args.document.decls) {
+    if (isFragmentDecl(decl) && decl.export) args.found.fragments.set(decl.name, decl);
+    if (isDecoDecl(decl) && decl.export) args.found.decos.set(decl.name, { decl, uri: args.uri });
+  }
+}

package/src/run/runner.types.ts

@@ -0,0 +1,67 @@
+import type { Host } from "@venn-lang/contracts";
+import type { Document, FragmentDecl, ImportedDeco, Problem, RunId } from "@venn-lang/core";
+import type { PluginDefinition } from "@venn-lang/sdk";
+import type { EventSink } from "../eventsink/index.js";
+import type { PortBinding, PortResolver } from "../ports/index.js";
+import type { Registry } from "../registry/index.js";
+import type { CleanupSink, ImportGraph, RunFilter } from "../scheduler/index.js";
+
+/** What one run reports back to the host that asked for it. */
+export interface RunResult {
+  run: RunId;
+  /** What the decorators refused before a statement ran. */
+  problems?: Problem[];
+  passed: number;
+  failed: number;
+  /**
+   * What `exit` asked the host to end with, absent if nothing called it. It
+   * overrides the tally on purpose: `exit 0` is a clean ending, `exit 3` is a
+   * failure the program named itself.
+   */
+  exitCode?: number;
+}
+
+/** The level-2 embedding API (§18): a host owns the process and drives runs. */
+export interface Runner {
+  /** Test mode: collect and run every `flow`, reporting each. */
+  run(document: Document): Promise<RunResult>;
+  /** Script mode: execute the file's top-level statements, top to bottom. */
+  script(document: Document): Promise<RunResult>;
+}
+
+/** Everything {@link Runner} needs, settled once and reused for every run. */
+export interface RunnerArgs {
+  host: Host;
+  plugins: readonly PluginDefinition[];
+  sink: EventSink;
+  ports?: readonly PortBinding[];
+  uri?: string;
+  /** Which flows and steps to include (`--tags`, `--flow`, `--step`). */
+  filter?: RunFilter;
+  /** Stop after the first flow that fails. */
+  bail?: boolean;
+  env?: Record<string, unknown>;
+  moduleFragments?: Map<string, FragmentDecl>;
+  /**
+   * The files the import graph reached, so a `pub fn` can be called with the
+   * module it was written in around it. Without this the name resolves and the
+   * call fails on its first line.
+   */
+  modules?: ImportGraph;
+  /** The `pub deco`s the imported files exported, so `@name` resolves across a file. */
+  moduleDecos?: Map<string, ImportedDeco>;
+  /**
+   * Where a script-mode program registers what it opened. The host owns when a
+   * program ends, so the host owns this list; without one the run keeps its own
+   * and nothing closes until the process does.
+   */
+  cleanup?: CleanupSink;
+}
+
+/** What one `run(document)` needs: the runner's args plus the once-built services. */
+export interface RunOnceInput {
+  args: RunnerArgs;
+  registry: Registry;
+  resolver: PortResolver;
+  document: Document;
+}

package/src/scheduler/absorb-exit.ts

@@ -0,0 +1,19 @@
+import type { Engine } from "./engine.types.js";
+import { ExitSignal } from "./signals.js";
+
+/**
+ * Run `body`, letting an `exit` end it.
+ *
+ * The exit code travels on the engine rather than on the throw, so whatever
+ * wraps this still finishes the way it always does: teardowns run, `run.finished`
+ * is emitted, and the host reads one number off the result. Any other error keeps
+ * unwinding untouched.
+ */
+export async function absorbExit(engine: Engine, body: () => Promise<void>): Promise<void> {
+  try {
+    await body();
+  } catch (error) {
+    if (!(error instanceof ExitSignal)) throw error;
+    engine.exit = error.code;
+  }
+}

package/src/scheduler/aliases.ts

@@ -0,0 +1,44 @@
+import { type Document, isLetStmt, isUseDecl } from "@venn-lang/core";
+import type { Registry } from "../registry/index.js";
+
+/**
+ * The namespaces this file actually brought in with `use`: the alias when one
+ * was given, otherwise the namespace the package contributes. A name outside
+ * this set was never imported, however well the registry knows it.
+ */
+export function collectNamespaces(document: Document, registry: Registry): Set<string> {
+  const names = new Set<string>();
+  for (const decl of document.imports) {
+    if (!isUseDecl(decl)) continue;
+    const bound = decl.alias ?? registry.namespaceOf(decl.pkg);
+    if (bound) names.add(bound);
+  }
+  return names;
+}
+
+/**
+ * The names this file binds at the top level.
+ *
+ * `page.click()` reads like a namespace and a verb, and is neither: it is a
+ * method on something the file holds. The registry has no schema for it, so it
+ * is known here and verified nowhere, which beats reporting it as an unknown
+ * action it plainly is not.
+ */
+export function collectBoundNames(document: Document): Set<string> {
+  const names = new Set<string>();
+  for (const decl of document.decls) {
+    if (isLetStmt(decl)) names.add(decl.name);
+  }
+  return names;
+}
+
+/** `use "venn/http" as h` → `{ h → "http" }`, resolved through the registry. */
+export function collectAliases(document: Document, registry: Registry): Map<string, string> {
+  const aliases = new Map<string, string>();
+  for (const decl of document.imports) {
+    if (!isUseDecl(decl) || !decl.alias) continue;
+    const namespace = registry.namespaceOf(decl.pkg);
+    if (namespace) aliases.set(decl.alias, namespace);
+  }
+  return aliases;
+}

package/src/scheduler/annotations.ts

@@ -0,0 +1,48 @@
+import { type Annotation, readMeta } from "@venn-lang/core";
+
+/** Any AST node that carries decorators (`@tags`, `@timeout`, `@retry`…). */
+export interface Annotated {
+  annotations: Annotation[];
+}
+
+/** A parsed `@retry(n, { backoff, factor })` specification. */
+export interface RetrySpec {
+  attempts: number;
+  backoffMs: number;
+  factor: number;
+}
+
+/**
+ * Whether a decorator set this flag on the node.
+ *
+ * The scheduler reads what expansion concluded, never the annotation list
+ * itself, so a decorator is understood in exactly one place.
+ */
+export function hasAnnotation(node: Annotated, name: string): boolean {
+  return readMeta<boolean>(node, name) === true;
+}
+
+/** `@timeout(90s)` → milliseconds, or undefined. */
+export function readTimeout(node: Annotated): number | undefined {
+  return readMeta<number>(node, "timeout");
+}
+
+/** `@retry(2, { backoff: 500ms, factor: 2 })` → a retry spec, or undefined. */
+export function readRetry(node: Annotated): RetrySpec | undefined {
+  return readMeta<RetrySpec>(node, "retry");
+}
+
+/** `@lock("orders")` → the lock name, or undefined. */
+export function readLock(node: Annotated): string | undefined {
+  return readMeta<string>(node, "lock");
+}
+
+/** `@flaky(0.05)` → the tolerated failure ratio; bare `@flaky` tolerates all. */
+export function readFlaky(node: Annotated): number | undefined {
+  return readMeta<number>(node, "flaky");
+}
+
+/** `@tags(smoke, critical)` → the tag names. */
+export function readTags(node: Annotated): string[] {
+  return readMeta<string[]>(node, "tags") ?? [];
+}

package/src/scheduler/base-scope.ts

@@ -0,0 +1,25 @@
+import { createScope, type Scope } from "../scope/index.js";
+import { bindNamespaces } from "./bind-namespaces.js";
+import { bindPrelude } from "./bind-prelude.js";
+import type { Engine } from "./engine.types.js";
+
+/**
+ * What every scope in a run starts as: the prelude, the plugin namespaces and
+ * `env`, plus the matrix variant when there is one.
+ *
+ * One place, shared by test runs and program runs, so `venn test` and `venn run`
+ * cannot drift apart. Every scope a module is read in starts here too, which is
+ * what makes an imported function see the same world its own file did.
+ */
+export function createBaseScope(args: {
+  engine: Engine;
+  /** The `matrix` variant, for a test run. A program has none. */
+  variant?: Record<string, unknown>;
+}): Scope {
+  const scope = createScope();
+  bindPrelude(scope);
+  bindNamespaces({ registry: args.engine.registry, ctx: args.engine.ctx, scope });
+  scope.set("env", args.engine.env);
+  if (args.variant) scope.set("matrix", args.variant);
+  return scope;
+}

package/src/scheduler/bind-globals.ts

@@ -0,0 +1,55 @@
+import {
+  closureOfDecl,
+  type Document,
+  decorateCallable,
+  evaluate,
+  isFnDecl,
+  isLetStmt,
+  type LetStmt,
+} from "@venn-lang/core";
+import type { Scope } from "../scope/index.js";
+
+/**
+ * Bind every top-level `fn` as a callable closure. Hoisted so a function can be
+ * called before its textual position, and so functions may reference each other
+ * regardless of order.
+ *
+ * A `deco` that asked to `wrap` one is honoured here, at the only moment the
+ * function becomes a value: the name a caller reaches is already the decorated
+ * one, so a middleware cannot be stepped around by calling from somewhere else.
+ */
+export function bindFunctions(doc: Document, scope: Scope): void {
+  for (const decl of doc.decls) {
+    if (isFnDecl(decl)) scope.set(decl.name, decorateCallable(decl, closureOfDecl(decl, scope)));
+  }
+}
+
+/**
+ * Bind top-level functions and the plain `const`/`let` globals every flow can
+ * read without anything having run yet.
+ *
+ * Only pure values. One that calls a verb is a statement, and statements run in
+ * the prologue, in order and once, before the flows: opening a database is
+ * something a program *does*, not something a name quietly is.
+ */
+export function bindGlobals(doc: Document, scope: Scope): void {
+  bindFunctions(doc, scope);
+  bindPlainValues(doc, scope);
+}
+
+/**
+ * The `const`/`let` globals, evaluated where they stand.
+ *
+ * Separate from the functions because the order matters across files: a function
+ * is hoisted and reads its scope when it is *called*, while a value is read now,
+ * so anything a value might depend on has to be in place first.
+ */
+export function bindPlainValues(doc: Document, scope: Scope): void {
+  for (const decl of doc.decls) {
+    if (isLetStmt(decl) && isPlainValue(decl)) scope.set(decl.name, evaluate(decl.value, scope));
+  }
+}
+
+function isPlainValue(decl: LetStmt): boolean {
+  return decl.args.length === 0 && !decl.opts;
+}

package/src/scheduler/bind-imports.ts

@@ -0,0 +1,130 @@
+import {
+  type Document,
+  isFnDecl,
+  isPackageSpecifier,
+  isValueImport,
+  type ValueImport,
+} from "@venn-lang/core";
+import type { Scope } from "../scope/index.js";
+import { bindFunctions, bindPlainValues } from "./bind-globals.js";
+
+/** The files an import graph reached, and how one file names another. */
+export interface ImportGraph {
+  modules: ReadonlyMap<string, Document>;
+  resolve: (from: string, spec: string) => string;
+  /** What each npm specifier loaded to, already a value. */
+  npm?: ReadonlyMap<string, Record<string, unknown>>;
+}
+
+/**
+ * Bind the names a document imported, each to the value its own module made.
+ *
+ * A `pub fn` is a closure over the file it was written in (it calls that file's
+ * private helpers and reads that file's globals) so it cannot simply be lifted
+ * into the importer's scope. Each module gets a scope of its own, built the same
+ * way the entry document's is, and the importer takes only the names it asked
+ * for out of it.
+ *
+ * Three passes over every module at once, not one walk per import: two files
+ * that call each other are ordinary, and no single order resolves them. Filling
+ * every scope first and wiring afterwards has no order to get wrong, because a
+ * function captures the scope object and wiring mutates that same object.
+ *
+ * Call this before the document's own globals are bound, so a local name of the
+ * same spelling wins. That is the rule fragments already follow.
+ */
+export function bindImports(args: {
+  document: Document;
+  uri: string;
+  scope: Scope;
+  graph: ImportGraph;
+  /** How the scope a module is read in gets made: the same one the entry gets. */
+  base: () => Scope;
+}): void {
+  const built = new Map<string, Scope>();
+  for (const [uri, module] of args.graph.modules) {
+    const scope = args.base();
+    built.set(uri, scope);
+    bindFunctions(module, scope);
+  }
+  for (const [uri, module] of args.graph.modules) {
+    wire({ document: module, uri, into: scopeAt(built, uri), graph: args.graph, built });
+  }
+  // Last, because a value is read where it stands: whatever it names, local
+  // function or imported one, has to be in place by now.
+  for (const [uri, module] of args.graph.modules) bindPlainValues(module, scopeAt(built, uri));
+  wire({ document: args.document, uri: args.uri, into: args.scope, graph: args.graph, built });
+}
+
+function scopeAt(built: ReadonlyMap<string, Scope>, uri: string): Scope {
+  return built.get(uri) as Scope;
+}
+
+interface Wiring {
+  document: Document;
+  uri: string;
+  into: Scope;
+  graph: ImportGraph;
+  built: ReadonlyMap<string, Scope>;
+}
+
+function wire(args: Wiring): void {
+  for (const decl of args.document.imports) {
+    if (!isValueImport(decl)) continue;
+    if (isPackageSpecifier(decl.path)) {
+      takePackage({ decl, graph: args.graph, into: args.into });
+      continue;
+    }
+    const from = args.graph.resolve(args.uri, decl.path);
+    const module = args.graph.modules.get(from);
+    const source = args.built.get(from);
+    if (module && source) take({ decl, module, source, into: args.into });
+  }
+}
+
+/**
+ * The names an installed package published.
+ *
+ * Everything it exports is fair game: a package has no `pub`, its exports *are*
+ * what it made public, so unlike a `.vn` module there is nothing to filter
+ * against. `default` is bound too, under whichever name the import gave it,
+ * because that is the only name it has here.
+ */
+function takePackage(args: { decl: ValueImport; graph: ImportGraph; into: Scope }): void {
+  const module = args.graph.npm?.get(args.decl.path);
+  if (!module) return;
+  if (args.decl.wildcard) return void args.into.set(args.decl.wildcard, { ...module });
+  if (args.decl.default) return void args.into.set(args.decl.default, module.default);
+  for (const name of args.decl.names) {
+    if (name in module) args.into.set(name, module[name]);
+  }
+}
+
+function take(args: { decl: ValueImport; module: Document; source: Scope; into: Scope }): void {
+  const exported = exportedFunctions(args.module);
+  if (args.decl.wildcard) {
+    args.into.set(args.decl.wildcard, gathered(exported, args.source));
+    return;
+  }
+  // Only what the module offered. Reaching a private name would work here and
+  // then stop working the day that file rearranges its own insides.
+  for (const name of args.decl.names) {
+    if (exported.has(name)) args.into.set(name, args.source.lookup(name));
+  }
+}
+
+/** `import * as u`: one value holding everything the module published. */
+function gathered(names: ReadonlySet<string>, source: Scope): Record<string, unknown> {
+  const out: Record<string, unknown> = {};
+  for (const name of names) out[name] = source.lookup(name);
+  return out;
+}
+
+/** The names a module made `pub` that are values a caller can hold. */
+function exportedFunctions(document: Document): Set<string> {
+  const names = new Set<string>();
+  for (const decl of document.decls) {
+    if (isFnDecl(decl) && decl.export) names.add(decl.name);
+  }
+  return names;
+}

package/src/scheduler/bind-namespaces.ts

@@ -0,0 +1,61 @@
+import { namespaceValue, nativeFn } from "@venn-lang/core";
+import type { ActionContext, ActionDefinition } from "@venn-lang/sdk";
+import type { Registry } from "../registry/index.js";
+import type { Scope } from "../scope/index.js";
+
+/**
+ * Put every plugin namespace in the root scope as a value, so its verbs work
+ * inside any expression (`print(fmt.table(rows))`, `"${crypto.hash(x)}"`) and
+ * not only as a bare statement.
+ *
+ * A verb reached this way runs synchronously: it is the pure corner of the
+ * stdlib (`fmt`, `data`, `crypto`). An action that does I/O returns its promise,
+ * so those stay written as statements, where the runtime awaits them.
+ */
+export function bindNamespaces(args: {
+  registry: Registry;
+  ctx: ActionContext;
+  scope: Scope;
+}): void {
+  const byNamespace = new Map<string, Record<string, unknown>>();
+  for (const entry of args.registry.actions()) {
+    const members = byNamespace.get(entry.namespace) ?? {};
+    place({ members, path: entry.name, action: entry.action, ctx: args.ctx });
+    byNamespace.set(entry.namespace, members);
+  }
+  for (const [namespace, members] of byNamespace) {
+    args.scope.set(namespace, namespaceValue(members));
+  }
+}
+
+/** `jwt.decode` nests: `{ jwt: { decode: fn } }`, so the dotted call reads naturally. */
+function place(args: {
+  members: Record<string, unknown>;
+  path: string;
+  action: ActionDefinition;
+  ctx: ActionContext;
+}): void {
+  const parts = args.path.split(".");
+  const leaf = parts.pop();
+  if (!leaf) return;
+  let level = args.members;
+  for (const part of parts) {
+    level[part] = (level[part] as Record<string, unknown>) ?? {};
+    level = level[part] as Record<string, unknown>;
+  }
+  level[leaf] = verb(args.action, args.ctx);
+}
+
+function verb(action: ActionDefinition, ctx: ActionContext): unknown {
+  return nativeFn((values) => action.run(ctx, { args: values, params: params(action) }));
+}
+
+/** No options map in expression position, so let the schema supply its defaults. */
+function params(action: ActionDefinition): unknown {
+  if (!action.params) return {};
+  try {
+    return action.params.parse({});
+  } catch {
+    return {};
+  }
+}

package/src/scheduler/bind-prelude.ts

@@ -0,0 +1,11 @@
+import { PRELUDE_VALUES } from "@venn-lang/core";
+import type { Scope } from "../scope/index.js";
+
+/**
+ * Put the pure prelude values (`len`, `range`, `pretty`, `str`, `min`…) in the
+ * root scope. Being values rather than verbs is what lets them appear in any
+ * expression: `xs.take(len(ys))`, `"${pretty(user)}"`.
+ */
+export function bindPrelude(scope: Scope): void {
+  for (const [name, value] of Object.entries(PRELUDE_VALUES)) scope.set(name, value);
+}