@procedures/client 0.24.1 → 0.26.0

package/src/procedures/core/operands.ts

@@ -7,25 +7,22 @@ import type { ProcedureContext, ProcedurePath } from "../types.js";
 import { PROCEDURE_REGISTRY } from "../registry.js";
 import {
   isAnyProcedureRef,
-  getRefWhen,
-  WHEN_NEVER,
-  normalizeRef,
-  hydrateInput,
-  createRefScope,
   isOutputRef,
   lookupOutputRef,
   rawInputRule,
-  withScope,
+  scopeOf,
   RUNS_REFS_TAG,
   type AnyProcedureRef,
   type RefScope,
 } from "../ref.js";
+import { hydrateInput, runControl, runOperand } from "../program.js";
 
 /**
- * The operands of a control-flow procedure arrive raw: the procedure runs them itself, with the
- * same rules as `exec()` (one interpreter, `hydrateInput`). The input of a control-flow procedure
- * carries the scope of its caller (`withScope`), so an operand can read the names of an outer
- * chain, `map` or `reduce` (deep dive CORE-13).
+ * The operands of a control-flow procedure arrive raw: the procedure runs them itself. Its logic
+ * is a runner of the coeval kernel (`program.ts`): the handler gives its input to the kernel as a
+ * document of its kind. The input of a control-flow procedure carries the scope of its caller
+ * (`withScope`), so an operand can read the names of an outer chain, `map` or `reduce` (deep dive
+ * CORE-13).
  */
 
 /** The raw-input rule of the caller's registry: the procedure at a path decides (deep dive CORE-5). */
@@ -33,6 +30,19 @@ export function rawRule(ctx: ProcedureContext): (path: ProcedurePath) => boolean
   return rawInputRule(ctx.registry ?? PROCEDURE_REGISTRY);
 }
 
+/** The call function of a context, as the executor of a run. */
+function executorOf(ctx: ProcedureContext): <TIn, TOut>(path: ProcedurePath, input: TIn) => Promise<TOut> {
+  return <TIn, TOut>(path: ProcedurePath, input: TIn): Promise<TOut> => ctx.client.call<TIn, TOut>(path, input);
+}
+
+/**
+ * This function runs a control-flow procedure on the kernel: the runner of its kind runs the
+ * operands of the input, in the scope of the caller.
+ */
+export function runControlFlow(name: string, input: unknown, ctx: ProcedureContext): Promise<unknown> {
+  return runControl(name, input, { executor: executorOf(ctx), rawInput: rawRule(ctx), onRef: ctx.onRef, scope: scopeOf(input) });
+}
+
 /**
  * Without a client, a control-flow procedure cannot run refs. This function gives a value with
  * its known `$ref` values resolved. A name that the scope does not have stays as it is.
@@ -59,38 +69,23 @@ export function substituteRefs(value: unknown, scope: RefScope): unknown {
  * in `scope` and runs its nested refs, as `exec()` does for an input.
  */
 export async function hydrateOperand(value: unknown, ctx: ProcedureContext, scope?: RefScope): Promise<unknown> {
-  return hydrateInput(value, (path, input) => ctx.client.call(path, input), { scope, rawInput: rawRule(ctx), onRef: ctx.onRef });
+  return hydrateInput(value, executorOf(ctx), { scope, rawInput: rawRule(ctx), onRef: ctx.onRef });
 }
 
 /**
- * This function runs an operand ref of a control-flow procedure.
+ * This function runs an operand ref of a control-flow procedure (`runOperand()`).
  *
  * It resolves the `$ref` values of the ref's input in `scope`, runs the nested refs of the
  * input and adds `extra` (the parent context, for example `cwd`). Then it calls the procedure.
  * A procedure that takes its input raw (control flow) gets the input unchanged, with the scope:
- * that procedure runs its own operands.
+ * that procedure runs its own operands. A `$never` operand is data with its full input.
  */
 export async function callOperand(
   ref: AnyProcedureRef,
   ctx: ProcedureContext,
   options: { scope?: RefScope | undefined; extra?: Record<string, unknown> | undefined } = {},
 ): Promise<unknown> {
-  // A $never operand is data with its full input, as a $never call anywhere (roadmap R4: before,
-  // only a chain step obeyed it, and a $never branch, task or fn started)
-  if (getRefWhen(ref) === WHEN_NEVER) {
-    return ref;
-  }
-  const normalized = normalizeRef(ref);
-  const raw = rawRule(ctx)(normalized.path);
-  let input: unknown = raw ? normalized.input : await hydrateOperand(normalized.input, ctx, options.scope);
-  const extra = options.extra ?? {};
-  if (Object.keys(extra).length > 0) {
-    input = { ...(typeof input === "object" && input !== null && !Array.isArray(input) ? input : {}), ...extra };
-  }
-  if (raw) {
-    input = withScope(input, options.scope);
-  }
-  return ctx.client.call(normalized.path, input);
+  return runOperand(ref, { executor: executorOf(ctx), rawInput: rawRule(ctx), onRef: ctx.onRef, scope: options.scope, extra: options.extra });
 }
 
 /** This function gives the value of any operand: a ref runs, another value is hydrated. */
@@ -105,21 +100,6 @@ export async function evaluateOperand(
   return hydrateOperand(value, ctx, options.scope);
 }
 
-/** The parent context that the control-flow procedures give to their operands. */
-export function parentContext(input: { cwd?: string | undefined; node?: unknown }): Record<string, unknown> {
-  return {
-    ...(input.cwd ? { cwd: input.cwd } : {}),
-    ...(input.node ? { node: input.node } : {}),
-  };
-}
-
-/** A child scope of `parent` with the given `$ref` names, for one call of `fn`. */
-export function scopeWith(values: Record<string, unknown>, parent?: RefScope): RefScope {
-  const scope = createRefScope(parent);
-  for (const [name, value] of Object.entries(values)) scope.outputs.set(name, value);
-  return scope;
-}
-
 /** The tags of a control-flow procedure: hydration gives it its input raw. */
 export function controlFlowTags(...tags: string[]): string[] {
   return ["core", ...tags, RUNS_REFS_TAG];

package/src/procedures/define-procedure.ts

@@ -41,13 +41,12 @@ import { anySchema, zodAdapter } from "./core/schemas.js";
 import { keyToPath, pathToKey } from "./types.js";
 import {
   createRefScope,
-  hydrateInput,
   isAnyProcedureRef,
   markDataDriven,
-  prepareRootCall,
   rawInputRule,
   RUNS_REFS_TAG,
 } from "./ref.js";
+import { hydrateInput, runProgram } from "./program.js";
 import { INFERRED_EFFECTS_METADATA_KEY, PURE_EFFECTS, REGISTRY_READ_EFFECTS, effectsValue, withEffects } from "./effects.js";
 import { MANIFEST_FORMAT, describeProcedure as describeEntry, describeRegistry, type ProcedureManifest } from "./manifest.js";
 import type { JsonSchema } from "./schema.js";
@@ -192,9 +191,9 @@ const DeleteOutputSchema = z.looseObject({
 /**
  * Create a handler that executes an aggregation definition.
  *
- * The body runs exactly as `exec()` runs the same program: `prepareRootCall()` gives the path
- * and the input of the root call (a `$when` on the root call does not apply, a list of calls in
- * the root input is an implicit chain), then the body starts the root call. The scope of the
+ * The body runs exactly as `exec()` runs the same program: one run of the kernel (`runProgram()`),
+ * whose root document is the root call (a `$when` on the root call does not apply, a list of calls
+ * in the root input is an implicit chain). The scope of the
  * body has one name, `input`: the input of the procedure. So `{ $ref: "input.dir" }` reads a
  * field of the input, and `$last`, the step names, `item`, `acc` and `index` keep their meaning
  * inside the body. (Deep dive CORE-4: before, a first pass replaced every `$ref` with a field of
@@ -213,8 +212,7 @@ function createAggregationHandler(
     const executor = <TIn, TOut>(path: ProcedurePath, value: TIn): Promise<TOut> => ctx.client.call<TIn, TOut>(path, value);
     const options = { scope, rawInput: rawInputRule(ctx.registry ?? PROCEDURE_REGISTRY), onRef: ctx.onRef };
     if (isAnyProcedureRef(aggregation)) {
-      const root = await prepareRootCall(aggregation, executor, options);
-      return ctx.client.call(root.path, root.input);
+      return runProgram(aggregation, { ...options, executor });
     }
     // A body that is not one call (an older stored program): hydrate it as a root value
     return hydrateInput(aggregation, executor, options);

package/src/procedures/envelope.ts

@@ -1,20 +1,26 @@
 /**
- * The envelope form of a program (coeval, decisions D6, D9 and D10, roadmap item M7).
+ * The envelope form of a program (coeval, decisions D6, D9, D10 and D12, roadmap item M7).
  *
  * A document of coeval is an envelope: `{ type, value }`. The header can also have `name` and
  * `own`. In the envelope form of a program:
  *
  * - A call is `{ type: path, value: input }`. The type is the path of a procedure, as an array of
  *   keys. A ref into the world, `{ type: "ref", value: ["world", ...path] }`, is also a type.
+ *   The arrays `["ref"]` and `["lit"]` are kinds of the kernel, not paths. A procedure with the
+ *   path `["ref"]` or `["lit"]` has a ref into the world as its type.
  * - A ref is `{ type: "ref", value: ["names", name, ...fields] }`. The name `last` reads the result
  *   of the step before.
- * - A literal is `{ type: "literal", value }`. Nothing in its value runs.
+ * - A literal is `{ type: "literal", value }`. Nothing in its value runs. The type `"lit"` (the kind
+ *   of the kernel, coeval D12) is an alias: `{ type: "lit", value }` is also a literal.
  *
- * The envelope form is a surface. `fromEnvelopeProgram()` changes it into the `$proc` form, and
- * `hydrateInput()` starts that form. In the input of a `$proc` program, an envelope is data. Thus
- * the root of a program decides its form, and the old programs do not change.
+ * The functions of this module read the envelopes with the helpers of `@mark1russell7/coeval`
+ * (`isTypeRef`, `typePath`, `ref`). `fromEnvelopeProgram()` changes the envelope form into the
+ * `$proc` form, and `client.exec()` then runs the program on the kernel (`program.ts`). In the
+ * input of a `$proc` program, an envelope is data. Thus the root of a program decides its form,
+ * and the old programs do not change.
  */
 
+import { ENVELOPE_KEYS, isTypeRef, ref as refDocument, typePath } from "@mark1russell7/coeval";
 import {
   LITERAL_KEY,
   MAX_DATA_DEPTH,
@@ -50,7 +56,10 @@ export interface EnvelopeRef {
   readonly value: EnvelopePath;
 }
 
-/** This document gives its value as data. Nothing in the value runs. */
+/**
+ * This document gives its value as data. Nothing in the value runs. The functions of this module
+ * write the type `"literal"`, and they also read the alias `"lit"` (coeval D12: the kind `lit`).
+ */
 export interface EnvelopeLiteral {
   readonly type: "literal";
   readonly value: unknown;
@@ -61,7 +70,14 @@ export interface EnvelopeLiteral {
  * procedure, or a ref into the world. The value is the input. A value of `null` gives no input.
  */
 export interface EnvelopeProgram {
-  /** The path of the procedure, or a ref into the world: `{ type: "ref", value: ["world", ...path] }`. */
+  /**
+   * The path of the procedure, or a ref into the world: `{ type: "ref", value: ["world", ...path] }`.
+   *
+   * The types `["ref"]` and `["lit"]` (arrays of one key) are the kinds `ref` and `lit` of the
+   * kernel (D26). Before 0.26.0, they were calls of the procedures with these paths. A ref into
+   * the world is always a call: it is the type of these procedures, for example
+   * `{ type: { type: "ref", value: ["world", "ref"] }, value }`.
+   */
   readonly type: readonly string[] | EnvelopeRef;
   /** The input of the call. `null` gives a call without input. */
   readonly value: unknown;
@@ -76,7 +92,7 @@ export interface EnvelopeProgram {
 // =============================================================================
 
 /** The keys of an envelope (coeval D6). */
-const HEADER_KEYS: ReadonlySet<string> = new Set(["type", "value", "name", "own"]);
+const HEADER_KEYS: ReadonlySet<string> = new Set<string>(ENVELOPE_KEYS);
 
 /** The keys that make an object a construct of the `$proc` form. */
 const NATIVE_KEYS: readonly string[] = [PROCEDURE_JSON_KEY, OUTPUT_REF_KEY, LITERAL_KEY];
@@ -88,7 +104,10 @@ const CALL_KEYS: ReadonlySet<string> = new Set([PROCEDURE_JSON_KEY, "input", PRO
 const RUNTIME_CALL_KEYS: ReadonlySet<string> = new Set(["path", "input", PROCEDURE_NAME_KEY, PROCEDURE_WHEN_KEY]);
 
 const REF_TYPE = "ref";
+/** The type of a literal that this module writes (the form of 0.23.0). */
 const LITERAL_TYPE = "literal";
+/** The alias of the literal type: the kind `lit` of the kernel (coeval D12). */
+const LIT_ALIAS = "lit";
 const NAMES_ROOT = "names";
 const WORLD_ROOT = "world";
 /** The name of the result of the step before, in the envelope form. */
@@ -102,7 +121,10 @@ type EnvelopeShape = Record<string, unknown> & { type: unknown; value: unknown;
 /** The kind of an envelope. `bad-type` is an envelope whose type is a ref that is not into the world. */
 type EnvelopeKind = "call" | "ref" | "literal" | "bad-type";
 
-/** This function gives true if a value is a plain object with `type`, `value` and only header keys. */
+/**
+ * This function gives true if a value is a plain object with `type`, `value` and only header keys.
+ * The test is looser than `isEnvelope()` of coeval: a bad header gives an error, not data.
+ */
 function isEnvelopeShape(value: unknown): value is EnvelopeShape {
   return (
     isPlainObject(value) &&
@@ -117,22 +139,39 @@ function isProcedurePath(value: unknown): value is readonly string[] {
   return Array.isArray(value) && value.length > 0 && value.every((key) => typeof key === "string");
 }
 
-/** This function gives the procedure path of the type of a call, or undefined for another type. */
+/** The keys of the type of an envelope (coeval `typePath()`), or undefined for a value that is not a type. */
+function keysOfType(type: unknown): readonly (string | number)[] | undefined {
+  // A ref into the world as a type has only the keys `type` and `value`
+  if (isPlainObject(type) && Object.keys(type).length !== 2) return undefined;
+  return isTypeRef(type) ? typePath(type) : undefined;
+}
+
+/** This function gives true if a procedure path is also a kind of the kernel as an array: `["ref"]` or `["lit"]`. */
+function isKindPath(keys: readonly unknown[]): boolean {
+  return keys.length === 1 && (keys[0] === REF_TYPE || keys[0] === LIT_ALIAS);
+}
+
+/**
+ * This function gives the procedure path of the type of a call, or undefined for another type. The
+ * type of a call is a list of keys, or a ref into the world. A text is a kind of the kernel or data.
+ * The arrays `["ref"]` and `["lit"]` are kinds. A ref into the world is always a call, so it is the
+ * type of the procedures `["ref"]` and `["lit"]`.
+ */
 function callPath(type: unknown): ProcedurePath | undefined {
-  if (isProcedurePath(type)) return [...type];
-  if (!isEnvelopeShape(type) || type.type !== REF_TYPE || Object.keys(type).length !== 2) return undefined;
-  const path = type.value;
-  if (!Array.isArray(path) || path[0] !== WORLD_ROOT) return undefined;
-  const steps: unknown[] = path.slice(1);
-  return isProcedurePath(steps) ? [...steps] : undefined;
+  if (typeof type === "string") return undefined;
+  const keys = keysOfType(type);
+  if (!isProcedurePath(keys) || (Array.isArray(type) && isKindPath(keys))) return undefined;
+  return [...keys];
 }
 
 /** This function gives the kind of an envelope, or undefined for data. */
 function envelopeKind(value: unknown): EnvelopeKind | undefined {
   if (!isEnvelopeShape(value)) return undefined;
   if (callPath(value.type)) return "call";
-  if (value.type === REF_TYPE) return "ref";
-  if (value.type === LITERAL_TYPE) return "literal";
+  const keys = keysOfType(value.type);
+  if (keys?.length === 1 && keys[0] === REF_TYPE) return "ref";
+  // The text "literal" (an array ["literal"] is a call, as in 0.23.0), or the alias "lit"
+  if (value.type === LITERAL_TYPE || (keys?.length === 1 && keys[0] === LIT_ALIAS)) return "literal";
   if (isPlainObject(value.type) && value.type["type"] === REF_TYPE) return "bad-type";
   return undefined;
 }
@@ -332,7 +371,8 @@ function fromValue(value: unknown, path: Set<object>, depth: number): unknown {
  *
  * The conversion:
  * - A call gives `{ type: path, value: input }`. A call without input gives the value `null`.
- *   `$name` gives the header `name`.
+ *   `$name` gives the header `name`. The path `["ref"]` or `["lit"]` gives a ref into the world as
+ *   the type, because these arrays are kinds of the kernel.
  * - `{ $ref: "$last" }` gives `["names", "last"]`. `{ $ref: "a.b" }` gives `["names", "a", "b"]`.
  * - `{ $literal: value }` gives a literal.
  * - Data that the envelope form reads in a different way gives a literal. This data is an object
@@ -396,7 +436,10 @@ function toCall(call: AnyProcedureRef, path: Set<object>, depth: number): Envelo
       : input === null
         ? { type: LITERAL_TYPE, value: null }
         : toValue(input, path, depth + 1);
-  return { type: [...procedurePath], value, ...($name !== undefined ? { name: $name } : {}) };
+  const type: EnvelopeProgram["type"] = isKindPath(procedurePath)
+    ? (refDocument(WORLD_ROOT, ...procedurePath) as unknown as EnvelopeRef)
+    : [...procedurePath];
+  return { type, value, ...($name !== undefined ? { name: $name } : {}) };
 }
 
 /** This function changes a `$ref` object into a ref envelope. */
@@ -414,7 +457,7 @@ function toRef(ref: OutputRef): EnvelopeRef {
         `["names", "last"] reads the result of the step before: give the step a different name`
     );
   }
-  return { type: REF_TYPE, value: [NAMES_ROOT, name === NATIVE_LAST ? LAST_NAME : name!, ...fields] };
+  return refDocument(NAMES_ROOT, name === NATIVE_LAST ? LAST_NAME : name!, ...fields) as unknown as EnvelopeRef;
 }
 
 /** This function gives true if a value of the `$proc` form is a call, a `$ref` or a `$literal`. */

package/src/procedures/idempotency.ts

@@ -136,6 +136,14 @@ export class IdempotencyCache<T> {
     return { result: entry.done, first: true };
   }
 
+  /**
+   * This method removes the result of a key, so the next call with the key runs again. It removes
+   * the entry only when `result` is its result: an entry of a newer call with the key stays.
+   */
+  forget(key: string, result: Promise<T>): void {
+    if (this.entries.get(key)?.done === result) this.entries.delete(key);
+  }
+
   /** This function removes the expired results, and the oldest finished results above the limit. */
   private evict(): void {
     const now = this.now();

package/src/procedures/index.ts

@@ -247,9 +247,6 @@ export {
   toJson,
   normalizeRef,
   // Hydration
-  hydrateInput,
-  prepareRootCall,
-  observeHydration,
   isControlFlowPath,
   isDataDriven,
   markDataDriven,
@@ -266,7 +263,6 @@ export {
   resolveOutputRef,
   withScope,
   scopeOf,
-  executeRef,
   // Templates
   extractTemplate,
   parseProcedureJson,
@@ -287,6 +283,40 @@ export type {
   RefScope,
 } from "./ref.js";
 
+// Programs on the coeval kernel (coeval D13, roadmap item M14): the run documents and their machine
+export {
+  hydrateInput,
+  prepareRootCall,
+  observeHydration,
+  executeRef,
+  runProgram,
+  runDocument,
+  runControl,
+  runOperand,
+  programMachine,
+  PROGRAM_WORLD,
+  desugarProgram,
+  desugarInput,
+  desugarControlValue,
+  originOf,
+  namesOfScope,
+  scopeOfNames,
+  nameKey,
+  nameOfKey,
+  documentAt,
+} from "./program.js";
+export type {
+  ProgramDocument,
+  DocumentOrigin,
+  DesugarOptions,
+  ProgramHost,
+  CallEffect,
+  CallPosition,
+  RunDocumentOptions,
+  RunProgramOptions,
+  RunOperandOptions,
+} from "./program.js";
+
 // The envelope form of a program (coeval D6, D9 and D10, roadmap item M7)
 export { isEnvelopeProgram, fromEnvelopeProgram, toEnvelopeProgram } from "./envelope.js";
 export type { EnvelopeProgram, EnvelopeRef, EnvelopeLiteral, EnvelopePath } from "./envelope.js";