openalgo-script 0.1.0-alpha.1 → 0.4.0

package/src/core/engine/verify-code.ts

@@ -0,0 +1,332 @@
+/**
+ * Checks 2 to 7 of `compiled-program.md` 3.5, over one instruction list.
+ *
+ * Verification is not optional and not a debug mode. An engine that skips it
+ * can be handed a malformed list and will read past the end of an array or jump
+ * into the middle of an expression, and the failure will look like a wrong
+ * number rather than a broken program. What these five checks buy is stated in
+ * the specification and is worth restating here, because it is what the engine
+ * is then allowed to assume and therefore not to test: a verified program
+ * cannot underflow the stack, cannot jump out of bounds, cannot address a slot
+ * that does not exist and cannot loop without charging the budget.
+ *
+ * Check 6 is the one that looks small and is not. The target of every backward
+ * jump must be a `TICK`, so no cycle in the control flow graph can run without
+ * charging the loop budget, and the whole of that guarantee is checkable by
+ * looking at one instruction rather than by analysing the graph.
+ */
+import { depthChange, isOpcode, operandCount, targetPositions } from '../emit/index.js';
+import type { Opcode } from '../emit/index.js';
+import type { Instruction } from './types.js';
+import { atInstruction } from './errors.js';
+import type { ShapeCheck } from './verify-shape.js';
+
+export interface ListLimits {
+  readonly slots: number;
+  readonly cells: number;
+  readonly states: number;
+  readonly registers: number;
+  readonly channels: number;
+  readonly libFunctions: number;
+  readonly callSites: number;
+  readonly loops: number;
+  readonly consts: number;
+  readonly series: number;
+  argcOf(site: number): number;
+}
+
+/** Which table each opcode's operands index, by position. */
+const OPERAND_TABLES: Partial<Record<Opcode, readonly (keyof ListLimits | 'target' | 'count')[]>> =
+  {
+    CONST: ['consts'],
+    LOAD: ['slots'],
+    STORE: ['slots'],
+    CELL_INIT: ['cells', 'target'],
+    LOADC: ['cells'],
+    STOREC: ['cells'],
+    SLOAD: ['registers'],
+    SSTORE: ['registers'],
+    HIST: ['registers'],
+    HISTP: ['series'],
+    AND_SHORT: ['target'],
+    OR_SHORT: ['target'],
+    JUMP: ['target'],
+    JUMP_FALSE: ['target'],
+    TICK: ['loops'],
+    FOR_INIT: ['loops', 'slots', 'slots', 'slots', 'target'],
+    FOR_NEXT: ['loops', 'slots', 'slots', 'slots', 'target'],
+    ARRAY: ['count'],
+    CALL_LIB: ['libFunctions', 'count', 'states'],
+    CALL_FN: ['callSites'],
+    EMIT: ['channels'],
+  };
+
+const TABLE_NAMES: Readonly<Record<string, string>> = {
+  slots: 'the frame',
+  cells: 'cells',
+  states: 'states',
+  registers: 'series',
+  channels: 'channels',
+  libFunctions: 'lib.functions',
+  callSites: 'callSites',
+  loops: 'loops',
+  consts: 'consts',
+  series: "the call site's series bindings",
+};
+
+/**
+ * Walks one instruction list.
+ *
+ * `terminator` is `HALT` for the bar's own code and `RET` for a function body,
+ * which is check 4, and the depth at it must be zero, which is check 5.
+ */
+export function checkList(
+  shape: ShapeCheck,
+  list: string,
+  code: readonly Instruction[],
+  limits: ListLimits,
+  terminator: 'HALT' | 'RET',
+): boolean {
+  if (code.length === 0) return shape.fail(list, 'an instruction list cannot be empty');
+
+  // Check 2, and every operand in range.
+  for (let i = 0; i < code.length; i += 1) {
+    const instruction = code[i];
+    const where = atInstruction(list, i);
+    if (!shape.array(instruction, where)) return false;
+    const opcode = instruction[0];
+    if (typeof opcode !== 'string' || !isOpcode(opcode)) {
+      return shape.fail(where, `${String(opcode)} is not an opcode this engine implements`);
+    }
+    const operands = instruction.slice(1);
+    if (operands.length !== operandCount(opcode)) {
+      return shape.fail(
+        where,
+        `${opcode} takes ${operandCount(opcode)} operands and carries ${operands.length}`,
+      );
+    }
+    const tables = OPERAND_TABLES[opcode] ?? [];
+    for (let k = 0; k < operands.length; k += 1) {
+      const operand = operands[k];
+      if (!shape.whole(operand, `${where} operand ${k}`)) return false;
+      const table = tables[k];
+      if (table === undefined || table === 'count') {
+        if ((operand as number) < 0) {
+          return shape.fail(`${where} operand ${k}`, 'a count cannot be negative');
+        }
+        continue;
+      }
+      if (table === 'target') {
+        // Check 3. A target past the terminator is out of the list.
+        if ((operand as number) < 0 || (operand as number) >= code.length) {
+          return shape.fail(where, `the jump target ${operand} is outside a list of ${code.length}`);
+        }
+        continue;
+      }
+      // A `CALL_LIB` with no state carries -1 rather than an index.
+      if (opcode === 'CALL_LIB' && table === 'states' && operand === -1) continue;
+      // A series binding a call site left unbound is -1 and no `HISTP` may reach it.
+      const size = limits[table];
+      if (typeof size !== 'number') continue;
+      if ((operand as number) < 0 || (operand as number) >= size) {
+        return shape.fail(
+          `${where} operand ${k}`,
+          `${operand} is outside ${TABLE_NAMES[table] ?? table}, which holds ${size}`,
+        );
+      }
+    }
+
+    // Check 6: the target of a backward jump is a `TICK`.
+    for (const position of targetPositions(opcode)) {
+      const target = operands[position];
+      if (typeof target !== 'number' || target > i) continue;
+      const landing = code[target];
+      if (!Array.isArray(landing) || landing[0] !== 'TICK') {
+        return shape.fail(
+          where,
+          `it jumps back to instruction ${target}, which is not a TICK, so the loop would ` +
+            'run without charging the budget',
+        );
+      }
+    }
+  }
+
+  // Check 4.
+  const last = code[code.length - 1];
+  if (!Array.isArray(last) || last[0] !== terminator) {
+    return shape.fail(
+      atInstruction(list, code.length - 1),
+      `the last instruction of this list must be ${terminator}`,
+    );
+  }
+  for (let i = 0; i < code.length - 1; i += 1) {
+    const opcode = code[i]?.[0];
+    if (opcode === 'HALT') {
+      return shape.fail(atInstruction(list, i), 'HALT is only ever the last instruction');
+    }
+  }
+
+  return checkDepths(shape, list, code, limits, terminator);
+}
+
+/**
+ * Check 5: the stack depth agrees on every path, never goes below zero, and is
+ * zero at the terminator.
+ *
+ * A disagreement at a join is a corrupt program, not a program with an unusual
+ * shape: every instruction has a fixed effect, so two paths reaching one
+ * instruction with different depths means one of them was built wrong.
+ */
+function checkDepths(
+  shape: ShapeCheck,
+  list: string,
+  code: readonly Instruction[],
+  limits: ListLimits,
+  terminator: 'HALT' | 'RET',
+): boolean {
+  const depths: (number | undefined)[] = new Array<number | undefined>(code.length).fill(undefined);
+  const pending: number[] = [0];
+  depths[0] = 0;
+
+  const reach = (index: number, depth: number, from: number): boolean => {
+    if (depth < 0) {
+      return shape.fail(atInstruction(list, from), 'it takes the stack below zero');
+    }
+    const known = depths[index];
+    if (known === undefined) {
+      depths[index] = depth;
+      pending.push(index);
+      return true;
+    }
+    if (known !== depth) {
+      return shape.fail(
+        atInstruction(list, index),
+        `two paths reach it with the stack ${known} and ${depth} deep`,
+      );
+    }
+    return true;
+  };
+
+  while (pending.length > 0) {
+    const index = pending.pop();
+    if (index === undefined) continue;
+    const instruction = code[index];
+    const depth = depths[index];
+    if (instruction === undefined || depth === undefined) continue;
+    const opcode = instruction[0] as Opcode;
+    const operands = instruction.slice(1) as number[];
+    const after = depth + depthChange(opcode, operands, (site) => limits.argcOf(site));
+    if (opcode === terminator) {
+      if (after !== 0) {
+        return shape.fail(
+          atInstruction(list, index),
+          `the stack is ${after} deep at ${terminator} and must be empty`,
+        );
+      }
+      continue;
+    }
+    for (const position of targetPositions(opcode)) {
+      const target = operands[position];
+      if (typeof target === 'number' && !reach(target, after, index)) return false;
+    }
+    if (opcode !== 'JUMP' && opcode !== 'RET' && opcode !== 'HALT') {
+      if (!reach(index + 1, after, index)) return false;
+    }
+  }
+
+  return true;
+}
+
+/**
+ * Check 7: every channel declared `once` is written exactly once on every path
+ * from instruction 0 to `HALT`.
+ *
+ * That is how a plot column is guaranteed a value, or an explicit absence, for
+ * every bar. The counts are capped at two, because "more than once" is the whole
+ * of what the check needs to know and a loop would otherwise never settle.
+ */
+export function checkOnce(
+  shape: ShapeCheck,
+  code: readonly Instruction[],
+  once: readonly boolean[],
+): boolean {
+  const channels = once.length;
+  if (channels === 0) return true;
+  const cap = 2;
+  const least: (number[] | undefined)[] = new Array<number[] | undefined>(code.length).fill(
+    undefined,
+  );
+  const most: (number[] | undefined)[] = new Array<number[] | undefined>(code.length).fill(
+    undefined,
+  );
+  const pending: number[] = [0];
+  least[0] = new Array<number>(channels).fill(0);
+  most[0] = new Array<number>(channels).fill(0);
+
+  const merge = (index: number, low: readonly number[], high: readonly number[]): void => {
+    if (index < 0 || index >= code.length) return;
+    const knownLow = least[index];
+    const knownHigh = most[index];
+    if (knownLow === undefined || knownHigh === undefined) {
+      least[index] = [...low];
+      most[index] = [...high];
+      pending.push(index);
+      return;
+    }
+    let changed = false;
+    for (let c = 0; c < channels; c += 1) {
+      const lower = Math.min(knownLow[c] ?? 0, low[c] ?? 0);
+      const upper = Math.min(cap, Math.max(knownHigh[c] ?? 0, high[c] ?? 0));
+      if (lower !== knownLow[c]) {
+        knownLow[c] = lower;
+        changed = true;
+      }
+      if (upper !== knownHigh[c]) {
+        knownHigh[c] = upper;
+        changed = true;
+      }
+    }
+    if (changed) pending.push(index);
+  };
+
+  let end = -1;
+  while (pending.length > 0) {
+    const index = pending.pop();
+    if (index === undefined) continue;
+    const instruction = code[index];
+    const low = least[index];
+    const high = most[index];
+    if (instruction === undefined || low === undefined || high === undefined) continue;
+    const opcode = instruction[0];
+    if (opcode === 'HALT') {
+      end = index;
+      continue;
+    }
+    const nextLow = [...low];
+    const nextHigh = [...high];
+    if (opcode === 'EMIT') {
+      const channel = instruction[1] ?? 0;
+      nextLow[channel] = Math.min(cap, (nextLow[channel] ?? 0) + 1);
+      nextHigh[channel] = Math.min(cap, (nextHigh[channel] ?? 0) + 1);
+    }
+    for (const position of targetPositions(opcode as Opcode)) {
+      const target = instruction[position + 1];
+      if (typeof target === 'number') merge(target, nextLow, nextHigh);
+    }
+    if (opcode !== 'JUMP' && opcode !== 'RET') merge(index + 1, nextLow, nextHigh);
+  }
+
+  if (end < 0) return shape.fail('code', 'no path reaches HALT');
+  const low = least[end] ?? [];
+  const high = most[end] ?? [];
+  for (let c = 0; c < channels; c += 1) {
+    if (!once[c]) continue;
+    if (low[c] !== 1 || high[c] !== 1) {
+      return shape.fail(
+        `channels[${c}]`,
+        `it is declared once and is written ${low[c] ?? 0} to ${high[c] ?? 0} times on a path to HALT`,
+      );
+    }
+  }
+  return true;
+}

package/src/core/engine/verify-requests.ts

@@ -0,0 +1,255 @@
+/**
+ * Check 1 of `compiled-program.md` 3.5 over `requests` and the bodies inside
+ * it.
+ *
+ * A read's body holds the tables a program holds for the machine of section 3,
+ * counted from zero, so it needs the same walk the program's own tables get and
+ * for the same reason: every later step indexes into them, and an index out of
+ * range that is not caught here is read past the end of an array during a bar.
+ *
+ * **A body that was not walked is the hole the whole verification exists to
+ * close.** It is reached by the same interpreter, with the same instruction
+ * list, over another instrument's history. Verifying the program and not the
+ * expression it carries would leave exactly the failure 3.5 is written to
+ * prevent, one level down where nobody would look for it.
+ */
+import type { ShapeCheck } from './verify-shape.js';
+
+const READS = ['timeframe', 'symbol'] as const;
+const MODES = ['confirmed', 'developing', 'lookahead'] as const;
+const CHART_FACTS = ['symbol', 'exchange', 'interval'] as const;
+
+/** The tables a machine indexes, wherever they sit in the program. */
+export interface MachineTables {
+  readonly series: readonly unknown[];
+  readonly cells: readonly unknown[];
+  readonly states: readonly unknown[];
+  readonly functions: readonly unknown[];
+  readonly callSites: readonly unknown[];
+  readonly loops: readonly unknown[];
+  readonly slots: number;
+  readonly libFunctions: number;
+}
+
+// The five the format declares (compiled-program.md section 2, series[].kind).
+// A verifier that listed only the kinds an engine happens to execute would
+// refuse a well-formed program with a message blaming the compiler that wrote
+// it, and would pre-empt the capability refusal that exists to say which
+// feature is missing. Verification answers "is this a valid program", and the
+// capability check answers "can I run it".
+const REGISTER_KINDS = ['bar', 'computed', 'argument', 'request', 'input'] as const;
+const CELL_KINDS = ['var', 'live'] as const;
+const LOOP_KINDS = ['for', 'forIn', 'while'] as const;
+
+/**
+ * Every table one machine indexes, and every index into one of them.
+ *
+ * Shared by the program and by every request body rather than written twice,
+ * because two copies of this walk would be two places for a table to gain a
+ * field and only one of them to learn about it.
+ */
+export function checkMachineTables(
+  shape: ShapeCheck,
+  prefix: string,
+  tables: MachineTables,
+): boolean {
+  for (let i = 0; i < tables.series.length; i += 1) {
+    const register = tables.series[i];
+    const path = `${prefix}series[${i}]`;
+    if (!shape.object(register, path)) return false;
+    if (register['id'] !== i) return shape.fail(`${path}.id`, 'a register id is its position');
+    if (!shape.one(register['kind'], `${path}.kind`, REGISTER_KINDS)) return false;
+    if (register['kind'] === 'bar' && !shape.string(register['field'], `${path}.field`)) {
+      return false;
+    }
+  }
+
+  for (let i = 0; i < tables.cells.length; i += 1) {
+    const cell = tables.cells[i];
+    const path = `${prefix}cells[${i}]`;
+    if (!shape.object(cell, path)) return false;
+    if (!shape.one(cell['kind'], `${path}.kind`, CELL_KINDS)) return false;
+  }
+
+  for (let i = 0; i < tables.states.length; i += 1) {
+    const state = tables.states[i];
+    const path = `${prefix}states[${i}]`;
+    if (!shape.object(state, path)) return false;
+    if (!shape.index(state['fn'], `${path}.fn`, tables.libFunctions, 'lib.functions')) return false;
+  }
+
+  for (let i = 0; i < tables.functions.length; i += 1) {
+    const one = tables.functions[i];
+    const path = `${prefix}functions[${i}]`;
+    if (!shape.object(one, path)) return false;
+    if (!shape.string(one['name'], `${path}.name`)) return false;
+    if (!shape.whole(one['params'], `${path}.params`)) return false;
+    if (!shape.whole(one['slots'], `${path}.slots`)) return false;
+    if (!shape.array(one['code'], `${path}.code`)) return false;
+    if (Number(one['params']) > Number(one['slots'])) {
+      return shape.fail(`${path}.slots`, 'a frame cannot be smaller than its parameter list');
+    }
+  }
+
+  for (let i = 0; i < tables.callSites.length; i += 1) {
+    const site = tables.callSites[i];
+    const path = `${prefix}callSites[${i}]`;
+    if (!shape.object(site, path)) return false;
+    if (!shape.index(site['fn'], `${path}.fn`, tables.functions.length, 'functions')) return false;
+    if (!shape.whole(site['argc'], `${path}.argc`)) return false;
+    if (!shape.whole(site['cellBase'], `${path}.cellBase`)) return false;
+    if (!shape.whole(site['stateBase'], `${path}.stateBase`)) return false;
+    if (!shape.array(site['series'], `${path}.series`)) return false;
+    const target = tables.functions[Number(site['fn'])] as Readonly<Record<string, unknown>>;
+    if (site['argc'] !== target['params']) {
+      return shape.fail(`${path}.argc`, "a site passes exactly the function's parameter count");
+    }
+    for (let k = 0; k < site['series'].length; k += 1) {
+      const bound = site['series'][k];
+      if (bound === -1) continue;
+      if (!shape.index(bound, `${path}.series[${k}]`, tables.series.length, 'series')) return false;
+    }
+  }
+
+  for (let i = 0; i < tables.loops.length; i += 1) {
+    const loop = tables.loops[i];
+    const path = `${prefix}loops[${i}]`;
+    if (!shape.object(loop, path)) return false;
+    if (!shape.one(loop['kind'], `${path}.kind`, LOOP_KINDS)) return false;
+    if (!shape.whole(loop['line'], `${path}.line`)) return false;
+    if (!shape.whole(loop['col'], `${path}.col`)) return false;
+  }
+
+  return true;
+}
+
+/**
+ * `requests` and every body inside it, 2.16.
+ *
+ * `series` is the register of the **enclosing** machine that the read's value
+ * lands in, so it is checked against the table the caller passes; everything
+ * inside `body` is the body's own and is checked against the body's.
+ */
+export function checkRequests(
+  shape: ShapeCheck,
+  prefix: string,
+  requests: readonly unknown[],
+  registers: number,
+  libFunctions: number,
+  keys: ReadonlySet<string>,
+): boolean {
+  const seen = new Set<number>();
+  return walk(shape, prefix, requests, registers, libFunctions, keys, seen);
+}
+
+function walk(
+  shape: ShapeCheck,
+  prefix: string,
+  requests: readonly unknown[],
+  registers: number,
+  libFunctions: number,
+  keys: ReadonlySet<string>,
+  seen: Set<number>,
+): boolean {
+  for (let i = 0; i < requests.length; i += 1) {
+    const request = requests[i];
+    const path = `${prefix}requests[${i}]`;
+    if (!shape.object(request, path)) return false;
+    if (!shape.whole(request['id'], `${path}.id`)) return false;
+    // The id is this read's handle and is what `req.isReady` is handed, so two
+    // reads sharing one would answer each other's question.
+    if (seen.has(request['id'])) {
+      return shape.fail(`${path}.id`, `${request['id']} is already another read's handle`);
+    }
+    seen.add(request['id']);
+    if (!shape.one(request['read'], `${path}.read`, READS)) return false;
+    if (!shape.one(request['mode'], `${path}.mode`, MODES)) return false;
+    if (!shape.index(request['series'], `${path}.series`, registers, 'series')) return false;
+    if (request['warmup'] !== null && !shape.whole(request['warmup'], `${path}.warmup`)) {
+      return false;
+    }
+    for (const field of ['symbol', 'exchange', 'timeframe']) {
+      if (!checkIdentity(shape, request[field], `${path}.${field}`, keys)) return false;
+    }
+    if (!checkBody(shape, `${path}.body.`, request['body'], libFunctions, keys, seen)) return false;
+  }
+  return true;
+}
+
+function checkBody(
+  shape: ShapeCheck,
+  prefix: string,
+  raw: unknown,
+  libFunctions: number,
+  keys: ReadonlySet<string>,
+  seen: Set<number>,
+): boolean {
+  const at = prefix.slice(0, -1);
+  if (!shape.object(raw, at)) return false;
+  for (const name of ['inputs', 'series', 'cells', 'states', 'functions', 'callSites', 'loops',
+    'requests', 'code', 'pos', 'fnPos']) {
+    if (!shape.array(raw[name], `${prefix}${name}`)) return false;
+  }
+  if (!shape.object(raw['frame'], `${prefix}frame`)) return false;
+  if (!shape.whole(raw['frame']['slots'], `${prefix}frame.slots`)) return false;
+
+  const series = raw['series'] as readonly unknown[];
+  if (!checkMachineTables(shape, prefix, {
+    series,
+    cells: raw['cells'] as readonly unknown[],
+    states: raw['states'] as readonly unknown[],
+    functions: raw['functions'] as readonly unknown[],
+    callSites: raw['callSites'] as readonly unknown[],
+    loops: raw['loops'] as readonly unknown[],
+    slots: Number(raw['frame']['slots']),
+    libFunctions,
+  })) {
+    return false;
+  }
+
+  const inputs = raw['inputs'] as readonly unknown[];
+  for (let i = 0; i < inputs.length; i += 1) {
+    const input = inputs[i];
+    const path = `${prefix}inputs[${i}]`;
+    if (!shape.object(input, path)) return false;
+    if (!shape.string(input['input'], `${path}.input`)) return false;
+    if (!keys.has(input['input'])) {
+      return shape.fail(path, `it names the input ${input['input']}, which inputs[] does not declare`);
+    }
+    if (!shape.index(input['series'], `${path}.series`, series.length, 'the body\'s series')) {
+      return false;
+    }
+  }
+
+  return walk(shape, prefix, raw['requests'] as readonly unknown[], series.length, libFunctions,
+    keys, seen);
+}
+
+/**
+ * One of the three forms a request's identity field takes, 2.16.
+ *
+ * A value, the setting that supplies it, or one of the three chart facts that
+ * identify a chart rather than describe it. The rest of the `chart` namespace
+ * describes the chart, and a request built from one of those would name nothing.
+ */
+function checkIdentity(
+  shape: ShapeCheck,
+  value: unknown,
+  path: string,
+  keys: ReadonlySet<string>,
+): boolean {
+  if (value === null || typeof value === 'string') return true;
+  if (!shape.object(value, path)) return false;
+  const named = value['input'];
+  if (named !== undefined) {
+    if (!shape.string(named, `${path}.input`)) return false;
+    if (!keys.has(named)) {
+      return shape.fail(path, `it names the input ${named}, which inputs[] does not declare`);
+    }
+    return true;
+  }
+  if (value['chart'] === undefined) {
+    return shape.fail(path, 'an object here names an input or a chart fact');
+  }
+  return shape.one(value['chart'], `${path}.chart`, CHART_FACTS);
+}