@solidxai/core 0.1.2 → 0.1.5-beta.0

package/src/testing/contracts/testing-metadata.types.ts

@@ -0,0 +1,46 @@
+export type ScenarioType = "api" | "ui" | "mixed";
+
+export interface TestingDataRecord {
+  modelUserKey: string;
+  recUserKeyValue: string;
+  data: Record<string, any>;
+}
+
+export interface TestingMetadata {
+  testing: {
+    specs?: string[];
+    data?: TestingDataRecord[];
+    scenarios: ScenarioSpec[];
+  };
+}
+
+export interface ScenarioSpec {
+  id: string;
+  name?: string;
+  type: ScenarioType;
+  params?: Record<string, any>;
+  tags?: string[];
+  timeoutMs?: number;
+  retries?: number;
+  steps: StepBlock[];
+}
+
+/**
+ * A step can be written in a phase block (Given/When/Then/And) or as a flat op step.
+ */
+export type StepBlock =
+  | { given: OpStep }
+  | { when: OpStep }
+  | { then: OpStep | OpStep[] }
+  | { and: OpStep }
+  | OpStep;
+
+export interface OpStep {
+  op: string;
+  with?: Record<string, any>;
+  saveAs?: string;
+  name?: string;
+  // spec is used by op "test.spec" to point to a registered custom spec implementation.
+  spec?: string;
+  timeoutMs?: number;
+}

package/src/testing/core/interpolation.ts

@@ -0,0 +1,189 @@
+import type { TestContext } from "../contracts/runtime-context.types";
+
+const TOKEN_REGEX = /\$\{([^}]+)\}/g;
+
+function getByPath(obj: Record<string, any>, path: string): unknown {
+  if (!path) return undefined;
+  const parts = path.split(".");
+  let current: any = obj;
+  for (const part of parts) {
+    if (current == null || typeof current !== "object") return undefined;
+    current = current[part];
+  }
+  return current;
+}
+
+function parsePathSegments(path: string): string[] {
+  const segments: string[] = [];
+  let buffer = "";
+  let i = 0;
+
+  const pushBuffer = () => {
+    if (buffer) {
+      segments.push(buffer);
+      buffer = "";
+    }
+  };
+
+  while (i < path.length) {
+    const ch = path[i];
+    if (ch === ".") {
+      pushBuffer();
+      i += 1;
+      continue;
+    }
+    if (ch === "[") {
+      pushBuffer();
+      i += 1;
+      if (i >= path.length) break;
+      let quote = "";
+      if (path[i] === '"' || path[i] === "'") {
+        quote = path[i];
+        i += 1;
+      }
+      let value = "";
+      while (i < path.length) {
+        const c = path[i];
+        if (quote) {
+          if (c === "\\" && i + 1 < path.length) {
+            value += path[i + 1];
+            i += 2;
+            continue;
+          }
+          if (c === quote) {
+            i += 1;
+            break;
+          }
+          value += c;
+          i += 1;
+          continue;
+        }
+        if (c === "]") break;
+        value += c;
+        i += 1;
+      }
+      while (i < path.length && path[i] !== "]") {
+        i += 1;
+      }
+      if (i < path.length && path[i] === "]") {
+        i += 1;
+      }
+      if (value) {
+        segments.push(value);
+      }
+      continue;
+    }
+    buffer += ch;
+    i += 1;
+  }
+
+  pushBuffer();
+  return segments;
+}
+
+function getByPathWithBrackets(obj: Record<string, any>, path: string): unknown {
+  if (!path) return undefined;
+  const parts = parsePathSegments(path);
+  let current: any = obj;
+  for (const part of parts) {
+    if (current == null || typeof current !== "object") return undefined;
+    current = current[part];
+  }
+  return current;
+}
+
+type TokenResolution = { value: unknown; raw: boolean };
+
+function resolveToken(token: string, ctx: TestContext): TokenResolution {
+  if (token.startsWith("env:")) {
+    const name = token.slice("env:".length);
+    if (!name) {
+      throw new Error('Invalid interpolation token: "env:"');
+    }
+    const value = process.env[name];
+    if (value === undefined) {
+      throw new Error(`Missing env var for token: "${token}"`);
+    }
+    return { value, raw: false };
+  }
+
+  if (token.startsWith("params.")) {
+    const path = token.slice("params.".length);
+    if (!path) {
+      throw new Error('Invalid interpolation token: "params."');
+    }
+    const value = getByPath(ctx.params, path);
+    if (value === undefined) {
+      throw new Error(`Missing param for token: "${token}"`);
+    }
+    return { value, raw: false };
+  }
+
+  if (token.startsWith("data:") || token.startsWith("data.")) {
+    const prefix = token.startsWith("data:") ? "data:" : "data.";
+    let path = token.slice(prefix.length);
+    if (!path) {
+      throw new Error(`Invalid interpolation token: "${prefix}"`);
+    }
+    let raw = false;
+    if (path.endsWith("._rec")) {
+      raw = true;
+      path = path.slice(0, -("._rec".length));
+    }
+    const value = getByPathWithBrackets((ctx as any).testData ?? {}, path);
+    if (value === undefined) {
+      throw new Error(`Missing test data for token: "${token}"`);
+    }
+    return { value, raw };
+  }
+
+  if (token.startsWith("res:")) {
+    const path = token.slice("res:".length);
+    if (!path) {
+      throw new Error('Invalid interpolation token: "res:"');
+    }
+    const value = ctx.resources.get(path);
+    if (value === undefined) {
+      throw new Error(`Missing resource for token: "${token}"`);
+    }
+    return { value, raw: false };
+  }
+
+  throw new Error(`Unknown interpolation token: "${token}"`);
+}
+
+export function interpolateString(input: string, ctx: TestContext): string {
+  return input.replace(TOKEN_REGEX, (_match, token: string) => {
+    const resolved = resolveToken(token, ctx);
+    return typeof resolved.value === "string"
+      ? resolved.value
+      : JSON.stringify(resolved.value);
+  });
+}
+
+export function interpolateDeep<T>(input: T, ctx: TestContext): T {
+  if (typeof input === "string") {
+    const tokenMatch = input.match(/^\$\{([^}]+)\}$/);
+    if (tokenMatch) {
+      const resolved = resolveToken(tokenMatch[1], ctx);
+      if (resolved.raw) {
+        return resolved.value as T;
+      }
+    }
+    return interpolateString(input, ctx) as T;
+  }
+
+  if (Array.isArray(input)) {
+    return input.map((item) => interpolateDeep(item, ctx)) as T;
+  }
+
+  if (input && typeof input === "object") {
+    const out: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(input as Record<string, any>)) {
+      out[key] = interpolateDeep(value, ctx);
+    }
+    return out as T;
+  }
+
+  return input;
+}

package/src/testing/core/normalize-steps.ts

@@ -0,0 +1,21 @@
+import type { StepPhase } from "../contracts/runtime-context.types";
+import type { OpStep, StepBlock } from "../contracts/testing-metadata.types";
+
+export function normalizeBlock(
+  block: StepBlock,
+): { phase: StepPhase; steps: OpStep[] } {
+  if ("given" in block) {
+    return { phase: "given", steps: [block.given] };
+  }
+  if ("when" in block) {
+    return { phase: "when", steps: [block.when] };
+  }
+  if ("then" in block) {
+    const steps = Array.isArray(block.then) ? block.then : [block.then];
+    return { phase: "then", steps };
+  }
+  if ("and" in block) {
+    return { phase: "and", steps: [block.and] };
+  }
+  return { phase: "step", steps: [block] };
+}

package/src/testing/core/resource-store.ts

@@ -0,0 +1,38 @@
+import type { ResourceStore } from "../contracts/runtime-context.types";
+
+export class SimpleResourceStore implements ResourceStore {
+  private readonly data: Record<string, any> = {};
+
+  get(path: string): unknown {
+    if (!path) return undefined;
+    const parts = path.split(".");
+    let current: any = this.data;
+    for (const part of parts) {
+      if (current == null || typeof current !== "object") return undefined;
+      current = current[part];
+    }
+    return current;
+  }
+
+  set(path: string, value: unknown): void {
+    if (!path) return;
+    const parts = path.split(".");
+    let current: any = this.data;
+    for (let i = 0; i < parts.length - 1; i += 1) {
+      const part = parts[i];
+      if (
+        current[part] == null ||
+        typeof current[part] !== "object" ||
+        Array.isArray(current[part])
+      ) {
+        current[part] = {};
+      }
+      current = current[part];
+    }
+    current[parts[parts.length - 1]] = value;
+  }
+
+  has(path: string): boolean {
+    return this.get(path) !== undefined;
+  }
+}

package/src/testing/core/spec-registry.ts

@@ -0,0 +1,33 @@
+import type { ISolidTestSpec } from "../contracts/test-spec.types";
+
+export type SpecFactory = () => ISolidTestSpec;
+
+export class SpecRegistry {
+  private readonly factories = new Map<string, SpecFactory>();
+
+  register(specId: string, factory: SpecFactory): void {
+    if (!specId) {
+      throw new Error("specId is required");
+    }
+    if (this.factories.has(specId)) {
+      throw new Error(`Spec already registered: "${specId}"`);
+    }
+    this.factories.set(specId, factory);
+  }
+
+  create(specId: string): ISolidTestSpec {
+    const factory = this.factories.get(specId);
+    if (!factory) {
+      throw new Error(`Spec not registered: "${specId}"`);
+    }
+    return factory();
+  }
+
+  has(specId: string): boolean {
+    return this.factories.has(specId);
+  }
+
+  list(): string[] {
+    return Array.from(this.factories.keys()).sort();
+  }
+}

package/src/testing/core/step-registry.ts

@@ -0,0 +1,27 @@
+import type { TestContext } from "../contracts/runtime-context.types";
+import type { OpStep } from "../contracts/testing-metadata.types";
+
+export type StepHandler = (ctx: TestContext, step: OpStep) => Promise<any>;
+
+export class StepRegistry {
+  private readonly handlers = new Map<string, StepHandler>();
+
+  register(op: string, handler: StepHandler): void {
+    if (this.handlers.has(op)) {
+      throw new Error(`Step handler already registered for op: "${op}"`);
+    }
+    this.handlers.set(op, handler);
+  }
+
+  get(op: string): StepHandler {
+    const handler = this.handlers.get(op);
+    if (!handler) {
+      throw new Error(`No step handler registered for op: "${op}"`);
+    }
+    return handler;
+  }
+
+  has(op: string): boolean {
+    return this.handlers.has(op);
+  }
+}

package/src/testing/core/testing-engine.ts

@@ -0,0 +1,127 @@
+import type { TestContext, StepPhase } from "../contracts/runtime-context.types";
+import type { OpStep, ScenarioSpec } from "../contracts/testing-metadata.types";
+import { interpolateDeep } from "./interpolation";
+import { normalizeBlock } from "./normalize-steps";
+import { StepRegistry } from "./step-registry";
+import { withTimeout } from "./timeout";
+
+export class TestingEngine {
+  private readonly registry: StepRegistry;
+  private readonly defaults?: { timeoutMs?: number; retries?: number };
+
+  constructor(
+    registry: StepRegistry,
+    defaults?: { timeoutMs?: number; retries?: number },
+  ) {
+    this.registry = registry;
+    this.defaults = defaults;
+  }
+
+  async runScenario(
+    scenario: ScenarioSpec,
+    ctxBase: Omit<TestContext, "scenarioId" | "scenarioType" | "params">,
+  ): Promise<void> {
+    const retries = scenario.retries ?? this.defaults?.retries ?? 0;
+    const maxAttempts = Math.max(0, retries) + 1;
+    const scenarioTimeoutMs =
+      scenario.timeoutMs ?? this.defaults?.timeoutMs;
+
+    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+      const ctx: TestContext = {
+        ...ctxBase,
+        scenarioId: scenario.id,
+        scenarioType: scenario.type,
+        params: scenario.params ?? {},
+      };
+
+      const reporter = ctx.reporter;
+      const scenarioStart = Date.now();
+      let scenarioError: unknown;
+
+      reporter.onScenarioStart(scenario);
+
+      try {
+        const execute = async () => {
+          for (const block of scenario.steps) {
+            const normalized = normalizeBlock(block);
+            await this.runBlock(normalized.phase, normalized.steps, ctx);
+          }
+        };
+
+        if (scenarioTimeoutMs !== undefined) {
+          await withTimeout(
+            execute(),
+            scenarioTimeoutMs,
+            `Scenario timeout after ${scenarioTimeoutMs}ms: "${scenario.id}"`,
+          );
+        } else {
+          await execute();
+        }
+      } catch (err) {
+        scenarioError = err;
+      } finally {
+        const durationMs = Date.now() - scenarioStart;
+        reporter.onScenarioEnd(scenario, {
+          ok: !scenarioError,
+          error: scenarioError,
+          durationMs,
+        });
+      }
+
+      if (!scenarioError) {
+        return;
+      }
+
+      if (attempt >= maxAttempts) {
+        throw scenarioError;
+      }
+    }
+  }
+
+  private async runBlock(
+    phase: StepPhase,
+    steps: OpStep[],
+    ctx: TestContext,
+  ): Promise<void> {
+    for (const step of steps) {
+      const reporter = ctx.reporter;
+      const stepStart = Date.now();
+      let stepError: unknown;
+      let resolvedStep: OpStep | undefined;
+
+      reporter.onStepStart({ scenarioId: ctx.scenarioId, phase, step });
+
+      try {
+        resolvedStep = interpolateDeep(step, ctx) as OpStep;
+        const handler = this.registry.get(resolvedStep.op);
+        const run = handler(ctx, resolvedStep);
+        const result =
+          resolvedStep.timeoutMs !== undefined
+            ? await withTimeout(
+              run,
+              resolvedStep.timeoutMs,
+              `Step timeout after ${resolvedStep.timeoutMs}ms: "${resolvedStep.op}"`,
+            )
+            : await run;
+
+        if (resolvedStep.saveAs) {
+          // console.log(`Step ${resolvedStep.name} attempting to saveAs ${resolvedStep.saveAs}`, JSON.stringify(result));
+          ctx.resources.set(resolvedStep.saveAs, result);
+        }
+      } catch (err) {
+        stepError = err;
+        throw err;
+      } finally {
+        const durationMs = Date.now() - stepStart;
+        reporter.onStepEnd({
+          scenarioId: ctx.scenarioId,
+          phase,
+          step: resolvedStep ?? step,
+          ok: !stepError,
+          error: stepError,
+          durationMs,
+        });
+      }
+    }
+  }
+}

package/src/testing/core/timeout.ts

@@ -0,0 +1,19 @@
+export async function withTimeout<T>(
+  promise: Promise<T>,
+  ms: number,
+  message: string,
+): Promise<T> {
+  let timeoutId: ReturnType<typeof setTimeout> | undefined;
+
+  const timeoutPromise = new Promise<never>((_resolve, reject) => {
+    timeoutId = setTimeout(() => reject(new Error(message)), ms);
+  });
+
+  try {
+    return await Promise.race([promise, timeoutPromise]);
+  } finally {
+    if (timeoutId !== undefined) {
+      clearTimeout(timeoutId);
+    }
+  }
+}

package/src/testing/reporter/attachments.ts

@@ -0,0 +1,25 @@
+import type { TestContext } from "../contracts/runtime-context.types";
+
+export function attachJson(
+  ctx: TestContext,
+  name: string,
+  obj: unknown,
+): void {
+  if (!ctx.reporter.attach) return;
+  ctx.reporter.attach({
+    scenarioId: ctx.scenarioId,
+    name,
+    contentType: "application/json",
+    data: JSON.stringify(obj, null, 2),
+  });
+}
+
+export function attachText(ctx: TestContext, name: string, text: string): void {
+  if (!ctx.reporter.attach) return;
+  ctx.reporter.attach({
+    scenarioId: ctx.scenarioId,
+    name,
+    contentType: "text/plain",
+    data: text,
+  });
+}