@solidxai/core 0.1.2 → 0.1.5-beta.0

package/src/testing/README.md

@@ -0,0 +1,364 @@
+# Testing Module
+
+## Folder Structure
+- `contracts/`: metadata + runtime context types
+- `core/`: engine, registry, interpolation, utilities
+- `adapters/`: API (axios) and UI (Playwright) adapters
+- `steps/`: step implementations grouped by domain
+- `reporter/`: reporting interfaces + console reporter
+- `runner/`: lifecycle helpers + metadata runner
+
+## Metadata Shape
+`TestingMetadata` lives under `testing` in module metadata JSON files.
+
+```json
+{
+  "testing": {
+    "specs": ["path/to/register-test-specs.js"],
+    "data": [
+      {
+        "modelUserKey": "stateMaster",
+        "recUserKeyValue": "Maharashtra",
+        "data": {
+          "name": "Maharashtra",
+          "description": "State of Maharashtra"
+        }
+      }
+    ],
+    "scenarios": [
+      {
+        "id": "api-authenticate-success",
+        "name": "Authenticate succeeds",
+        "type": "api",
+        "params": {
+          "username": "alice"
+        },
+        "tags": ["smoke"],
+        "timeoutMs": 30000,
+        "retries": 1,
+        "steps": [
+          { "given": { "op": "api.request", "with": { "method": "POST", "url": "..." } } }
+        ]
+      }
+    ]
+  }
+}
+```
+
+## Step Shape
+Steps can be written as Given/When/Then blocks or as flat ops.
+
+```ts
+{
+  id: "login-happy-path",
+  type: "ui",
+  steps: [
+    { given: { op: "ui.goto", with: { url: "/login" } } },
+    { when: { op: "ui.fill", with: { selector: "#user", value: "alice" } } },
+    { and: { op: "ui.fill", with: { selector: "#pass", value: "secret" } } },
+    { then: { op: "ui.click", with: { selector: "button[type=submit]" } } }
+  ]
+}
+```
+
+General step fields:
+- `op` (required): operation name
+- `with` (optional): op-specific options
+- `saveAs` (optional): store the return value in the resource store (read via `${res:...}`)
+- `name` (optional): label for reporting
+- `timeoutMs` (optional): per-step timeout override
+- `spec` (optional): used by `test.spec` to identify the spec
+
+## Interpolation
+Available tokens:
+- `${env:NAME}` (environment variables)
+- `${params.foo}` (scenario params)
+- `${res:saveAs.path}` (saved step results)
+- `${data:modelUserKey["recUserKeyValue"].field}` (test data from `testing.data`)
+- `${data:modelUserKey["recUserKeyValue"]._rec}` (raw record object when used alone)
+
+Test data lookup details:
+- Data is indexed as `data:<modelUserKey>["<recUserKeyValue>"]`.
+- You can access fields with `.fieldName`.
+- Bracket syntax is recommended for keys with spaces or punctuation.
+- Use `._rec` to return the full object when the token is the entire value.
+- `data.` is still supported for backward compatibility, but `data:` is preferred.
+
+Examples:
+```json
+{
+  "params": {
+    "state": "${data:stateMaster["Maharashtra"].name}"
+  },
+  "steps": [
+    {
+      "given": {
+        "op": "api.request",
+        "with": {
+          "method": "POST",
+          "url": "${env:API_BASE_URL}/api/example",
+          "json": {
+            "stateName": "${params.state}",
+            "city": "${data:cityMaster["New Delhi"].name}",
+            "cityRecord": "${data:cityMaster["New Delhi"]._rec}"
+          }
+        }
+      }
+    }
+  ]
+}
+```
+
+## Ops Reference
+
+### **Op: `api.request`**
+Description: Executes an HTTP request using the API adapter.
+
+Options in `with`:
+- `method` (required)
+- `url` (required)
+- `headers` (optional)
+- `json` (optional)
+- `bodyText` (optional)
+- `query` (optional, object or querystring)
+- `formData` (optional, array or object)
+- `body` (optional alias for `formData`)
+
+Returns:
+- `status`
+- `headers`
+- `bodyText`
+- `bodyJson` (when JSON is detected)
+- `body` (alias for `bodyJson ?? bodyText`)
+
+Notes:
+- Sets `ctx.last.apiResponse` for `assert.httpStatus`.
+- `formData` can be an array of items or a plain object.
+- If `formData` is an object, keys become field names and values become field values.
+- File values can be `file:/absolute/path` or `url:https://...` and will be uploaded as files.
+- Non-string object values are JSON-stringified before sending.
+
+Form data item notes:
+- Array form items: `{ name, value, type?: "text" | "file", filename?, contentType? }`.
+- File values can be `file:/absolute/path` or `url:https://...`.
+- If `formData` is an object, keys become field names and values become field values.
+- Use `formData`/`body` instead of `json` or `bodyText`.
+
+Example (array form):
+```json
+{
+  "when": {
+    "op": "api.request",
+    "with": {
+      "method": "POST",
+      "url": "${env:API_BASE_URL}/api/lead",
+      "formData": [
+        { "name": "venueUserKey", "value": "Kolhapur-Karvir" },
+        { "name": "coupon", "type": "file", "value": "file:/abs/path/image.jpg" },
+        { "name": "campaignUserKey", "value": "Campaign: Drawing Competition" }
+      ]
+    }
+  }
+}
+```
+
+Example (object form from `testing.data`):
+```json
+{
+  "when": {
+    "op": "api.request",
+    "with": {
+      "method": "POST",
+      "url": "${env:API_BASE_URL}/api/lead",
+      "formData": "${data:lead[\"LeadWithFile\"]._rec}"
+    }
+  }
+}
+```
+
+### **Op: `api.auth.bearerFromLogin`**
+Description: Logs in and returns a bearer access token from the response.
+
+Options in `with`:
+- `url` (required)
+- `username` (required)
+- `password` (required)
+
+Returns:
+- access token string
+
+### **Op: `ui.goto`**
+Description: Navigates the browser to a URL.
+
+Options in `with`:
+- `url` (required)
+
+### **Op: `ui.expectUrl`**
+Description: Asserts the current page URL.
+
+Options in `with`:
+- `equals` (optional)
+- `contains` (optional)
+
+### **Op: `ui.fill`**
+Description: Fills an input or editable element.
+
+Options in `with`:
+- `selector` (required)
+- `value` (required)
+
+### **Op: `ui.select`**
+Description: Selects an option in a select element.
+
+Options in `with`:
+- `selector` (required)
+- `value` (required)
+
+### **Op: `ui.click`**
+Description: Clicks an element located by selector.
+
+Options in `with`:
+- `selector` (required)
+
+### **Op: `ui.press`**
+Description: Presses a keyboard key on a focused element.
+
+Options in `with`:
+- `selector` (required)
+- `key` (required)
+
+### **Op: `ui.expectVisible`**
+Description: Waits for an element to be visible.
+
+Options in `with`:
+- `selector` (required)
+
+### **Op: `ui.expectText`**
+Description: Asserts the text content of an element.
+
+Options in `with`:
+- `selector` (required)
+- `equals` (optional)
+- `contains` (optional)
+
+### **Op: `assert.equals`**
+Description: Asserts strict equality between two values.
+
+Options in `with`:
+- `actual` (required)
+- `expected` (required)
+
+### **Op: `assert.contains`**
+Description: Asserts a string contains a substring.
+
+Options in `with`:
+- `actual` (required)
+- `expected` (required)
+
+### **Op: `assert.matches`**
+Description: Asserts a string matches a regex pattern.
+
+Options in `with`:
+- `actual` (required)
+- `pattern` (required)
+
+### **Op: `assert.httpStatus`**
+Description: Asserts the HTTP status of the last API response (or a provided response).
+
+Options in `with`:
+- `is` (required)
+- `from` (optional, defaults to `ctx.last.apiResponse`)
+
+### **Op: `assert.jsonPath`**
+Description: Asserts a JSONPath-resolved value equals an expected value.
+
+Options in `with`:
+- `from` (required)
+- `path` (required)
+- `equals` (required)
+
+### **Op: `util.log`**
+Description: Logs a message (and optional data) to the console.
+
+Options in `with`:
+- `message` (required)
+- `data` (optional)
+
+### **Op: `util.sleep`**
+Description: Pauses execution for a fixed duration.
+
+Options in `with`:
+- `ms` (required)
+
+### **Op: `util.require`**
+Description: Fails the step if a required resource is missing.
+
+Options in `with`:
+- `resource` (required)
+- `message` (optional)
+
+### **Op: `test.spec`**
+Description: Runs a custom test spec registered via testing.specs.
+
+Options in `with`:
+- `input` (optional, free-form object passed to the spec)
+- `specId` (optional alternative to `step.spec`)
+
+Step fields:
+- `spec` (required unless `with.specId` is provided)
+
+Returns:
+- `SolidTestSpecResult` (saved if `saveAs` is provided)
+
+## Custom Specs (test.spec)
+Purpose: escape hatch for fully custom test logic.
+
+Step shape example:
+```json
+{
+  "when": {
+    "op": "test.spec",
+    "spec": "example.customHealth",
+    "with": { "input": { "url": "${env:API_BASE_URL}/health" } },
+    "saveAs": "custom.health"
+  }
+}
+```
+
+Register specs in the consuming project:
+```ts
+import { runFromMetadata } from "./runner/run-from-metadata";
+
+await runFromMetadata({
+  // ...
+  specs: (specRegistry) => {
+    specRegistry.register("example.customHealth", () => new CustomHealthSpec());
+  }
+});
+```
+
+When using `solidctl test run`, specs are loaded from `testing.specs` in module metadata.
+Use `--skip-scenario-ids` to exclude scenarios by id (comma-separated).
+Use `--print-api-logs` to print full API request/response details for `api.request` steps.
+Use `--print-api-logs` to print full API request/response details for `api.request` steps.
+
+## Run From Metadata
+```ts
+import { runFromMetadata } from "./runner/run-from-metadata";
+import metadata from "./testing-metadata.json";
+
+await runFromMetadata({
+  metadata,
+  includeTags: ["smoke"],
+  defaults: { timeoutMs: 30_000, retries: 1 },
+  api: { baseUrl: "https://api.example.com" },
+  ui: { baseUrl: "https://app.example.com", headless: true }
+});
+```
+
+## Add A New Step (SOP)
+1. Create a new `*.step.ts` in the right domain folder.
+2. Implement a `registerXSteps(registry)` function and `registry.register("op.name", handler)`.
+3. Validate required `step.with` fields and throw clear errors.
+4. Use adapters via `ctx.api` / `ctx.ui` and update `ctx.last` when helpful.
+5. Export and register it from the domain `index.ts`.

package/src/testing/__examples__/register-example-specs.ts

@@ -0,0 +1,6 @@
+import { SpecRegistry } from "../core/spec-registry";
+import { CustomHealthSpec } from "./specs/custom-health.spec";
+
+export function registerExampleSpecs(specRegistry: SpecRegistry): void {
+  specRegistry.register("example.customHealth", () => new CustomHealthSpec());
+}

package/src/testing/__examples__/specs/custom-health.spec.ts

@@ -0,0 +1,29 @@
+import type { ISolidTestSpec, SolidTestSpecArgs } from "../../contracts/test-spec.types";
+
+export class CustomHealthSpec implements ISolidTestSpec {
+  async run({ ctx, input }: SolidTestSpecArgs) {
+    if (!ctx.api) {
+      throw new Error("Missing API adapter on context for CustomHealthSpec");
+    }
+    const url = input?.url as string | undefined;
+    if (!url) {
+      throw new Error('Missing "url" in input for CustomHealthSpec');
+    }
+
+    const response = await ctx.api.http({ method: "GET", url });
+
+    return {
+      ok: response.status === 200,
+      name: "Custom API health spec",
+      details: { status: response.status },
+      attachments: [
+        {
+          name: "health-response",
+          contentType: "application/json",
+          data: JSON.stringify(response, null, 2),
+          encoding: "utf8" as const,
+        },
+      ],
+    };
+  }
+}

package/src/testing/__examples__/testing.sample.json

@@ -0,0 +1,82 @@
+{
+  "testing": {
+    "scenarios": [
+      {
+        "id": "api-health",
+        "name": "API health check",
+        "type": "api",
+        "steps": [
+          {
+            "given": {
+              "op": "api.request",
+              "with": {
+                "method": "GET",
+                "url": "${env:API_BASE_URL}/health"
+              },
+              "saveAs": "lastHealth"
+            }
+          },
+          {
+            "then": {
+              "op": "assert.httpStatus",
+              "with": {
+                "is": 200
+              }
+            }
+          }
+        ]
+      },
+      {
+        "id": "ui-home",
+        "name": "UI home page",
+        "type": "ui",
+        "steps": [
+          {
+            "given": {
+              "op": "ui.goto",
+              "with": {
+                "url": "${env:UI_BASE_URL}/"
+              }
+            }
+          },
+          {
+            "then": {
+              "op": "ui.expectUrl",
+              "with": {
+                "contains": "/"
+              }
+            }
+          }
+        ]
+      },
+      {
+        "id": "custom-spec-health",
+        "name": "Custom spec health check",
+        "type": "api",
+        "steps": [
+          {
+            "when": {
+              "op": "test.spec",
+              "spec": "example.customHealth",
+              "with": {
+                "input": {
+                  "url": "${env:API_BASE_URL}/health"
+                }
+              },
+              "saveAs": "customHealthResult"
+            }
+          },
+          {
+            "then": {
+              "op": "assert.equals",
+              "with": {
+                "actual": "${res:customHealthResult.ok}",
+                "expected": "true"
+              }
+            }
+          }
+        ]
+      }
+    ]
+  }
+}

package/src/testing/adapters/api/api-adapter.ts

@@ -0,0 +1,85 @@
+import axios from "axios";
+
+import type { ApiResponse } from "../../contracts/runtime-context.types";
+import type { ApiAdapterOptions, ApiRequestOptions } from "./api.types";
+
+function hasHeader(headers: Record<string, string>, name: string): boolean {
+  return Object.prototype.hasOwnProperty.call(headers, name.toLowerCase());
+}
+
+export class ApiAdapter {
+  private readonly baseUrl?: string;
+  private readonly defaultHeaders?: Record<string, string>;
+
+  constructor(opts?: ApiAdapterOptions) {
+    this.baseUrl = opts?.baseUrl;
+    this.defaultHeaders = opts?.defaultHeaders;
+  }
+
+  async http(req: ApiRequestOptions): Promise<ApiResponse> {
+    const headers: Record<string, string> = {
+      ...(this.defaultHeaders ?? {}),
+      ...(req.headers ?? {}),
+    };
+
+    if (req.formData) {
+      const formHeaders = req.formData.getHeaders();
+      for (const [key, value] of Object.entries(formHeaders)) {
+        headers[key.toLowerCase()] = String(value);
+      }
+    } else if (req.json !== undefined && !hasHeader(headers, "content-type")) {
+      headers["content-type"] = "application/json";
+    }
+
+    const response = await axios.request({
+      baseURL: this.baseUrl,
+      url: req.url,
+      method: req.method as any,
+      headers,
+      data: req.formData ?? (req.json !== undefined ? req.json : req.bodyText),
+      responseType: "text",
+      transformResponse: (data) => data,
+      validateStatus: () => true,
+    });
+
+    const bodyText = typeof response.data === "string"
+      ? response.data
+      : String(response.data);
+    const responseHeaders = Object.fromEntries(
+      Object.entries(response.headers ?? {}).map(([key, value]) => [
+        key.toLowerCase(),
+        Array.isArray(value) ? value.join(", ") : String(value),
+      ]),
+    ) as Record<string, string>;
+
+    let bodyJson: unknown;
+    const contentType = responseHeaders["content-type"] ?? "";
+    if (contentType.toLowerCase().includes("application/json")) {
+      try {
+        bodyJson = JSON.parse(bodyText);
+      } catch {
+        // ignore parse errors
+      }
+    }
+
+    if (bodyJson === undefined) {
+      const trimmed = bodyText.trim();
+      if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
+        try {
+          bodyJson = JSON.parse(trimmed);
+        } catch {
+          // ignore parse errors
+        }
+      }
+    }
+
+    const apiResponse: ApiResponse = {
+      status: response.status,
+      headers: responseHeaders,
+      bodyText,
+      ...(bodyJson !== undefined ? { bodyJson } : {}),
+    };
+
+    return apiResponse;
+  }
+}

package/src/testing/adapters/api/api.types.ts

@@ -0,0 +1,15 @@
+import type FormData from "form-data";
+
+export interface ApiAdapterOptions {
+  baseUrl?: string;
+  defaultHeaders?: Record<string, string>;
+}
+
+export interface ApiRequestOptions {
+  method: string;
+  url: string;
+  headers?: Record<string, string>;
+  json?: unknown;
+  bodyText?: string;
+  formData?: FormData;
+}

package/src/testing/adapters/ui/playwright-adapter.ts

@@ -0,0 +1,54 @@
+import type { Browser, BrowserContext, Page } from "playwright";
+
+import type { PlaywrightAdapterOptions } from "./ui.types";
+
+function isAbsoluteUrl(url: string): boolean {
+  return /^https?:\/\//i.test(url);
+}
+
+export class PlaywrightAdapter {
+  private readonly baseUrl?: string;
+  private readonly headless: boolean;
+  private browser?: Browser;
+  private context?: BrowserContext;
+  public page?: Page;
+
+  constructor(opts?: PlaywrightAdapterOptions) {
+    this.baseUrl = opts?.baseUrl;
+    this.headless = opts?.headless ?? true;
+  }
+
+  async start(): Promise<void> {
+    const { chromium } = await import('playwright');
+    this.browser = await chromium.launch({ headless: this.headless });
+    this.context = await this.browser.newContext();
+    this.page = await this.context.newPage();
+  }
+
+  async stop(): Promise<void> {
+    try {
+      if (this.context) {
+        await this.context.close();
+      }
+    } finally {
+      this.context = undefined;
+      this.page = undefined;
+    }
+
+    try {
+      if (this.browser) {
+        await this.browser.close();
+      }
+    } finally {
+      this.browser = undefined;
+    }
+  }
+
+  resolveUrl(url: string): string {
+    if (isAbsoluteUrl(url)) return url;
+    if (this.baseUrl) {
+      return new URL(url, this.baseUrl).toString();
+    }
+    return url;
+  }
+}

package/src/testing/adapters/ui/ui.types.ts

@@ -0,0 +1,4 @@
+export interface PlaywrightAdapterOptions {
+  headless?: boolean;
+  baseUrl?: string;
+}

package/src/testing/contracts/runtime-context.types.ts

@@ -0,0 +1,36 @@
+import type { ApiAdapter } from "../adapters/api/api-adapter";
+import type { PlaywrightAdapter } from "../adapters/ui/playwright-adapter";
+import type { Reporter } from "../reporter/reporter.types";
+import type { SpecRegistry } from "../core/spec-registry";
+import type { ScenarioType } from "./testing-metadata.types";
+
+export type StepPhase = "given" | "when" | "then" | "and" | "step";
+
+export interface ResourceStore {
+  get(path: string): unknown;
+  set(path: string, value: unknown): void;
+  has(path: string): boolean;
+}
+
+export interface ApiResponse {
+  status: number;
+  headers: Record<string, string>;
+  bodyText: string;
+  bodyJson?: unknown;
+}
+
+export interface TestContext {
+  scenarioId: string;
+  scenarioType: ScenarioType;
+  params: Record<string, any>;
+  resources: ResourceStore;
+  api?: ApiAdapter;
+  ui?: PlaywrightAdapter;
+  last?: { apiResponse?: ApiResponse };
+  reporter: Reporter;
+  specRegistry?: SpecRegistry;
+  testData?: Record<string, Record<string, any>>;
+  options?: {
+    printApiLogs?: boolean;
+  };
+}

package/src/testing/contracts/test-spec.types.ts

@@ -0,0 +1,24 @@
+import type { TestContext } from "../contracts/runtime-context.types";
+
+export type SolidTestSpecArgs = {
+  ctx: TestContext;
+  input: Record<string, any>;
+};
+
+export type SolidTestSpecAttachment = {
+  name: string;
+  contentType: string;
+  data: string;
+  encoding?: "utf8" | "base64";
+};
+
+export type SolidTestSpecResult = {
+  ok: boolean;
+  name?: string;
+  details?: Record<string, any>;
+  attachments?: SolidTestSpecAttachment[];
+};
+
+export interface ISolidTestSpec {
+  run(args: SolidTestSpecArgs): Promise<SolidTestSpecResult>;
+}