@lowdefy/server-dev 6.1.0 → 7.0.0

package/lib/docs/observe/createJourneyAppErrors.js

@@ -0,0 +1,236 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import path from 'node:path';
+import { type } from '@lowdefy/helpers';
+import { getStepKey, INTERACTION_STEP_KEYS } from '@lowdefy/node-utils';
+
+import claimExpectedErrors from './claimExpectedErrors.js';
+
+import evaluateInvariants from './evaluateInvariants.js';
+import installStepObserver from './installStepObserver.js';
+import readStepObserver from './readStepObserver.js';
+import resolveConfigKeySource from './resolveConfigKeySource.js';
+import takeJourneyEvents from './takeJourneyEvents.js';
+import { takeRunErrors } from '../runErrorBuffers.js';
+import toAppErrorFailure from './toAppErrorFailure.js';
+import TRANSIENT_SELECTOR from './TRANSIENT_SELECTOR.js';
+import usesSearchStage from './usesSearchStage.js';
+import waitForClientErrorReports from './waitForClientErrorReports.js';
+import watchJourneyContext from './watchJourneyContext.js';
+import withAppErrors from './withAppErrors.js';
+
+function isExpectError(step) {
+  return getStepKey(step ?? {}) === 'expect' && getStepKey(step.expect) === 'error';
+}
+
+// Steps that cannot cause an error a page reports later, so their window
+// closes without waiting for client error reports in flight. An expect.error
+// waits, so a late report of the interaction before it can be claimed.
+function isQuietStep(step) {
+  return ['expect', 'wait', 'screenshot'].includes(getStepKey(step)) && !isExpectError(step);
+}
+
+// What fails a journey: an app error. A dead click is only a warning; a
+// journey asserts it with expect.effect.
+function isAppError(finding) {
+  return finding.severity === 'error' || finding.kind === 'environment';
+}
+
+// Watches a journey run for app errors and judges each step's window with the
+// app error invariants (evaluateInvariants), each error keyed by findingKey. Every
+// actor's context is watched from before its first request (onContext). The
+// windows are contiguous, from the page open to the final drain after the
+// last step, so an error that lands between two steps is never lost. Each
+// window takes what the run's actors buffered (page errors, app API
+// responses) and the error entries the run's own error buffer claimed (see
+// runErrorBuffers), and nothing a developer's own tab or another run caused.
+function createJourneyAppErrors({ origin, basePath, recording, configDirectory }) {
+  const buildDirectory = path.join(process.cwd(), 'build');
+  const run = { run: recording.run.id, journey: recording.run.journey };
+  const events = {
+    pageErrors: [],
+    requests: [],
+    responses: [],
+    abortedRequests: [],
+    pendingClientErrors: new Set(),
+  };
+  let since = 0;
+  let current = null;
+  // The window the last step closed, which an expect.effect step reads for
+  // the interaction just before it.
+  let closed = null;
+
+  function onContext({ context }) {
+    watchJourneyContext({ context, events, origin, basePath });
+  }
+
+  function takeWindow({ page, observed, urlBefore }) {
+    const until = Date.now();
+    const window = {
+      since,
+      until,
+      urlBefore,
+      urlAfter: page.isClosed() ? null : page.url(),
+      emits: observed?.emits ?? [],
+      mutationCount: observed?.mutationCount ?? null,
+      ...takeJourneyEvents({ events, since, until }),
+      errors: takeRunErrors({ ...run, since, until }),
+    };
+    since = until;
+    return window;
+  }
+
+  async function findAppErrors({ step, result, window, pageId }) {
+    const findings = await evaluateInvariants({
+      step,
+      result,
+      window,
+      pageId,
+      basePath,
+      configDirectory,
+      usesSearchStage: (entry) => usesSearchStage({ buildDirectory, entry }),
+      resolveSource: (configKey) =>
+        resolveConfigKeySource({ buildDirectory, configDirectory, configKey }),
+    });
+    return findings.filter(isAppError);
+  }
+
+  // A page mid-navigation, or one that has crashed, has nothing to report.
+  async function readPageId(page) {
+    return page.evaluate(() => window.lowdefy?.pageId ?? null).catch(() => null);
+  }
+
+  // Judges what opening the journey's first page caused (its onInit and
+  // onMount requests included). Returns a failure with phase 'open', or
+  // undefined.
+  async function judgeOpen({ page }) {
+    await waitForClientErrorReports({ events });
+    const pageId = await readPageId(page);
+    const window = takeWindow({ page, observed: null, urlBefore: page.url() });
+    const findings = await findAppErrors({ step: {}, result: { status: 'ok' }, window, pageId });
+    if (findings.length === 0) {
+      return undefined;
+    }
+    return toAppErrorFailure({ findings, phase: 'open' });
+  }
+
+  // Opens a step's window on the page the step starts on. The observer is
+  // installed in each new document; a page mid-navigation cannot take it, and
+  // its window then has no trace emits (its error reports still count).
+  async function openWindow({ page }) {
+    await page.evaluate(installStepObserver).catch(() => null);
+    current = { page, urlBefore: page.url(), pageId: await readPageId(page) };
+  }
+
+  // Closes the step's window and returns the app errors it held, as findings
+  // ({ kind, severity, message, pageId, source, configKey, key }).
+  async function closeWindow({ journey, step, result }) {
+    if (!isQuietStep(step)) {
+      await waitForClientErrorReports({ events });
+    }
+    const { page, urlBefore, pageId } = current;
+    current = null;
+    const observed = page.isClosed()
+      ? null
+      : await page
+          .evaluate(readStepObserver, { transientSelector: TRANSIENT_SELECTOR })
+          .catch(() => null);
+    const window = takeWindow({ page: journey.actors.current().page, observed, urlBefore });
+    closed = window;
+    return findAppErrors({ step, result, window, pageId });
+  }
+
+  // The window of the step before the one running now: an expect step opens
+  // its own window, so the last one closed is the interaction's before it.
+  function previousWindow() {
+    return closed;
+  }
+
+  // After the last step: what its effects reported late (a client error
+  // report still in flight when its window closed) is judged against it.
+  async function drain({ journey, step, result }) {
+    await waitForClientErrorReports({ events });
+    const page = journey.actors.current().page;
+    const pageId = await readPageId(page);
+    const window = takeWindow({ page, observed: null, urlBefore: page.url() });
+    return findAppErrors({ step, result, window, pageId });
+  }
+
+  // The app errors of an interaction an expect.error step follows, held for
+  // that step to claim instead of failing the interaction at once.
+  let held = null;
+
+  // Closes the step's window and returns the journey's failure once its app
+  // errors are counted (see withAppErrors), or undefined while it passes.
+  // `failure` is the step's own failure, if any. An interaction followed by
+  // an expect.error step holds its app errors for that step to claim (see
+  // claimExpectedErrors); it fails on them only if they are not claimed.
+  async function judgeStep({ journey, steps, index, results, failure, leftOrigin }) {
+    const step = steps[index];
+    const result = results[index];
+    const findings = await closeWindow({ journey, step, result });
+    if (isExpectError(step) && type.isUndefined(failure)) {
+      const claim = claimExpectedErrors({ step, index, held, findings });
+      held = null;
+      if (!type.isUndefined(claim)) {
+        results[claim.index].status = 'failed';
+      }
+      return claim;
+    }
+    if (
+      type.isUndefined(failure) &&
+      INTERACTION_STEP_KEYS.includes(getStepKey(step)) &&
+      isExpectError(steps[index + 1])
+    ) {
+      held = { index, step, findings };
+      return undefined;
+    }
+    if (findings.length === 0) {
+      return failure;
+    }
+    result.status = 'failed';
+    return withAppErrors({ failure, leftOrigin, findings, index, step });
+  }
+
+  // After the last step passed, what its effects reported late (a client
+  // error report still in flight when its window closed) is judged against
+  // it.
+  async function judgeDrain({ journey, steps, results }) {
+    const index = steps.length - 1;
+    const step = steps[index];
+    const result = results[index];
+    const findings = await drain({ journey, step, result });
+    if (findings.length === 0) {
+      return undefined;
+    }
+    result.status = 'failed';
+    return withAppErrors({ findings, index, step });
+  }
+
+  return {
+    onContext,
+    judgeOpen,
+    openWindow,
+    judgeStep,
+    judgeDrain,
+    closeWindow,
+    drain,
+    previousWindow,
+  };
+}
+
+export default createJourneyAppErrors;

package/lib/docs/observe/createJourneyAppErrors.test.mjs

@@ -0,0 +1,385 @@
+/*
+  Copyright 2020-2026 Lowdefy, Inc
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import { jest } from '@jest/globals';
+
+jest.unstable_mockModule('./waitForClientErrorReports.js', () => ({ default: async () => {} }));
+
+const { default: createJourneyAppErrors } = await import('./createJourneyAppErrors.js');
+const { default: installStepObserver } = await import('./installStepObserver.js');
+const { default: readStepObserver } = await import('./readStepObserver.js');
+const { recordRunError, registerRunBuffer, releaseRunBuffer } = await import(
+  '../runErrorBuffers.js'
+);
+
+const origin = 'http://localhost:3227';
+const run = { run: '20261005T120000Z-ae0001', journey: null };
+const recording = { source: 'journey', run: { id: run.run, by: 'test', journey: null } };
+
+let emits;
+
+function createPage() {
+  return {
+    evaluate: jest.fn(async (fn) => {
+      if (fn === installStepObserver) return true;
+      if (fn === readStepObserver) return { emits, mutationCount: 1 };
+      return 'orders';
+    }),
+    url: () => `${origin}/orders`,
+    isClosed: () => false,
+  };
+}
+
+// A fake browser context: the watch's listeners, called as Playwright would.
+function createContext() {
+  const listeners = {};
+  return {
+    on: (event, callback) => {
+      listeners[event] = callback;
+    },
+    abort({ url }) {
+      listeners.requestfailed({
+        url: () => url,
+        failure: () => ({ errorText: 'net::ERR_ABORTED' }),
+      });
+    },
+    respond({ url, status }) {
+      listeners.response({
+        url: () => url,
+        status: () => status,
+        request: () => ({ method: () => 'POST' }),
+      });
+    },
+  };
+}
+
+function serverEntry({ message, recordingRun = run.run }) {
+  return {
+    timestamp: new Date().toISOString(),
+    message,
+    source: '/app/pages/orders.yaml:12',
+    requestId: 'broken',
+    pageId: 'orders',
+    store: 'server',
+    recording: { source: 'journey', run: recordingRun, journey: null },
+  };
+}
+
+let page;
+let journey;
+let context;
+let appErrors;
+
+beforeEach(() => {
+  emits = [];
+  registerRunBuffer(run);
+  appErrors = createJourneyAppErrors({ origin, basePath: '', recording, configDirectory: '/app' });
+  context = createContext();
+  appErrors.onContext({ context });
+  page = createPage();
+  journey = { actors: { current: () => ({ page }) } };
+});
+
+afterEach(() => {
+  releaseRunBuffer(run);
+});
+
+async function runStepWindow({ step, during = () => {} }) {
+  await appErrors.openWindow({ page });
+  during();
+  return appErrors.closeWindow({ journey, step, result: { status: 'ok' } });
+}
+
+test('a server error entry the run claimed fails the step, with its kind, source and key', async () => {
+  const findings = await runStepWindow({
+    step: { click: 'broken' },
+    during: () => recordRunError(serverEntry({ message: 'Unrecognized pipeline stage' })),
+  });
+  expect(findings).toEqual([
+    expect.objectContaining({
+      kind: 'server-error',
+      severity: 'error',
+      message: 'Unrecognized pipeline stage',
+      source: 'pages/orders.yaml:12',
+      key: 'server-error|orders|pages/orders.yaml:12',
+    }),
+  ]);
+});
+
+test("an error entry of another run is not this journey's", async () => {
+  const other = { run: '20261005T120000Z-ae0002', journey: null };
+  registerRunBuffer(other);
+  const findings = await runStepWindow({
+    step: { click: 'broken' },
+    during: () => recordRunError(serverEntry({ message: 'elsewhere', recordingRun: other.run })),
+  });
+  releaseRunBuffer(other);
+  expect(findings).toEqual([]);
+});
+
+test('an unexplained 5xx from an app API route fails the step as request-failed', async () => {
+  const findings = await runStepWindow({
+    step: { click: 'save' },
+    during: () => context.respond({ url: `${origin}/api/request/orders/save`, status: 500 }),
+  });
+  expect(findings.map((finding) => finding.kind)).toEqual(['request-failed']);
+});
+
+test.each([
+  ['AuthorizationError', 403],
+  ['AuthenticationError', 401],
+  ['TwoFactorEnrolmentRequiredError', 403],
+])(
+  'a CallAPI an auth gate refused (%s, %i) is not an app error: the refusal and its emit both pass',
+  async (errorName, status) => {
+    emits = [
+      {
+        blockId: 'admin_button',
+        eventName: 'onClick',
+        success: false,
+        failure: { errorName, actionId: 'call_admin', actionType: 'CallAPI', configKey: 'k1' },
+      },
+    ];
+    const findings = await runStepWindow({
+      step: { click: 'admin_button' },
+      during: () => context.respond({ url: `${origin}/api/endpoints/admin_only`, status }),
+    });
+    expect(findings).toEqual([]);
+  }
+);
+
+test('a UserError and a dead click are not app errors', async () => {
+  emits = [
+    {
+      blockId: 'throw_button',
+      eventName: 'onClick',
+      success: false,
+      failure: { errorName: 'UserError', actionId: 'throw', actionType: 'Throw' },
+    },
+  ];
+  const thrown = await runStepWindow({ step: { click: 'throw_button' } });
+  expect(thrown).toEqual([]);
+  emits = [];
+  page.evaluate.mockImplementation(async (fn) => {
+    if (fn === readStepObserver) return { emits: [], mutationCount: 0 };
+    return fn === installStepObserver ? true : 'orders';
+  });
+  const dead = await runStepWindow({ step: { click: 'dead_button' } });
+  expect(dead).toEqual([]);
+});
+
+test('an action that fails with an error that is not a UserError fails the step as action-error', async () => {
+  emits = [
+    {
+      blockId: 'explode_button',
+      eventName: 'onClick',
+      success: false,
+      failure: { errorName: 'RequestError', actionId: 'call_explode', actionType: 'CallAPI' },
+    },
+  ];
+  const findings = await runStepWindow({ step: { click: 'explode_button' } });
+  expect(findings.map((finding) => finding.kind)).toEqual(['action-error']);
+});
+
+test('an error that lands between two steps is judged in the next window, not lost', async () => {
+  expect(await runStepWindow({ step: { click: 'a' } })).toEqual([]);
+  recordRunError(serverEntry({ message: 'between steps' }));
+  const findings = await runStepWindow({ step: { expect: { visible: 'a' } } });
+  expect(findings.map((finding) => finding.message)).toEqual(['between steps']);
+});
+
+test('judgeOpen fails with phase open on an error raised while the page opened', async () => {
+  recordRunError(serverEntry({ message: 'onInit failed' }));
+  const failure = await appErrors.judgeOpen({ page });
+  expect(failure).toEqual({
+    phase: 'open',
+    kind: 'app-error',
+    message: 'Opening the page caused an app error: server-error: onInit failed',
+    expected: 'no app error',
+    actual: ['server-error: onInit failed'],
+    errors: [
+      {
+        kind: 'server-error',
+        message: 'onInit failed',
+        source: 'pages/orders.yaml:12',
+        configKey: null,
+        key: 'server-error|orders|pages/orders.yaml:12',
+      },
+    ],
+  });
+  expect(await appErrors.judgeOpen({ page })).toBeUndefined();
+});
+
+test('drain judges what arrived after the last window closed', async () => {
+  expect(await runStepWindow({ step: { click: 'a' } })).toEqual([]);
+  recordRunError(serverEntry({ message: 'late' }));
+  const findings = await appErrors.drain({
+    journey,
+    step: { click: 'a' },
+    result: { status: 'ok' },
+  });
+  expect(findings.map((finding) => finding.message)).toEqual(['late']);
+});
+
+// Runs steps through judgeStep as runJourneySteps does, with `during(index)`
+// raising what each step causes.
+async function judgeSteps({ steps, during = () => {} }) {
+  const results = [];
+  let failure;
+  for (let index = 0; index < steps.length; index += 1) {
+    if (failure !== undefined) {
+      results.push({ index, status: 'skipped' });
+      continue;
+    }
+    await appErrors.openWindow({ page });
+    during(index);
+    results.push({ index, status: 'ok' });
+    failure = await appErrors.judgeStep({
+      journey,
+      steps,
+      index,
+      results,
+      failure,
+      leftOrigin: false,
+    });
+  }
+  return { results, failure };
+}
+
+const duplicateSteps = [
+  { click: 'save' },
+  { expect: { error: 'duplicate key' } },
+  { expect: { visible: 'already_exists' } },
+];
+
+test('expect.error claims the matching app error of the click before it, and the journey passes', async () => {
+  const { results, failure } = await judgeSteps({
+    steps: duplicateSteps,
+    during: (index) => {
+      if (index === 0) recordRunError(serverEntry({ message: 'E11000 duplicate key error' }));
+    },
+  });
+  expect(failure).toBeUndefined();
+  expect(results.map((result) => result.status)).toEqual(['ok', 'ok', 'ok']);
+});
+
+test('expect.error fails at the expect step when no app error matches', async () => {
+  const { results, failure } = await judgeSteps({ steps: duplicateSteps });
+  expect(failure).toEqual({
+    index: 1,
+    step: { expect: { error: 'duplicate key' } },
+    expected: 'an app error containing "duplicate key"',
+    actual: 'no app error',
+    message: 'Expected an app error containing "duplicate key" but found no app error.',
+  });
+  expect(results.map((result) => result.status)).toEqual(['ok', 'failed', 'skipped']);
+});
+
+test('an app error expect.error does not claim still fails the click', async () => {
+  const { results, failure } = await judgeSteps({
+    steps: duplicateSteps,
+    during: (index) => {
+      if (index !== 0) return;
+      recordRunError(serverEntry({ message: 'E11000 duplicate key error' }));
+      recordRunError({
+        ...serverEntry({ message: 'Connection lost' }),
+        source: '/app/pages/orders.yaml:30',
+      });
+    },
+  });
+  expect(failure).toEqual(
+    expect.objectContaining({
+      index: 0,
+      step: { click: 'save' },
+      kind: 'app-error',
+      actual: ['server-error: Connection lost'],
+    })
+  );
+  expect(results.map((result) => result.status)).toEqual(['failed', 'ok', 'skipped']);
+});
+
+test('an app error with no expect.error after it fails its step', async () => {
+  const { results, failure } = await judgeSteps({
+    steps: [{ click: 'save' }, { expect: { visible: 'saved' } }],
+    during: (index) => {
+      if (index === 0) recordRunError(serverEntry({ message: 'E11000 duplicate key error' }));
+    },
+  });
+  expect(failure).toEqual(expect.objectContaining({ index: 0, kind: 'app-error' }));
+  expect(results.map((result) => result.status)).toEqual(['failed', 'skipped']);
+});
+
+test('expect.error claims the failed action that reported the error it matched', async () => {
+  const failedInsert = {
+    blockId: 'save',
+    eventName: 'onClick',
+    success: false,
+    failure: { errorName: 'ServiceError', actionId: 'insert', actionType: 'Request' },
+  };
+  const { failure } = await judgeSteps({
+    steps: duplicateSteps,
+    during: (index) => {
+      emits = index === 0 ? [failedInsert] : [];
+      if (index === 0) recordRunError(serverEntry({ message: 'E11000 duplicate key error' }));
+    },
+  });
+  expect(failure).toBeUndefined();
+});
+
+test('previousWindow is the window the last step closed, for expect.effect to read', async () => {
+  expect(appErrors.previousWindow()).toBeNull();
+  emits = [{ blockId: 'save', eventName: 'onClick', success: true }];
+  await runStepWindow({ step: { click: 'save' } });
+  expect(appErrors.previousWindow()).toEqual(
+    expect.objectContaining({
+      emits,
+      mutationCount: 1,
+      requests: [],
+      urlBefore: `${origin}/orders`,
+      urlAfter: `${origin}/orders`,
+    })
+  );
+});
+
+function clientEntry({ message }) {
+  return {
+    timestamp: new Date().toISOString(),
+    name: 'TypeError',
+    message,
+    source: null,
+    store: 'client',
+    recording: { source: 'journey', run: run.run, journey: null },
+  };
+}
+
+test("a goto whose navigation aborted the old page's requests is not failed by their fetch errors", async () => {
+  const findings = await runStepWindow({
+    step: { goto: 'onboarding-step-connect' },
+    during: () => {
+      context.abort({ url: `${origin}/api/request/step/get_header` });
+      recordRunError(clientEntry({ message: 'Failed to fetch at onboarding-step-2.' }));
+    },
+  });
+  expect(findings).toEqual([]);
+});
+
+test('a failed fetch with no aborted request still fails a goto step', async () => {
+  const findings = await runStepWindow({
+    step: { goto: 'onboarding-step-connect' },
+    during: () => recordRunError(clientEntry({ message: 'Failed to fetch at onboarding-step-2.' })),
+  });
+  expect(findings.map((finding) => finding.kind)).toEqual(['client-error']);
+});