npm - @alwaysmeticulous/cli - Versions diffs - 2.326.1 → 2.328.0 - Mend

@alwaysmeticulous/cli 2.326.1 → 2.328.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (456) hide show

package/dist/commands/onboard/templates/docs/how-to/recorder-script.ts ADDED Viewed

@@ -0,0 +1,139 @@
+import { METICULOUS_SETUP_CALENDLY_LINK } from "src/lib/next/next.constants";
+import {
+  INSTALLATION_INSTRUCTIONS_ANCHOR,
+  INSTALL_RECORDER_AS_NPM_DEPENDENCY_INSTALLATION_INSTRUCTIONS_URL,
+  NEXTJS_APP_ROUTER_ADDITIONAL_SETUP_URL,
+} from "src/lib/utils/internal-urls/docs-urls.utils";
+import { SNIPPET_URL } from "../recorder-snippets/constants";
+import { nextJsInstructions } from "../recorder-snippets/script-based-instructions/next-js-instructions";
+import { nuxtJsInstructions } from "../recorder-snippets/script-based-instructions/nuxtjs-instructions";
+import { rsbuildInstructions } from "../recorder-snippets/script-based-instructions/rsbuild-instructions";
+import { storybookInstructions } from "../recorder-snippets/script-based-instructions/storybook-instructions";
+import { svelteKitInstructions } from "../recorder-snippets/script-based-instructions/sveltekit-instructions";
+import { viteInstructions } from "../recorder-snippets/script-based-instructions/vite-instructions";
+import { scriptRequirementsCalloutCard } from "../recorder-snippets/script-requirements";
+import { STEPS_AFTER_INSTALLING_RECORDER } from "../recorder-snippets/steps-after-installing-recorder";
+const BOOK_A_CALL_TIP = `If you have any issues setting up the recorder then click [here](${METICULOUS_SETUP_CALENDLY_LINK}) to book a call with us.`;
+const POST_TAB_CONTENT = `
+If you have any cross-origin or sandboxed iFrames then the recorder should be added to each of these iFrames as well as the main frame. ${BOOK_A_CALL_TIP}
+${STEPS_AFTER_INSTALLING_RECORDER}
+`;
+export const document = `---
+{
+  "title": "Install the Meticulous recorder via a script tag"
+}
+---
+{% anchor id="${INSTALLATION_INSTRUCTIONS_ANCHOR}" /%}
+# {% $frontmatter.title %}
+Please select your framework or build tool:
+{% tabs direction="grid" noTabSelectedByDefault=true %}
+{% tab label="NextJS with the /pages directory" %}
+## Installing on NextJS with the /pages directory
+${scriptRequirementsCalloutCard({ isNextJs: "yes" })}
+Add a script tag to your \`_document.js\` file within \`Head\`. If the layout doesn't yet have a \`<Head>\` tag then
+you can add one within the \`<Html>\` tag.
+${nextJsInstructions("Head")}
+${POST_TAB_CONTENT}
+{% /tab %}
+{% tab label="NextJS with the /app directory" %}
+## Installing on NextJS with the /app directory
+${scriptRequirementsCalloutCard({ isNextJs: "yes" })}
+Add a script tag to your \`/app/layout.tsx\` or \`/app/layout.jsx\` file within \`head\`. If the layout doesn't yet have a \`<head>\` tag then
+you can add one within the \`<html>\` tag.
+${nextJsInstructions("head")}
+After adding the snippet you'll need to follow a [few additional steps](${NEXTJS_APP_ROUTER_ADDITIONAL_SETUP_URL}) to ensure Meticulous can
+correctly test your app.
+${POST_TAB_CONTENT}
+{% /tab %}
+{% tab label="Nuxt" %}
+## Installing on NuxtJS
+${scriptRequirementsCalloutCard({ isNextJs: "no" })}
+${nuxtJsInstructions()}
+${POST_TAB_CONTENT}
+{% /tab %}
+{% tab label="SvelteKit" %}
+## Installing on SvelteKit
+${scriptRequirementsCalloutCard({ isNextJs: "no" })}
+${svelteKitInstructions()}
+${POST_TAB_CONTENT}
+{% /tab %}
+{% tab label="Vite" %}
+## Installing on Vite
+${scriptRequirementsCalloutCard({ isNextJs: "no" })}
+${viteInstructions()}
+${POST_TAB_CONTENT}
+{% /tab %}
+{% tab label="rsbuild" %}
+## Installing on rsbuild
+${scriptRequirementsCalloutCard({ isNextJs: "no" })}
+${rsbuildInstructions()}
+${POST_TAB_CONTENT}
+{% /tab %}
+{% tab label="Storybook" %}
+## Installing on Storybook
+${scriptRequirementsCalloutCard({ isNextJs: "no" })}
+${storybookInstructions()}
+${POST_TAB_CONTENT}
+{% /tab %}
+{% tab label="Any other framework or build tool" %}
+## Installing on any other framework or build tool
+${scriptRequirementsCalloutCard({ isNextJs: "no" })}
+Add the recorder as the first script tag in your \`<head>\` tag. If you only want to record sessions in non-production environments then
+you will need to template your HTML to only include the script tag in non-production environments (if this is not possible then you can
+ [use an NPM dependency instead of a script tag](${INSTALL_RECORDER_AS_NPM_DEPENDENCY_INSTALLATION_INSTRUCTIONS_URL})).
+{% code_with_project_selector %}
+\`\`\`html
+<head>
+  ...
+  <script
+    data-recording-token="{% project_recording_token /%}"
+    data-is-production-environment="<true/false>"
+    src="${SNIPPET_URL}">
+  </script>
+  <!--Meticulous snippet should be added before your app -->
+  ...
+  <script src="main_app.js"></script>
+</head>
+\`\`\`
+{% /code_with_project_selector %}
+${POST_TAB_CONTENT}
+{% /tab %}
+{% /tabs %}
+`;

package/dist/commands/onboard/templates/docs/how-to/testing-feature-flags.ts ADDED Viewed

@@ -0,0 +1,97 @@
+import {
+  METICULOUS_WINDOW_OBJECT_URL,
+  NETWORK_STUBBING_EXPLANATION_URL,
+  RECORD_SESSION_CONTEXT_URL,
+  TESTING_POOL_URL,
+  TYPESCRIPT_TYPES_URL,
+} from "src/lib/utils/internal-urls/docs-urls.utils";
+export const document = `---
+{
+  "title": "Testing Feature Flags with Meticulous"
+}
+---
+# {% $frontmatter.title %}
+> **Note:** To learn how to *register* feature flags in your sessions, see [Recording the context of a user session](${RECORD_SESSION_CONTEXT_URL}). This page explains how Meticulous *tests* feature flags.
+By default Meticulous will snapshot the network responses, local storage values, cookies and session storage values when a session is
+originally recorded, and [stub out and replay those values](${NETWORK_STUBBING_EXPLANATION_URL}) when later replaying the session. This allows
+Meticulous to test across varied feature flag configurations.
+And since different feature flag combinations activate different code paths, and Meticulous
+[optimizes the test suite to cover all code paths](${TESTING_POOL_URL}), Meticulous will normally automatically include
+test cases for all relevant recorded feature flag combinations.
+For example, say user 1 records a first user session, session A, and user 2 records a second user session, session B, and the value of the feature
+flag *my_new_feature* is false for user 1 and true for user 2, then when Meticulous replays session A it'll replay with *my_new_feature* disabled,
+and when Meticulous replays session B it'll replay with *my_new_feature* enabled. Meticulous will likely select both sessions into the test
+suite since they cover different code paths.
+### Improving Test Coverage with New Feature Flags
+As mentioned above Meticulous will automatically test feature flag combinations if sessions are recorded with the different feature flags enabled.
+However any sessions recorded *prior* to the feature flag being introduced will likely not test the new feature since they'll replay old saved
+network responses and local storage values without an entry for the new feature flag. This means test coverage of new features gated
+behind feature flags can be limited until new sessions are recorded with that feature flag enabled.
+You can fix this by enabling the feature flags by default when running as part of a Meticulous test in the case that the feature flag cannot be found in
+the network response or local storage (i.e. when Meticulous is replaying old network responses or local storage values from before the
+feature flag was introduced). We've included instructions for Statsig below, but a similar approach can be applied for any feature flagging
+framework. **We recommend making this change if you often develop new features gated behind feature flags.**
+### Configuring Meticulous with Statsig
+Before checking a feature flag value first check [window.Meticulous?.isRunningAsTest](${METICULOUS_WINDOW_OBJECT_URL}), and if so then
+check if configuration for the feature flag is missing entirely - if it is then default the feature flag to enabled. This then allows
+Meticulous to use old sessions to test new features. Here's an example for Statsig, however similar approaches can be followed for other
+feature flagging frameworks, and for Statsig's React integration:
+\`\`\`typescript
+import { StatsigClient } from '@statsig/js-client';
+const myStatsigClient = new StatsigClient(
+  YOUR_CLIENT_KEY,
+  { userID: 'a-user' },
+  ...
+);
+await myStatsigClient.initializeAsync();
+// We wrap checkGate, and use the wrapped version in our code instead of directly calling myStatsigClient.checkGate
+const checkGate = (gateName: string) => {
+  // If the application is running as part of a Meticulous test, and Meticulous is replaying old network responses or local storage values
+  // from before the feature gate was introduced then let's default the feature to on, so that Meticulous can use old user sessions to test
+  // new features.
+  //
+  // See https://docs.statsig.com/sdk/debugging
+  if (window.Meticulous?.isRunningAsTest
+      && myStatsigClient.getFeatureGate(gateName).details.reason.endsWith("Unrecognized")) {
+    return true;
+  }
+  return myStatsigClient.checkGate(gateName);
+}
+\`\`\`
+### Configuring Meticulous with LaunchDarkly
+When accessing a variation default it to true if [window.Meticulous?.isRunningAsTest](${METICULOUS_WINDOW_OBJECT_URL}) is true.
+Before:
+\`\`\`typescript
+client.variation('my_new_feature', false);
+\`\`\`
+After:
+\`\`\`typescript
+client.variation('my_new_feature', window.Meticulous?.isRunningAsTest ?? false);
+\`\`\`
+We recommend wrapping your client to make this the automatic behavior rather than updating every call site.
+## Related Pages
+- [Recording the context of a user session](${RECORD_SESSION_CONTEXT_URL}) - Learn how to record feature flags and other session context
+- [TypeScript Types for window.Meticulous](${TYPESCRIPT_TYPES_URL}) - Get type definitions for the \`window.Meticulous\` object
+`;

package/dist/commands/onboard/templates/docs/how-to/testing-multiple-apps-or-app-variants.ts ADDED Viewed

@@ -0,0 +1,40 @@
+import { METICULOUS_SUPPORT_EMAIL } from "src/lib/next/next.constants";
+import { BASE_URL_EXPLANATION_URL } from "src/lib/utils/internal-urls/docs-urls.utils";
+export const document = `---
+{
+  "title": "Testing Multiple Apps or App Variants"
+}
+---
+# {% $frontmatter.title %}
+### I have a single app, but with multiple variants deployed under different configurations to different URLs. How do I use Meticulous to test them?
+When Meticulous simulates sessions it is configured to simulate the sessions against a particular base URL - for example the \`report-diffs-action\` \`appUrl\` or the URL of the Vercel deployment. This URL will
+normally be different to the URL the session was recorded at. When simulating a session, Meticulous takes the URL the session was recorded at and swaps out the origin with the new base URL. Learn more [here](${BASE_URL_EXPLANATION_URL}).
+You'll therefore need to make sure that the base URL you are simulating sessions against serves up the same app under the same configuration as the base URL sessions are recorded on.
+If you have multiple URLs that serve different variants of your app, for example, customized for different customers, then you can set up multiple Meticulous projects and multiple Vercel deployments or \`report-diffs-action\` calls - one to test each variant of the app using the sessions recorded for that variant. See below for more details.
+### I have multiple independent apps in the same monorepo. How do I use Meticulous to test them?
+You'll most likely want to set up multiple Meticulous projects for the same GitHub repo. Each Meticulous project will have its own recording token, allowing you
+to set up each app to record sessions in a different Meticulous project.
+If you're using Github Actions to run Meticulous in CI you can then set up a separate call
+to \`report-diffs-action\` for each Meticulous project, passing in the API token of the relevant project, and pointing it to a URL that serves the correct application.
+If you're using Vercel, or another service that provides preview URLs, then you'll want to configure Vercel deployments for each application separately.
+Navigate to the project settings page for each Meticulous project and configure that project to test against only the relevant Vercel deployments by
+selecting the appropriate environments in the \`Environments to Test Against\` section.
+### GitHub check names with multiple projects
+When you have multiple Meticulous projects connected to the same GitHub repository, the GitHub status check name includes the project name to differentiate them. For example, instead of *'Meticulous Tests'*, the checks will be named *'Meticulous Tests (project-a)'* and *'Meticulous Tests (project-b)'*.
+If you have Meticulous configured as a [required check](/docs/make-check-blocking) in your branch protection rules, make sure the required check names match the actual check names. Adding a second project to a repo that previously only had one will change the check name, which can cause PRs to hang waiting for a check that no longer exists under the old name.
+Reach out to [${METICULOUS_SUPPORT_EMAIL}](mailto:${METICULOUS_SUPPORT_EMAIL}) and we'll help you get set up.
+`;

package/dist/commands/onboard/templates/docs/how-to/troubleshoot-auth.ts ADDED Viewed

@@ -0,0 +1,58 @@
+import { METICULOUS_SUPPORT_EMAIL } from "src/lib/next/next.constants";
+import {
+  AUTH_BYPASSING_AUTH_URL,
+  AUTH_ENABLING_FULL_AUTH_URL,
+} from "src/lib/utils/internal-urls/docs-urls.utils";
+export const document = `---
+{
+  "title": "Troubleshoot Authentication & Authorisation Issues"
+}
+---
+# {% $frontmatter.title %}
+Auth issues are some of the most common issues that you might encounter while setting up Meticulous.
+This class of issues is easily identifiable by sessions recorded on logged-in pages which, when simulated, consistently redirect to
+log-in pages or 401 screens.
+By default Meticulous automatically stubs all XHR, Fetch and WebSocket requests to your backend, and therefore does not necessarily need to
+authenticate to your backend. In addition Meticulous will also automatically record and replay cookies,
+local storage & session storage, and so will often be able to authenticate automatically.
+However if you use server side rendering (SSR),
+React server components, or wish to test your backend then you may need Meticulous to be able
+to authenticate correctly with your backend. This works out of the box if your cookie expiries are long enough (at least a week) and
+you're not using http only cookies. If this isn't the case click [here](${AUTH_ENABLING_FULL_AUTH_URL}) for docs on how to enable
+Meticulous to authenticate correctly with your backend.
+However even if you are only using Meticulous to test your frontend, there are still some common issues that can arise:
+### 1. Backend auth checks when serving the document's HTML
+When requesting your app's document/HTML your backend may check the user's authentication and redirect them to a login page if they are no
+longer authenticated. If this is the case you'll either need to [disable this redirect](${AUTH_BYPASSING_AUTH_URL}) when replaying the Meticulous tests against CI/preview URLs, or allow
+[Meticulous to authenticate correctly against your backend](${AUTH_ENABLING_FULL_AUTH_URL}).
+### 2. Different auth setups across environments
+This can be an issue if all of the following hold:
+  - The environment you replay sessions against in CI and the environment you record sessions on (e.g. localhost) use different auth setups, and:
+  - Your _frontend_ code checks the user's authentication status, and:
+  - This check would fail if the cookies or local storage were from an environment with a different auth setup (for example, your FE code
+    redirects the user to login unless a cookie with name \`auth.\${environment-name}\` exists).
+In this case you'll either need to [disable the FE check when running Meticulous tests](${AUTH_BYPASSING_AUTH_URL}), or standardize your auth provider client across environments.
+### 3. You are using Auth0
+In this case you may need to configure Auth0 to store user session data in local storage, or not use httpOnly cookies. See [here](${AUTH_ENABLING_FULL_AUTH_URL}?tab=Auth0) for more information.
+## Issues / questions?
+We're always happy to help you with any issues you encounter while setting up or with anything else you might be unsure about.
+Get in touch by emailing [${METICULOUS_SUPPORT_EMAIL}](mailto:${METICULOUS_SUPPORT_EMAIL}).
+`;

package/dist/commands/onboard/templates/docs/how-to/troubleshoot-failed-simulations.ts ADDED Viewed

@@ -0,0 +1,97 @@
+import {
+  METICULOUS_SETUP_CALENDLY_LINK,
+  METICULOUS_SUPPORT_EMAIL,
+} from "src/lib/next/next.constants";
+import {
+  ENSURE_RECORDER_CAPTURES_ALL_REQUESTS_URL,
+  FIX_FALSE_POSITIVES_URL,
+  RECORD_AND_REPLAY_ON_DIFFERENT_ENVIRONMENTS_URL,
+  TESTING_POOL_URL,
+  TROUBLESHOOTING_FAILED_SIMULATIONS_STEPS_ANCHOR,
+} from "src/lib/utils/internal-urls/docs-urls.utils";
+import { METICULOUS_DEBUG_PR_LABEL, SIMULATION_TAB_NAMES } from "../constants";
+export const document = `---
+  {
+    "title": "Troubleshoot Failed or Inaccurate Simulations"
+  }
+  ---
+  # {% $frontmatter.title %}
+  ## Some simulations on my PR test run are failing or inaccurate
+  Simulations can fail or replay inaccurately for a few reasons:
+  1. **The session is out of date**: Your application has changed significantly since the session was originally recorded, such that the session can't be replayed against
+  the new version of your application. This could be due to UI changes, such that the session can't interact with the same elements. Or
+  it could be due to your application's API changing, triggering errors when Meticulous replays old out-of-date network requests from the
+  original session. Such failing replays can be safely ignored: Meticulous will automatically detect if the session is no longer able to cover certain code paths or user
+  flows on your new codebase and select one or more new sessions, with new network responses and user interactions, to correctly cover these paths.
+  2. **There is an error with your recorder setup**, for example [it's not being initialized early enough so not capturing all network responses](${ENSURE_RECORDER_CAPTURES_ALL_REQUESTS_URL}).
+  3. **There are fundamental differences between the environment the session was recorded on and the environment it is being replayed in**, [such
+  that sessions recorded on one environment can't be replayed on another](${RECORD_AND_REPLAY_ON_DIFFERENT_ENVIRONMENTS_URL}).
+  Follow the steps below to distinguish between these causes and troubleshoot why simulations may be failing.
+  {% anchor id="${TROUBLESHOOTING_FAILED_SIMULATIONS_STEPS_ANCHOR}" /%}
+  ## Troubleshooting why simulations may be failing
+  ### 1. Check if the user session can be simulated accurately *against the environment from which it was recorded*
+  To verify whether the user session can be simulated accurately in the same environment in which it was recorded, you can:
+  1. Find a session from the **All Sessions** or **Selected Sessions** tab of your {% project_link %}Meticulous dashboard{% /project_link %}.
+  2. Follow the command to "Replay this session" on the **Simulate** tab of the session page.
+  3. Note that if the session was originally recorded on localhost then the \`--appUrl\` flag will be set to a localhost URL, and you'll
+  need to serve your app on that same port locally to replay the session (or edit the \`--appUrl\`).
+  While simulating the session, keep an eye out for common symptoms of an unsuccessfully simulated session:
+  - The simulation hits an error screen
+  - The simulation fails to populate pages with data
+  - The simulation hits an error state while navigating through a form and stays there for the duration of the simulation
+  If you see any of these symptoms, and the application has changed somewhat since the session was first recorded, then the
+  session is likely failing to replay because it is out of date, and there has since been a breaking API schema change or a breaking UI change.
+  Such replay failures can be safely ignored: Meticulous will automatically detect if the session is no longer able to cover certain code paths or user
+  flows on your new codebase and select one or more new sessions, with new network responses and user interactions, to correctly cover these paths. For
+  more information on how Meticulous selects sessions, see [here](${TESTING_POOL_URL}).
+  If the simulation was successful, proceed to the next troubleshooting step.
+  ### 2. Check if the user session can be simulated accurately *against the environment used for test runs*
+  To verify whether the user session can be simulated accurately in the environment used for test runs, you can:
+  1. Navigate back to the session you selected in the previous step.
+  2. Click on the **Simulations** tab to find a recent simulation in CI. You'll be able to tell it's a CI simulation since the 'Simulated Against'
+  URL will show a Meticulous secure tunnel URL (\`*.tunnels.meticulous.ai\`) or a Vercel, Netlify or Cloudflare preview URL.
+  3. Select one of the simulations and follow the command to "Re-run the simulation with the Meticulous CLI" on the **${SIMULATION_TAB_NAMES.DEBUG_LOCALLY}** tab of the simulation page.
+  If you're triggering Meticulous tests in CI against Vercel, Netlify or Cloudflare preview URLs then you can run this command as is:
+  the \`--appUrl\` it is set to run against will most likely still be a valid preview URL.
+  If however you're triggering Meticulous tests in
+  CI against a Meticulous secure tunnel URL or a localhost server running in your CI runner then you'll need to edit the \`--appUrl\` to point to a valid URL. If you're
+  using the cloud replay action you can get a URL to run against by opening a new PR with \`${METICULOUS_DEBUG_PR_LABEL}\` in the title. This
+  will keep the secure tunnel open for debugging. Meticulous will post a comment on your PR on how to access the secure tunnel to debug
+  your application setup, and you can use this as the \`--appUrl\` to replay the simulation against. It will also post a username and password
+  which you must make available as \`METICULOUS_TUNNEL_USERNAME\` and \`METICULOUS_TUNNEL_PASSWORD\` environment variables when running the Meticulous
+  CLI.
+  Look out for any of the symptoms described in step 1. If the simulation fails when simulating against the test run environment but not when simulating against the original recording
+  environment then the issue is likely environmental differences between the recording and test run environments. For more
+  information on how to debug environmental differences, see [here](${FIX_FALSE_POSITIVES_URL}).
+  If the simulation was successful, please get in touch by emailing [${METICULOUS_SUPPORT_EMAIL}](mailto:${METICULOUS_SUPPORT_EMAIL}), or
+  [book a call with us](${METICULOUS_SETUP_CALENDLY_LINK}), and we will help you debug this issue.
+  ### 3. View the simulation timeline to see why it failed to replay
+  Find the simulation in the Meticulous web UI and click on the **${SIMULATION_TAB_NAMES.TIMELINE_AND_LOGS}** tab. From here you can zoom in using the scroll wheel and
+  pan by dragging. You'll be able to see each screenshot taken, each click, each console log recorded, and each network request made. Failures
+  are highlighted in red.
+  Alternatively, simulating a session locally via the **${SIMULATION_TAB_NAMES.DEBUG_LOCALLY}** tab with the flags \`--debugger --devTools\`
+  will let you step through the simulation one user event at a time and pinpoint exactly where the issue is occurring.
+  `;

package/dist/commands/onboard/templates/docs/how-to/typescript-types.ts ADDED Viewed

@@ -0,0 +1,51 @@
+export const document = `---
+{
+  "title": "TypeScript Types for window.Meticulous"
+}
+---
+# {% $frontmatter.title %}
+TypeScript definitions for the \`window.Meticulous\` object are available in the [\`@alwaysmeticulous/sdk-bundles-api\`](https://www.npmjs.com/package/@alwaysmeticulous/sdk-bundles-api) package.
+## Installation
+Install the package as a dev dependency:
+\`\`\`bash
+npm install --save-dev @alwaysmeticulous/sdk-bundles-api
+\`\`\`
+## Usage
+Import the type and extend the Window interface:
+\`\`\`typescript
+import type { MeticulousPublicApi } from '@alwaysmeticulous/sdk-bundles-api';
+declare global {
+  interface Window {
+    Meticulous?: MeticulousPublicApi;
+  }
+}
+\`\`\`
+Now you have full type safety when using the \`window.Meticulous\` object:
+\`\`\`typescript
+// Detect if running as a test
+if (window.Meticulous?.isRunningAsTest) {
+  console.log('Running as a Meticulous test');
+}
+// Record session context with type safety
+window.Meticulous?.context.recordUserId('user-123');
+window.Meticulous?.context.recordUserEmail('user@example.com');
+window.Meticulous?.context.recordFeatureFlag('myFlag', true);
+window.Meticulous?.context.recordCustomContext('userRole', 'admin');
+\`\`\`
+## Full API Reference
+For the complete type definition, see [public-window-api.ts](https://github.com/alwaysmeticulous/meticulous-sdk/blob/main/packages/sdk-bundles-api/src/window-api/public-window-api.ts) in the meticulous-sdk repository.
+`;