@xoxoai/checkmate 0.4.0 → 0.4.2

package/package.json

@@ -1,11 +1,11 @@
 {
 	"name": "@xoxoai/checkmate",
-	"version": "0.4.0",
+	"version": "0.4.2",
 	"description": "AI‑driven e2e test automation framework built on Playwright Test and OpenAI API",
 	"homepage": "https://github.com/dawiddiwad/checkmate",
 	"repository": {
 		"type": "git",
-		"url": "https://github.com/dawiddiwad/checkmate.git"
+		"url": "git+https://github.com/dawiddiwad/checkmate.git"
 	},
 	"keywords": [
 		"test-automation",
@@ -20,13 +20,19 @@
 		"groq"
 	],
 	"files": [
+		"bin",
 		"dist",
 		"docs/EXTENSIONS.md",
 		"docs/GUIDE.md",
 		"docs/ROADMAP.md",
+		"playwright.config.ts",
 		"README.md",
+		"test/examples",
 		"LICENSE"
 	],
+	"bin": {
+		"checkmate": "bin/checkmate.js"
+	},
 	"publishConfig": {
 		"access": "public"
 	},
@@ -102,5 +108,12 @@
 		"typescript": "^5.9.3",
 		"typescript-eslint": "^8.58.2",
 		"vitest": "^4.1.2"
+	},
+	"directories": {
+		"doc": "docs",
+		"test": "test"
+	},
+	"bugs": {
+		"url": "https://github.com/dawiddiwad/checkmate/issues"
 	}
 }

package/playwright.config.ts

@@ -0,0 +1,39 @@
+import { defineConfig } from '@playwright/test'
+import { config as envConfig } from 'dotenv'
+
+envConfig({ quiet: true })
+
+export default defineConfig({
+	projects: [
+		{
+			name: 'salesforce',
+			testDir: './test/examples/salesforce',
+		},
+		{
+			name: 'web',
+			testDir: './test/examples/web',
+		},
+	],
+	outputDir: process.env.CI ? undefined : './test-reports/results',
+	reporter: [
+		['junit', { outputFile: './test-reports/junit/results.xml' }],
+		['html', { outputFolder: './test-reports/html' }],
+		['list'],
+	],
+	timeout: 10 * 60000,
+	repeatEach: 1,
+	retries: 1,
+	workers: 1,
+	expect: {
+		timeout: 1 * 10000,
+	},
+	use: {
+		viewport: { width: 1360, height: 768 },
+		browserName: 'chromium',
+		actionTimeout: 1 * 5000,
+		navigationTimeout: 1 * 30000,
+		screenshot: 'only-on-failure',
+		trace: 'retain-on-failure',
+		video: 'on',
+	},
+})

package/test/examples/salesforce/trial-dev-org.spec.ts

@@ -0,0 +1,82 @@
+/**
+ * @fileoverview
+ * Playwright E2E test for Salesforce Developer org.
+ *
+ * @summary
+ * Example of a multi-step test that creates a new Account record in the Sales app.
+ *
+ * @description
+ * This test automates the following user journey in a Salesforce Developer or trial org:
+ * 1. Log in to the Salesforce org.
+ * 2. Open the Sales app via the App Launcher.
+ * 3. Navigate to the Accounts tab.
+ * 4. Create a new Account with a random name.
+ * 5. Save the new Account record.
+ *
+ * @preconditions
+ * - A Salesforce Developer or trial org is required. Sign-up: https://www.salesforce.com/form/developer-signup/
+ * - Salesforce CLI is recommended for authorizing the org: https://developer.salesforce.com/tools/salesforcecli
+ * - The org must be authorized (for example: `sf org login web --set-default`) before running the test.
+ * - The user executing the test should have access to the Sales app and the Accounts tab in Lightning Experience.
+ *
+ * @see {@link https://developer.salesforce.com/tools/salesforcecli} - Salesforce CLI installation and documentation.
+ * @see {@link https://www.salesforce.com/form/developer-signup/} - Sign up for a Salesforce Developer org.
+ *
+ * @note
+ * All tests use the `ai` fixture and call `ai.run({ action, expect })`
+ * to describe actions and assert visible outcomes.
+ */
+import { test } from '@xoxoai/checkmate/salesforce'
+
+test.describe('trial dev org', async () => {
+	test('creating new account in sales app', async ({ ai }) => {
+		await test.step('Login to Salesforce Org', async () => {
+			await ai.run({
+				action: `
+               	Login to Salesforce org`,
+				expect: `
+               	Salesforce org loads successfully and user is authenticated/not on the login page.`,
+			})
+		})
+
+		await test.step('Open Sales App from App Launcher', async () => {
+			await ai.run({
+				action: `
+				Click the App Launcher icon (nine dots) in the top left corner.
+				Type 'Sales' into the App Launcher 'Search apps and items' search bar.
+				Click on the app that is named exactly the 'Sales' app from the results.`,
+				expect: `
+				Sales app opens successfully in Lightning context.`,
+			})
+		})
+
+		await test.step('Switch to Accounts tab', async () => {
+			await ai.run({
+				action: `
+				Click the 'Accounts' tab within the Sales app.`,
+				expect: `
+				The Accounts tab is active and a list of accounts is displayed.`,
+			})
+		})
+
+		await test.step('Start creating a new Account', async () => {
+			await ai.run({
+				action: `
+				Click the 'New' button on the Accounts tab to create a new Account record.
+				Fill 'Account Name' field with 'Agentic Test Account' followed by space and some random alphanumeric string
+				Don't save the record or fill any other fields.`,
+				expect: `
+				'New Account' form is displayed and filled with random data`,
+			})
+		})
+
+		await test.step('Save new Account record', async () => {
+			await ai.run({
+				action: `
+				Click the 'Save' button on the 'New Account' form.`,
+				expect: `
+				Account record was saved successfully and details view is displayed.`,
+			})
+		})
+	})
+})

package/test/examples/web/website-testing.spec.ts

@@ -0,0 +1,281 @@
+/**
+ * @fileoverview
+ * Playwright E2E tests for various websites.
+ *
+ * @summary
+ * Examples of single-step and multi-step AI-driven web interactions using Playwright and _checkmate_.
+ *
+ * @description
+ * - Single-step:
+ *   - Ollama: search for "qwen3" and open the qwen3-vl model page.
+ *   - HuggingFace: search for "Qwen3-VL-4B" and open the model page.
+ *   - Mojeek: search Playwright docs and open the Planner Agent page.
+ *   - NYPL: check availability for "The Catcher in the Rye" - with and without fuzzy search.
+ *
+ * - Multi-step (polymer-shop):
+ *   - Full AI mode: all steps executed via `ai.run`.
+ *   - Hybrid mode: mix of `ai.run` steps and direct Playwright commands.
+ *
+ * @note
+ * All tests use the `ai` fixture and call `ai.run({ action, expect })`
+ * to describe actions and assert visible outcomes.
+ */
+import { test } from '@xoxoai/checkmate/playwright'
+
+test.describe('single-step : quick examples', async () => {
+	test('browsing ollama models', async ({ ai }) => {
+		await test.step('Navigate to ollama.com and search for model details', async () => {
+			await ai.run({
+				action: `
+                Navigate to https://ollama.com
+                Type 'qwen3-vl' into the 'Search models' search bar
+                Click on 'qwen3-vl' link from the results,
+                Click on 'qwen3-vl:235b' link from the models list,`,
+				expect: `
+				qwen3-vl:235b model page is displayed with model details,
+                describing its features and capabilities.`,
+			})
+		})
+	})
+
+	test('browsing huggingface models', async ({ ai }) => {
+		await test.step('Navigate to huggingface.co', async () => {
+			await ai.run({
+				action: `
+				Navigate to the https://huggingface.co website.
+				Type 'Qwen3-VL-4B' in the search bar.
+				Click on the 'Qwen/Qwen3-VL-4B-Instruct' link from the search results.`,
+				expect: `
+				Qwen3-VL-4B-Instruct model page is displayed with model details`,
+			})
+		})
+	})
+
+	test('searching with mojeek', async ({ ai }) => {
+		await test.step('Search for Playwright docs using mojeek.com', async () => {
+			await ai.run({
+				action: `
+                Test: Mojeek Search
+                Navigate to the Mojeek search engine at https://www.mojeek.com/.
+                In the search field, type playwright test automation and press Enter.
+                From the search results, click on the link that goes to playwright.dev.
+                Once on the Playwright website, click the site's search icon and type Agent into the Search docs field.
+                Click on the Planner result displayed in the documentation search.
+                `,
+				expect: `
+				The Planner Agent documentation page is displayed, providing details on how to use this agent within Playwright.`,
+			})
+		})
+	})
+
+	test('searching new york public library', async ({ ai }) => {
+		await test.step('the catcher in the rye availability', async () => {
+			await ai.run({
+				action: `
+                Navigate to https://www.nypl.org. 
+                Search for 'The Catcher in the Rye'.`,
+				expect: `
+				'The Catcher in the Rye' physical branch's bookshelf availability information is displayed.`,
+			})
+		})
+	})
+
+	test('searching new york public library - fuzzy search enabled', async ({ ai }) => {
+		await test.step('the catcher in the rye availability', async () => {
+			await ai.run({
+				action: `
+                Navigate to https://www.nypl.org. 
+                Search for 'The Catcher in the Rye'.`,
+				expect: `
+				'The Catcher in the Rye' physical branch's bookshelf availability information is displayed.`,
+
+				search: ['search', 'the catcher in the rye', 'shelf'],
+				topPercent: 10, // aggressive snapshot filtering (top 10% of elements on page)
+			})
+		})
+	})
+})
+
+test.describe('multi-step : Polymer Shop : full AI mode', async () => {
+	test.beforeEach(async ({ ai }) => {
+		await ai.run({
+			action: `Navigate to https://shop.polymer-project.org/`,
+			expect: `Polymer Shop home page is loaded`,
+		})
+	})
+
+	test('successful end-to-end purchase flow', async ({ ai }) => {
+		await test.step('Navigate to shop and select product', async () => {
+			await ai.run({
+				action: `
+				Click "Shop Now" on "Men's Outerwear" category
+				Click on the first product: "Men's Tech Shell Full-Zip"`,
+				expect: `Product detail page is displayed with title, price, and selectors`,
+			})
+		})
+
+		await test.step('Add product to cart and proceed to checkout', async () => {
+			await ai.run({
+				action: `
+				Click "Add to Cart"
+				Click "Checkout" in the "Added to cart" dialog`,
+				expect: `Checkout page loads showing the 'Order Summary' with correct total`,
+			})
+		})
+
+		await test.step('Fill checkout details and place order', async () => {
+			await ai.run({
+				action: `
+				Fill in the required fields in one go:
+				- Email: 'test@example.com'
+				- Phone: '1234567890'
+				- Address: '123 Test St'
+				- City: 'Test City'
+				- State: 'Test State'
+				- Zip: '12345'
+				- Country: 'Canada'
+				- Cardholder: 'Test User'
+				- Card Number: '1111222233334444'
+				- Expiry month: 'Dec'
+				- Expiry year: '2026'
+				- CVV: '123'
+				Click "Place Order"`,
+				expect: `Checkout process completed message is displayed along with Finish button`,
+			})
+		})
+
+		await test.step('Finish and return to home', async () => {
+			await ai.run({
+				action: `Finish and return to home`,
+				expect: `User is returned to the home page`,
+			})
+		})
+	})
+
+	test('cart management - adding multiple items & verifying total', async ({ ai }) => {
+		await test.step('Add items from different categories', async () => {
+			await ai.run({
+				action: `
+				Go to 'Ladies Outerwear' and add an item to the cart.
+				Press page up to return to the top.
+				Go back to home page, navigate to 'Men's T-Shirts' and add an item to the cart.`,
+				expect: `Cart counter in the header updates to '2'`,
+			})
+		})
+
+		await test.step('Verify cart total', async () => {
+			await ai.run({
+				action: `Click the Cart icon in the header.`,
+				expect: `The 'Your Cart' page shows both items and the 'Total' matches the sum of individual prices.`,
+			})
+		})
+	})
+
+	test('cart management - updating quantity and removing items', async ({ ai }) => {
+		await test.step('Add item and update quantity', async () => {
+			await ai.run({
+				action: `
+				Go to 'Men's Outerwear' and add an item to cart, then click "View Cart".
+				Select "2" from the "Quantity" dropdown for the item.`,
+				expect: `The quantity is updated and total price reflects the change.`,
+			})
+		})
+
+		await test.step('Remove item from cart', async () => {
+			await ai.run({
+				action: `Click the 'Delete' (trash icon) button for the item.`,
+				expect: `The item is removed, cart shows 'Your cart is empty', and header counter is '0'.`,
+			})
+		})
+	})
+
+	test('category navigation & product display consistency', async ({ ai }) => {
+		await test.step('Navigate between categories', async () => {
+			await ai.run({
+				action: `
+				Click "Ladies T-Shirts" in the top navigation menu.
+				Click "Men's Outerwear" in the top navigation menu.`,
+				expect: `User can navigate between categories correctly.`,
+			})
+		})
+
+		await test.step('Return to home via SHOP logo', async () => {
+			await ai.run({
+				action: `Click "SHOP" logo in the header.`,
+				expect: `User is returned to the home page.`,
+			})
+		})
+	})
+
+	test('checkout form validation (error handling)', async ({ ai }) => {
+		await test.step('Submit empty checkout form', async () => {
+			await ai.run({
+				action: `
+				Add any item to cart and proceed to "Checkout".
+				Click "Place Order" without filling any fields.`,
+				expect: `Validation error messages like "Invalid Email" and "Invalid Address" are visible.`,
+			})
+		})
+
+		await test.step('Verify persistent errors for invalid input', async () => {
+			await ai.run({
+				action: `Type 'invalid-email' in the Email field and click "Place Order".`,
+				expect: `"Invalid Email" error message is still displayed.`,
+			})
+		})
+	})
+})
+
+test.describe('multi-step : Polymer Shop : hybrid mode', async () => {
+	test.beforeEach(async ({ page }) => {
+		await page.goto('https://shop.polymer-project.org/')
+	})
+
+	test('successful end-to-end purchase flow', async ({ page, ai }) => {
+		await test.step('Navigate to shop and select product', async () => {
+			await page.getByRole('link', { name: "Men's Outerwear Shop Now" }).click()
+			await page.getByRole('link', { name: "Men's Tech Shell Full-Zip Men" }).click()
+			await ai.run({
+				action: ``,
+				expect: `Product detail page is displayed with title, price, and selectors`,
+			})
+		})
+
+		await test.step('Add product to cart and proceed to checkout', async () => {
+			await page.getByRole('button', { name: 'Add this item to cart' }).click()
+			await page.getByRole('link', { name: 'Checkout' }).click()
+			await ai.run({
+				action: ``,
+				expect: `Checkout page loads showing the 'Order Summary' with correct total`,
+			})
+		})
+
+		await test.step('Fill checkout details and place order', async () => {
+			await ai.run({
+				action: `
+				Fill in the required fields in one go:
+				- Email: 'test@example.com'
+				- Phone: '1234567890'
+				- Address: '123 Test St'
+				- City: 'Test City'
+				- State: 'Test State'
+				- Zip: '12345'
+				- Country: 'Canada'
+				- Cardholder: 'Test User'
+				- Card Number: '1111222233334444'
+				- Expiry month: 'Dec'
+				- Expiry year: '2026'
+				- CVV: '123'
+				Click "Place Order"`,
+				expect: `Demo checkout process completed message is displayed along with Finish button`,
+			})
+		})
+
+		await test.step('Finish and return to home', async () => {
+			await ai.run({
+				action: `Finish and return to home`,
+				expect: `User is returned to the home page`,
+			})
+		})
+	})
+})