@sayer/agent-step 0.1.0

package/dist/agent-step.js

@@ -0,0 +1,68 @@
+import { test } from '@playwright/test';
+import { AgentRuntime, AgentStepError } from './agent/agent-runtime.js';
+import { redactObject } from './agent/redaction.js';
+export function createAgentStep({ page, testInfo, }) {
+    const runtime = new AgentRuntime({ page, testInfo });
+    return async (input) => {
+        testInfo.annotations.push({
+            type: 'agent-expect',
+            description: input.expect.join(' | '),
+        });
+        return test.step(`agent: ${input.action}`, async (step) => {
+            try {
+                const result = await runtime.runStep(input);
+                await step.attach('agent-transcript.json', {
+                    body: JSON.stringify(redactObject(result.actionTranscript, input.secrets ?? {}), null, 2),
+                    contentType: 'application/json',
+                });
+                await step.attach('agent-verification.json', {
+                    body: JSON.stringify(result.verification, null, 2),
+                    contentType: 'application/json',
+                });
+                if (result.rawVerifierContent) {
+                    await step.attach('agent-verifier-raw.json', {
+                        body: result.rawVerifierContent,
+                        contentType: 'application/json',
+                    });
+                }
+                await step.attach('agent-token-usage.json', {
+                    body: JSON.stringify(result.tokenUsage, null, 2),
+                    contentType: 'application/json',
+                });
+                return result;
+            }
+            catch (error) {
+                const screenshot = await page
+                    .screenshot({ fullPage: true })
+                    .catch(() => null);
+                if (screenshot) {
+                    await step.attach('agent-failure.png', {
+                        body: screenshot,
+                        contentType: 'image/png',
+                    });
+                }
+                if (error instanceof AgentStepError && error.result) {
+                    await step.attach('agent-transcript.json', {
+                        body: JSON.stringify(redactObject(error.result.actionTranscript, input.secrets ?? {}), null, 2),
+                        contentType: 'application/json',
+                    });
+                    await step.attach('agent-verification.json', {
+                        body: JSON.stringify(error.result.verification, null, 2),
+                        contentType: 'application/json',
+                    });
+                    if (error.result.rawVerifierContent) {
+                        await step.attach('agent-verifier-raw.json', {
+                            body: error.result.rawVerifierContent,
+                            contentType: 'application/json',
+                        });
+                    }
+                }
+                throw error;
+            }
+        });
+    };
+}
+export async function agentStep(page, input) {
+    return createAgentStep({ page, testInfo: test.info() })(input);
+}
+//# sourceMappingURL=agent-step.js.map

package/dist/agent-step.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-step.js","sourceRoot":"","sources":["../src/agent-step.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAA4B,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAUpD,MAAM,UAAU,eAAe,CAAC,EAC9B,IAAI,EACJ,QAAQ,GACe;IACvB,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAErD,OAAO,KAAK,EAAE,KAAK,EAAE,EAAE;QACrB,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC;YACxB,IAAI,EAAE,cAAc;YACpB,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;SACtC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACxD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAE5C,MAAM,IAAI,CAAC,MAAM,CAAC,uBAAuB,EAAE;oBACzC,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB,YAAY,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,EAC1D,IAAI,EACJ,CAAC,CACF;oBACD,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;gBAEH,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,EAAE;oBAC3C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;oBAClD,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;gBAEH,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;oBAC9B,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,EAAE;wBAC3C,IAAI,EAAE,MAAM,CAAC,kBAAkB;wBAC/B,WAAW,EAAE,kBAAkB;qBAChC,CAAC,CAAC;gBACL,CAAC;gBAED,MAAM,IAAI,CAAC,MAAM,CAAC,wBAAwB,EAAE;oBAC1C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;oBAChD,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;gBAEH,OAAO,MAAM,CAAC;YAChB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,UAAU,GAAG,MAAM,IAAI;qBAC1B,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;qBAC9B,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;gBACrB,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE;wBACrC,IAAI,EAAE,UAAU;wBAChB,WAAW,EAAE,WAAW;qBACzB,CAAC,CAAC;gBACL,CAAC;gBAED,IAAI,KAAK,YAAY,cAAc,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;oBACpD,MAAM,IAAI,CAAC,MAAM,CAAC,uBAAuB,EAAE;wBACzC,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,EAAE,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,EAChE,IAAI,EACJ,CAAC,CACF;wBACD,WAAW,EAAE,kBAAkB;qBAChC,CAAC,CAAC;oBACD,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,EAAE;wBAC3C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;wBACxD,WAAW,EAAE,kBAAkB;qBAChC,CAAC,CAAC;oBACH,IAAI,KAAK,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;wBACpC,MAAM,IAAI,CAAC,MAAM,CAAC,yBAAyB,EAAE;4BAC3C,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,kBAAkB;4BACrC,WAAW,EAAE,kBAAkB;yBAChC,CAAC,CAAC;oBACL,CAAC;gBACL,CAAC;gBAED,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAU,EACV,KAAqB;IAErB,OAAO,eAAe,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;AACjE,CAAC","sourcesContent":["import { test, type Page, type TestInfo } from '@playwright/test';\nimport { AgentRuntime, AgentStepError } from './agent/agent-runtime.js';\nimport { redactObject } from './agent/redaction.js';\nimport type { AgentStepInput, AgentStepResult } from './agent/types.js';\n\nexport type AgentStep = (input: AgentStepInput) => Promise<AgentStepResult>;\n\nexport interface CreateAgentStepOptions {\n  page: Page;\n  testInfo: TestInfo;\n}\n\nexport function createAgentStep({\n  page,\n  testInfo,\n}: CreateAgentStepOptions): AgentStep {\n  const runtime = new AgentRuntime({ page, testInfo });\n\n  return async (input) => {\n    testInfo.annotations.push({\n      type: 'agent-expect',\n      description: input.expect.join(' | '),\n    });\n\n    return test.step(`agent: ${input.action}`, async (step) => {\n      try {\n        const result = await runtime.runStep(input);\n\n        await step.attach('agent-transcript.json', {\n          body: JSON.stringify(\n            redactObject(result.actionTranscript, input.secrets ?? {}),\n            null,\n            2,\n          ),\n          contentType: 'application/json',\n        });\n\n        await step.attach('agent-verification.json', {\n          body: JSON.stringify(result.verification, null, 2),\n          contentType: 'application/json',\n        });\n\n        if (result.rawVerifierContent) {\n          await step.attach('agent-verifier-raw.json', {\n            body: result.rawVerifierContent,\n            contentType: 'application/json',\n          });\n        }\n\n        await step.attach('agent-token-usage.json', {\n          body: JSON.stringify(result.tokenUsage, null, 2),\n          contentType: 'application/json',\n        });\n\n        return result;\n      } catch (error) {\n        const screenshot = await page\n          .screenshot({ fullPage: true })\n          .catch(() => null);\n        if (screenshot) {\n          await step.attach('agent-failure.png', {\n            body: screenshot,\n            contentType: 'image/png',\n          });\n        }\n\n        if (error instanceof AgentStepError && error.result) {\n          await step.attach('agent-transcript.json', {\n            body: JSON.stringify(\n              redactObject(error.result.actionTranscript, input.secrets ?? {}),\n              null,\n              2,\n            ),\n            contentType: 'application/json',\n          });\n            await step.attach('agent-verification.json', {\n              body: JSON.stringify(error.result.verification, null, 2),\n              contentType: 'application/json',\n            });\n            if (error.result.rawVerifierContent) {\n              await step.attach('agent-verifier-raw.json', {\n                body: error.result.rawVerifierContent,\n                contentType: 'application/json',\n              });\n            }\n        }\n\n        throw error;\n      }\n    });\n  };\n}\n\nexport async function agentStep(\n  page: Page,\n  input: AgentStepInput,\n): Promise<AgentStepResult> {\n  return createAgentStep({ page, testInfo: test.info() })(input);\n}\n"]}

package/dist/agent-test.d.ts

@@ -0,0 +1,10 @@
+import type { Fixtures, Page } from '@playwright/test';
+import { type AgentStep } from './agent-step.js';
+export type AgentStepFixtures = {
+    agentStep: AgentStep;
+};
+export type AgentFixtures = AgentStepFixtures;
+export declare const agentStepFixture: Fixtures<AgentStepFixtures, {}, {
+    page: Page;
+}>;
+//# sourceMappingURL=agent-test.d.ts.map

package/dist/agent-test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-test.d.ts","sourceRoot":"","sources":["../src/agent-test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAmB,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAElE,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,SAAS,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,iBAAiB,CAAC;AAE9C,eAAO,MAAM,gBAAgB,EAAE,QAAQ,CACrC,iBAAiB,EACjB,EAAE,EACF;IAAE,IAAI,EAAE,IAAI,CAAA;CAAE,CAKf,CAAC"}

package/dist/agent-test.js

@@ -0,0 +1,7 @@
+import { createAgentStep } from './agent-step.js';
+export const agentStepFixture = {
+    agentStep: async ({ page }, use, testInfo) => {
+        await use(createAgentStep({ page, testInfo }));
+    },
+};
+//# sourceMappingURL=agent-test.js.map

package/dist/agent-test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-test.js","sourceRoot":"","sources":["../src/agent-test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAkB,MAAM,iBAAiB,CAAC;AAQlE,MAAM,CAAC,MAAM,gBAAgB,GAIzB;IACF,SAAS,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;QAC3C,MAAM,GAAG,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;IACjD,CAAC;CACF,CAAC","sourcesContent":["import type { Fixtures, Page } from '@playwright/test';\nimport { createAgentStep, type AgentStep } from './agent-step.js';\n\nexport type AgentStepFixtures = {\n  agentStep: AgentStep;\n};\n\nexport type AgentFixtures = AgentStepFixtures;\n\nexport const agentStepFixture: Fixtures<\n  AgentStepFixtures,\n  {},\n  { page: Page }\n> = {\n  agentStep: async ({ page }, use, testInfo) => {\n    await use(createAgentStep({ page, testInfo }));\n  },\n};\n"]}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { agentStep, createAgentStep, type AgentStep, type CreateAgentStepOptions, } from './agent-step.js';
+export { agentStepFixture, type AgentFixtures, type AgentStepFixtures, } from './agent-test.js';
+export type { AgentStepInput, AgentStepResult, CriterionVerdict, TokenUsage, TranscriptEntry, VerificationResult, } from './agent/types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,SAAS,EACT,eAAe,EACf,KAAK,SAAS,EACd,KAAK,sBAAsB,GAC5B,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,gBAAgB,EAChB,KAAK,aAAa,EAClB,KAAK,iBAAiB,GACvB,MAAM,iBAAiB,CAAC;AAEzB,YAAY,EACV,cAAc,EACd,eAAe,EACf,gBAAgB,EAChB,UAAU,EACV,eAAe,EACf,kBAAkB,GACnB,MAAM,kBAAkB,CAAC"}

package/dist/index.js

@@ -0,0 +1,3 @@
+export { agentStep, createAgentStep, } from './agent-step.js';
+export { agentStepFixture, } from './agent-test.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,SAAS,EACT,eAAe,GAGhB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,gBAAgB,GAGjB,MAAM,iBAAiB,CAAC","sourcesContent":["export {\n  agentStep,\n  createAgentStep,\n  type AgentStep,\n  type CreateAgentStepOptions,\n} from './agent-step.js';\n\nexport {\n  agentStepFixture,\n  type AgentFixtures,\n  type AgentStepFixtures,\n} from './agent-test.js';\n\nexport type {\n  AgentStepInput,\n  AgentStepResult,\n  CriterionVerdict,\n  TokenUsage,\n  TranscriptEntry,\n  VerificationResult,\n} from './agent/types.js';\n"]}

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@sayer/agent-step",
+  "version": "0.1.0",
+  "description": "Runtime agentic test steps for Playwright using plain-language actions and expectations",
+  "author": "Jack Sayer",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Sayer122/agent-step.git"
+  },
+  "bugs": {
+    "url": "https://github.com/Sayer122/agent-step/issues"
+  },
+  "homepage": "https://github.com/Sayer122/agent-step#readme",
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "playwright",
+    "testing",
+    "agentic",
+    "llm",
+    "e2e",
+    "automation"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "clean": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"",
+    "build": "npm run clean && tsc -p tsconfig.build.json",
+    "test": "playwright test",
+    "test:framework": "playwright test --project=framework",
+    "test:agentic": "playwright test --project=agentic",
+    "test:live": "playwright test --project=live",
+    "test:package": "npm run build && tsc -p examples/consumer/tsconfig.json && node --input-type=module -e \"const pkg = await import('@sayer/agent-step'); if (!pkg.agentStep || !pkg.createAgentStep || !pkg.agentStepFixture) throw new Error('Missing public exports')\"",
+    "typecheck": "tsc --noEmit",
+    "prepack": "npm run build",
+    "prepublishOnly": "npm run typecheck && npm run test:framework && npm run test:package"
+  },
+  "devDependencies": {
+    "@playwright/test": "^1.63.0",
+    "@types/node": "^22.10.0",
+    "typescript": "^5.7.0"
+  },
+  "peerDependencies": {
+    "@playwright/test": ">=1.63.0 <2"
+  },
+  "dependencies": {
+    "openai": "^4.77.0",
+    "zod": "^3.24.0"
+  }
+}