@agentmark-ai/cli 0.9.0 → 0.10.1

package/README.md

@@ -1 +1,143 @@
-# Agentmark CLI
+# AgentMark CLI
+
+The command-line tool for developing, testing, and evaluating AI agents with AgentMark.
+
+## Installation
+
+```bash
+npm install -g @agentmark-ai/cli
+```
+
+Or use it directly with `npx`:
+
+```bash
+npx @agentmark-ai/cli dev
+```
+
+## Quick Start
+
+```bash
+# Scaffold a new project
+npm create agentmark@latest
+
+# Start the dev server (API + trace UI + hot reload)
+agentmark dev
+
+# Run a prompt with its test props
+agentmark run-prompt my-prompt.prompt.mdx
+
+# Run an experiment against a dataset
+agentmark run-experiment my-prompt.prompt.mdx
+```
+
+## Commands
+
+### `agentmark dev`
+
+Start the local development environment: API server, webhook server, and trace UI.
+
+```bash
+agentmark dev
+agentmark dev --api-port 9418 --app-port 3000
+agentmark dev --remote    # Connect to AgentMark Cloud (login + trace forwarding)
+agentmark dev --tunnel    # Expose webhook server publicly
+```
+
+The dev server auto-detects your project language (TypeScript or Python), finds your virtual environment, and handles port conflicts automatically.
+
+### `agentmark run-prompt <filepath>`
+
+Execute a single prompt and display the result.
+
+```bash
+# Run with test props from the prompt's frontmatter
+agentmark run-prompt customer-support.prompt.mdx
+
+# Run with custom props
+agentmark run-prompt customer-support.prompt.mdx --props '{"customer_question": "Where is my order?"}'
+
+# Run with props from a file
+agentmark run-prompt customer-support.prompt.mdx --props-file test-data.json
+```
+
+Output includes the LLM response, token usage, cost, and a link to the trace in the local UI.
+
+### `agentmark run-experiment <filepath>`
+
+Run a prompt against every item in its dataset, with optional evaluations.
+
+```bash
+# Run with evals (default)
+agentmark run-experiment my-prompt.prompt.mdx
+
+# Skip evals
+agentmark run-experiment my-prompt.prompt.mdx --skip-eval
+
+# Output as JSON instead of table
+agentmark run-experiment my-prompt.prompt.mdx --format json
+
+# Fail if pass rate is below 80%
+agentmark run-experiment my-prompt.prompt.mdx --threshold 80
+```
+
+Output formats: `table` (default), `csv`, `json`, `jsonl`.
+
+### `agentmark build`
+
+Pre-compile `.prompt.mdx` files into JSON for production use with the file loader.
+
+```bash
+agentmark build
+agentmark build --out dist/prompts
+```
+
+### `agentmark generate-types`
+
+Generate TypeScript type definitions from your prompts for type-safe usage in code.
+
+```bash
+agentmark generate-types
+```
+
+### `agentmark generate-schema`
+
+Generate a JSON Schema for `.prompt.mdx` frontmatter, enabling IDE validation and autocomplete.
+
+```bash
+agentmark generate-schema
+agentmark generate-schema --out .agentmark
+```
+
+### `agentmark pull-models`
+
+Interactively select and add LLM models from a provider to your `agentmark.json`.
+
+```bash
+agentmark pull-models
+```
+
+### `agentmark login` / `agentmark logout`
+
+Authenticate with AgentMark Cloud.
+
+```bash
+agentmark login
+agentmark logout
+```
+
+### `agentmark link`
+
+Link your project to an AgentMark Cloud app for trace forwarding.
+
+```bash
+agentmark link
+agentmark link --app-id <uuid>
+```
+
+## Documentation
+
+Full documentation at [docs.agentmark.co](https://docs.agentmark.co/agentmark/).
+
+## License
+
+[MIT](../../LICENSE.md)

package/dist/.next/BUILD_ID

@@ -1 +1 @@
-_uH98Yva4C-QS08PL-_0P
+2NZm040KzCzDPOX6MqoKG

package/dist/.next/app-build-manifest.json

@@ -12,9 +12,47 @@
       "static/chunks/87c73c54-09e1ba5c70e60a51.js",
       "static/chunks/18-175d23c8e99dd60b.js",
       "static/chunks/main-app-0b5725cb98f081b4.js",
-      "static/chunks/902-8d8c521535a2c32f.js",
-      "static/chunks/602-1f23cff16de7187a.js",
-      "static/chunks/app/layout-eaddb6da201ab019.js"
+      "static/chunks/860-75914efffd5add16.js",
+      "static/chunks/757-19ef6a21d87937d9.js",
+      "static/chunks/866-87ad78bf0bef953a.js",
+      "static/chunks/app/layout-3af953219517dd3e.js"
+    ],
+    "/experiments/[id]/page": [
+      "static/chunks/webpack-f10b0ea54196b48e.js",
+      "static/chunks/87c73c54-09e1ba5c70e60a51.js",
+      "static/chunks/18-175d23c8e99dd60b.js",
+      "static/chunks/main-app-0b5725cb98f081b4.js",
+      "static/chunks/860-75914efffd5add16.js",
+      "static/chunks/729-21a669506b8ae33c.js",
+      "static/chunks/444-d88bd1e37e750aee.js",
+      "static/chunks/352-d7aa9d2da963e9e5.js",
+      "static/chunks/757-19ef6a21d87937d9.js",
+      "static/chunks/181-396b329bd35e4de0.js",
+      "static/chunks/app/experiments/[id]/page-1522aa0a86d6d004.js"
+    ],
+    "/experiments/compare/page": [
+      "static/chunks/webpack-f10b0ea54196b48e.js",
+      "static/chunks/87c73c54-09e1ba5c70e60a51.js",
+      "static/chunks/18-175d23c8e99dd60b.js",
+      "static/chunks/main-app-0b5725cb98f081b4.js",
+      "static/chunks/860-75914efffd5add16.js",
+      "static/chunks/729-21a669506b8ae33c.js",
+      "static/chunks/352-d7aa9d2da963e9e5.js",
+      "static/chunks/757-19ef6a21d87937d9.js",
+      "static/chunks/181-396b329bd35e4de0.js",
+      "static/chunks/app/experiments/compare/page-b99f5e9ebb5488b4.js"
+    ],
+    "/experiments/page": [
+      "static/chunks/webpack-f10b0ea54196b48e.js",
+      "static/chunks/87c73c54-09e1ba5c70e60a51.js",
+      "static/chunks/18-175d23c8e99dd60b.js",
+      "static/chunks/main-app-0b5725cb98f081b4.js",
+      "static/chunks/860-75914efffd5add16.js",
+      "static/chunks/729-21a669506b8ae33c.js",
+      "static/chunks/444-d88bd1e37e750aee.js",
+      "static/chunks/757-19ef6a21d87937d9.js",
+      "static/chunks/49-6efbcf2b4f20d950.js",
+      "static/chunks/app/experiments/page-e17f8133f9467da7.js"
     ],
     "/page": [
       "static/chunks/webpack-f10b0ea54196b48e.js",
@@ -31,10 +69,14 @@
       "static/css/a00826fc162bd22d.css",
       "static/chunks/baeaa4ff-f55976a85e939da3.js",
       "static/chunks/6b3bb038-04520a4579611038.js",
-      "static/chunks/902-8d8c521535a2c32f.js",
-      "static/chunks/397-361c64519c49911b.js",
-      "static/chunks/41-f3c56d50aeb34db9.js",
-      "static/chunks/app/requests/page-91666bcc2877f9ee.js"
+      "static/chunks/860-75914efffd5add16.js",
+      "static/chunks/729-21a669506b8ae33c.js",
+      "static/chunks/444-d88bd1e37e750aee.js",
+      "static/chunks/352-d7aa9d2da963e9e5.js",
+      "static/chunks/49-6efbcf2b4f20d950.js",
+      "static/chunks/349-9266e48c00c8ac76.js",
+      "static/chunks/41-f512c46d7ea943f6.js",
+      "static/chunks/app/requests/page-aa9725ed8b67059b.js"
     ],
     "/sessions/page": [
       "static/chunks/webpack-f10b0ea54196b48e.js",
@@ -44,10 +86,14 @@
       "static/css/a00826fc162bd22d.css",
       "static/chunks/baeaa4ff-f55976a85e939da3.js",
       "static/chunks/6b3bb038-04520a4579611038.js",
-      "static/chunks/902-8d8c521535a2c32f.js",
-      "static/chunks/397-361c64519c49911b.js",
-      "static/chunks/41-f3c56d50aeb34db9.js",
-      "static/chunks/app/sessions/page-679425471f82f56d.js"
+      "static/chunks/860-75914efffd5add16.js",
+      "static/chunks/729-21a669506b8ae33c.js",
+      "static/chunks/444-d88bd1e37e750aee.js",
+      "static/chunks/352-d7aa9d2da963e9e5.js",
+      "static/chunks/49-6efbcf2b4f20d950.js",
+      "static/chunks/349-9266e48c00c8ac76.js",
+      "static/chunks/41-f512c46d7ea943f6.js",
+      "static/chunks/app/sessions/page-bf2019c391763736.js"
     ],
     "/traces/page": [
       "static/chunks/webpack-f10b0ea54196b48e.js",
@@ -57,10 +103,14 @@
       "static/css/a00826fc162bd22d.css",
       "static/chunks/baeaa4ff-f55976a85e939da3.js",
       "static/chunks/6b3bb038-04520a4579611038.js",
-      "static/chunks/902-8d8c521535a2c32f.js",
-      "static/chunks/397-361c64519c49911b.js",
-      "static/chunks/41-f3c56d50aeb34db9.js",
-      "static/chunks/app/traces/page-15500a82980abfc8.js"
+      "static/chunks/860-75914efffd5add16.js",
+      "static/chunks/729-21a669506b8ae33c.js",
+      "static/chunks/444-d88bd1e37e750aee.js",
+      "static/chunks/352-d7aa9d2da963e9e5.js",
+      "static/chunks/49-6efbcf2b4f20d950.js",
+      "static/chunks/349-9266e48c00c8ac76.js",
+      "static/chunks/41-f512c46d7ea943f6.js",
+      "static/chunks/app/traces/page-c43b6aa5cd13fdf1.js"
     ]
   }
 }

package/dist/.next/app-path-routes-manifest.json

@@ -1,5 +1,8 @@
 {
   "/_not-found/page": "/_not-found",
+  "/experiments/[id]/page": "/experiments/[id]",
+  "/experiments/compare/page": "/experiments/compare",
+  "/experiments/page": "/experiments",
   "/page": "/",
   "/requests/page": "/requests",
   "/sessions/page": "/sessions",

package/dist/.next/build-manifest.json

@@ -5,8 +5,8 @@
   "devFiles": [],
   "ampDevFiles": [],
   "lowPriorityFiles": [
-    "static/_uH98Yva4C-QS08PL-_0P/_buildManifest.js",
-    "static/_uH98Yva4C-QS08PL-_0P/_ssgManifest.js"
+    "static/2NZm040KzCzDPOX6MqoKG/_buildManifest.js",
+    "static/2NZm040KzCzDPOX6MqoKG/_ssgManifest.js"
   ],
   "rootMainFiles": [
     "static/chunks/webpack-f10b0ea54196b48e.js",

package/dist/.next/cache/.previewinfo

@@ -1 +1 @@
-{"previewModeId":"0265d9bac88f8f6ab3c8829fc5b41b3d","previewModeSigningKey":"1b94d4ca5cd0226ac9b678f2d555dd18320d6b02934b175b11c0481dfe9d9701","previewModeEncryptionKey":"e3aa292891db53e66185dc67f41f42e49d4fe1462b2c6ce2eea6f7bbb12cf584","expireAt":1772742981486}
+{"previewModeId":"3d2ff63eaf01f7abb0ba0caeb3c17d0c","previewModeSigningKey":"981a172e225e7b17909282f71dbf463040b54bf9bf7eba4644fd99aa08e76454","previewModeEncryptionKey":"aca121c70f6014285fd5e771c8346ae5ee08d3e6a8a14de58b70d582604c589b","expireAt":1775126363538}

package/dist/.next/cache/.rscinfo

@@ -1 +1 @@
-{"encryption.key":"jwd7Nr+LbIL3aUoenjOD6a8YW3lBEYYd7GKSxdDWdKI=","encryption.expire_at":1772742980967}
+{"encryption.key":"ht9K53UtRRttTiNu7t5Oj65n4lCPbGM1sV7diyI4Lcs=","encryption.expire_at":1775126363195}