@zibby/skills 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +5 -2
- package/dist/index.js +42 -39
- package/dist/memory.js +3 -3
- package/dist/package.json +1 -1
- package/docs/cli-reference.md +91 -2
- package/docs/cloud/triggering.md +14 -0
- package/docs/intro.md +12 -2
- package/docs/legacy/test-automation.md +3 -1
- package/docs/packages/ui-memory.md +245 -0
- package/docs/recipes/test.md +36 -0
- package/docs/tests/memory.md +131 -0
- package/package.json +1 -1
package/dist/memory.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{createRequire as s}from"module";import{execFileSync as n}from"child_process";import{join as o}from"path";import{existsSync as a}from"fs";var c=s(import.meta.url);function m(){if(process.env.MCP_MEMORY_PATH)return process.env.MCP_MEMORY_PATH;try{return c.resolve("@zibby/memory/mcp-server")}catch{return null}}var d={id:"memory",serverName:"memory",allowedTools:["mcp__memory__*"],envKeys:[],description:"Zibby Memory MCP Server (test history, selectors, page model)",async middleware(){try{let{createMemoryMiddleware:e}=await import("@zibby/memory");return e()}catch{return null}},promptFragment:`BEFORE executing browser actions:
|
|
1
|
+
import{createRequire as s}from"module";import{execFileSync as n}from"child_process";import{join as o}from"path";import{existsSync as a}from"fs";var c=s(import.meta.url);function m(){if(process.env.MCP_MEMORY_PATH)return process.env.MCP_MEMORY_PATH;try{return c.resolve("@zibby/ui-memory/mcp-server")}catch{return null}}var d={id:"memory",serverName:"memory",allowedTools:["mcp__memory__*"],envKeys:[],description:"Zibby Memory MCP Server (test history, selectors, page model)",async middleware(){try{let{createMemoryMiddleware:e}=await import("@zibby/ui-memory");return e()}catch{return null}},promptFragment:`BEFORE executing browser actions:
|
|
2
2
|
- Review any test memory/history above. Prefer selectors proven to work.
|
|
3
3
|
- If a previous run failed, avoid the same approach.
|
|
4
4
|
- After setup/login completes, navigate directly to the target page instead of clicking through menus.
|
|
@@ -12,8 +12,8 @@ AFTER completing the test, you MUST call memory_save_insight at least once:
|
|
|
12
12
|
- Category: selector_tip | timing | navigation | workaround | flaky | general
|
|
13
13
|
- Be specific \u2014 future runs will read your insights.`,resolve(){let e=m();if(!e)throw new Error(`\u274C Memory MCP server not found
|
|
14
14
|
|
|
15
|
-
Install @zibby/memory:
|
|
16
|
-
npm install @zibby/memory`);let r=o(process.cwd(),".zibby","memory");if(!a(o(r,".dolt")))throw new Error(`\u274C Memory database not initialized
|
|
15
|
+
Install @zibby/ui-memory:
|
|
16
|
+
npm install @zibby/ui-memory`);let r=o(process.cwd(),".zibby","memory");if(!a(o(r,".dolt")))throw new Error(`\u274C Memory database not initialized
|
|
17
17
|
|
|
18
18
|
Run:
|
|
19
19
|
zibby init --mem`);try{let t=n("dolt",["sql","-q","SELECT COUNT(*) AS cnt FROM test_runs","-r","json"],{cwd:r,encoding:"utf-8",timeout:5e3}),i=JSON.parse(t.trim()).rows||[];if(!i[0]||i[0].cnt===0)return console.log("[memory] Database empty \u2014 memory tools activate after first completed run"),null}catch(t){throw new Error(`\u274C Dolt not found or memory database error
|
package/dist/package.json
CHANGED
package/docs/cli-reference.md
CHANGED
|
@@ -22,14 +22,23 @@ Every command lives under `zibby workflow <verb>` for consistency. The bare top-
|
|
|
22
22
|
| [`zibby workflow start <name>`](#workflow-start) | Long-lived dev server (Studio integration). Most users want `run`. |
|
|
23
23
|
| [`zibby workflow env <verb>`](#workflow-env) | Manage per-workflow encrypted env vars: `list`, `set`, `unset`, `push`. |
|
|
24
24
|
|
|
25
|
-
Plus
|
|
25
|
+
Plus the test recipe + memory + project setup:
|
|
26
|
+
|
|
27
|
+
| Command | What it does |
|
|
28
|
+
|---|---|
|
|
29
|
+
| [`zibby test [spec]`](#test) | Run a test spec (or inline string). Drives the browser via Cursor / Claude / Codex / Gemini. |
|
|
30
|
+
| [`zibby memory <verb>`](#memory) | Local + remote test-memory DB: `init`, `stats`, `cost`, `compact`, `reset`, `pull`, `push`, `remote …`. |
|
|
31
|
+
| [`zibby init [project]`](#init) | Bootstrap a Zibby project (config + creds). `-t <template>` to also scaffold a workflow. |
|
|
32
|
+
| [`zibby template <verb>`](#template) | List or add workflow templates: `list`, `add <name>`. |
|
|
33
|
+
|
|
34
|
+
Auth + admin:
|
|
26
35
|
|
|
27
36
|
| Command | What it does |
|
|
28
37
|
|---|---|
|
|
29
38
|
| `zibby login` | OAuth in browser. Writes session to `~/.zibby/config.json`. |
|
|
30
39
|
| `zibby logout` | Clear session. |
|
|
31
40
|
| `zibby status` | Show current auth state. |
|
|
32
|
-
| `zibby
|
|
41
|
+
| `zibby list` | List projects + API tokens you have access to. |
|
|
33
42
|
|
|
34
43
|
## workflow new {#workflow-new}
|
|
35
44
|
|
|
@@ -176,6 +185,86 @@ zibby workflow delete <uuid>
|
|
|
176
185
|
|
|
177
186
|
Removes the workflow from cloud (and its trigger URL). Local files are not touched.
|
|
178
187
|
|
|
188
|
+
## test {#test}
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
zibby test [spec-path]
|
|
192
|
+
zibby test "go to example.com and verify the title is 'Example Domain'" # inline
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
Built-in browser-test recipe. Reads a `.txt` spec (or inline string), drives a real browser via the configured agent, generates a Playwright script + verification video. See [Recipes → Test](./recipes/test) for the long form.
|
|
196
|
+
|
|
197
|
+
Common options:
|
|
198
|
+
- `--agent <claude|cursor|codex|gemini>` — override the configured agent for this run
|
|
199
|
+
- `--workflow <name>` — use a non-default workflow (e.g. `QuickSmokeWorkflow`)
|
|
200
|
+
- `--headless` — run headless (default is headed)
|
|
201
|
+
- `-m, --mem` — enable test memory (Dolt-backed knowledge from prior runs)
|
|
202
|
+
- `--sources <ids> --execution <id>` — run cloud-stored test cases (comma-separated IDs)
|
|
203
|
+
- `--node <name> --session <id|last>` — re-execute one phase from a prior session
|
|
204
|
+
- `--sync` / `--no-sync` — force / skip cloud upload
|
|
205
|
+
- `--collection <id|name>` and `--folder <path>` — categorize the run on the cloud dashboard
|
|
206
|
+
- `--verbose` / `--debug` — escalate log levels
|
|
207
|
+
- `-o, --open` — open results in browser after completion
|
|
208
|
+
- `--auto-approve` — auto-approve MCP tools (CI/CD)
|
|
209
|
+
|
|
210
|
+
## memory {#memory}
|
|
211
|
+
|
|
212
|
+
Test memory is a local-first Dolt SQL DB at `.zibby/memory/.dolt/` that learns selectors, page model, navigation, and run history from every `zibby test` run. See [Test memory](./tests/memory) for the deeper explainer.
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
zibby memory init # initialize the local DB (auto-runs on first `zibby test -m`)
|
|
216
|
+
zibby memory stats # row counts, last commit, per-spec breakdown
|
|
217
|
+
zibby memory cost # real LLM token spend per spec / per domain
|
|
218
|
+
zibby memory compact # prune old runs + Dolt GC (default: --max-runs 50, --max-age 90)
|
|
219
|
+
zibby memory reset -f # wipe the DB
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
Optional team sync:
|
|
223
|
+
|
|
224
|
+
```bash
|
|
225
|
+
zibby memory remote add <url> # BYO: aws://, gs://, https://, file:///
|
|
226
|
+
zibby memory remote use --hosted # Zibby-managed S3 (signed-in users only)
|
|
227
|
+
zibby memory remote info # show current remote
|
|
228
|
+
zibby memory remote remove [name] # drop the remote (default: origin)
|
|
229
|
+
|
|
230
|
+
zibby memory pull # pull from remote (auto on test start when configured)
|
|
231
|
+
zibby memory push # push to remote (auto after passing tests)
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
When `memorySync.remote` is set in `.zibby.config.mjs` (`'hosted'` or an `aws://...` URL), `zibby init` auto-wires the remote on first run — teammates clone, run init, and they're plugged in.
|
|
235
|
+
|
|
236
|
+
## init {#init}
|
|
237
|
+
|
|
238
|
+
```bash
|
|
239
|
+
zibby init [project-name]
|
|
240
|
+
zibby init -t browser-test-automation # also scaffold the test recipe
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
Bare init by default — writes `.zibby.config.mjs`, sets up agent credentials, configures memory. Pass `-t <template>` to also scaffold a workflow template into `.zibby/`.
|
|
244
|
+
|
|
245
|
+
Common options:
|
|
246
|
+
- `-t, --template <name>` — workflow template to scaffold (see `zibby template list`). Default: none (config + creds only).
|
|
247
|
+
- `--agent <claude|cursor|codex|gemini>` — pick the agent up front instead of prompting
|
|
248
|
+
- `--memory-backend <dolt|mem0>` — memory backend (default: `dolt`; `mem0` is experimental)
|
|
249
|
+
- `--skip-install` / `--skip-memory` — skip `npm install` / skip memory setup
|
|
250
|
+
- `-f, --force` — overwrite existing config
|
|
251
|
+
- `--api-key <key>` — non-interactive Zibby API key (for `--cloud-sync`)
|
|
252
|
+
- `--cloud-sync` — enable cloud sync and install the Zibby MCP
|
|
253
|
+
|
|
254
|
+
## template {#template}
|
|
255
|
+
|
|
256
|
+
```bash
|
|
257
|
+
zibby template list # see what's available
|
|
258
|
+
zibby template add <name> # copy template into .zibby/ (overwrites = doubles as update)
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
Templates are starter workflow scaffolds. `add` overwrites existing files in place — use it to upgrade an outdated agent helpers block, or to grab a recipe you didn't pick at `init` time.
|
|
262
|
+
|
|
263
|
+
`zibby template add zibby-workflow-claude` (or `-cursor`, `-codex`) refreshes the per-agent guidance files emitted by this template — the `<!-- zibby-template-version: N -->` markers make the upgrade idempotent.
|
|
264
|
+
|
|
265
|
+
Options on `add`:
|
|
266
|
+
- `--skip-memory` — strip `SKILLS.MEMORY` from copied `execute-live.mjs` (browser-test template only)
|
|
267
|
+
|
|
179
268
|
## Environment variables
|
|
180
269
|
|
|
181
270
|
| Var | Purpose |
|
package/docs/cloud/triggering.md
CHANGED
|
@@ -18,6 +18,20 @@ Auth: set `ZIBBY_API_KEY` in the environment, or run `zibby login` once on a wor
|
|
|
18
18
|
|
|
19
19
|
The flag surface is identical to `zibby workflow run` (local) — same `-p / --input / --input-file`, plus `--idempotency-key` for the cloud-only dedup case below. Flip the verb and the same call shape goes from local to remote.
|
|
20
20
|
|
|
21
|
+
### Input precedence
|
|
22
|
+
|
|
23
|
+
When more than one input source is provided, they merge with this **precedence (highest → lowest)**:
|
|
24
|
+
|
|
25
|
+
1. `-p key=value` (repeatable) — wins over everything; great for shell-friendly tweaks on top of a base payload
|
|
26
|
+
2. `--input '<json>'` — full JSON payload as a string
|
|
27
|
+
3. `--input-file path.json` — full JSON/YAML payload from a file (lowest precedence)
|
|
28
|
+
|
|
29
|
+
This means the common pattern of "load a base payload from disk, override a few keys for this run" works without manual merging:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
zibby workflow trigger <uuid> --input-file payload.json -p priority=urgent
|
|
33
|
+
```
|
|
34
|
+
|
|
21
35
|
## CI / cron
|
|
22
36
|
|
|
23
37
|
Anywhere you can shell out, call the CLI. Don't hand-roll an HTTP client — you'll just rebuild what the CLI already does (project lookup from UUID, idempotency, quota error messages, retries).
|
package/docs/intro.md
CHANGED
|
@@ -29,12 +29,22 @@ graph
|
|
|
29
29
|
|
|
30
30
|
```bash
|
|
31
31
|
npm install -g @zibby/cli
|
|
32
|
-
zibby
|
|
32
|
+
zibby init # bare init (config + creds; no template)
|
|
33
|
+
zibby workflow new my-pipeline # scaffold a custom workflow
|
|
33
34
|
zibby workflow run my-pipeline # run locally (one-shot)
|
|
34
35
|
|
|
35
36
|
zibby login
|
|
36
37
|
zibby workflow deploy my-pipeline # ship to cloud
|
|
37
|
-
zibby workflow trigger <uuid>
|
|
38
|
+
zibby workflow trigger <uuid> # fire it
|
|
39
|
+
zibby workflow logs <uuid> -t # tail logs
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Want a starter recipe instead of a blank slate? Use `-t`:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
zibby init -t browser-test-automation # scaffold the test recipe at init time
|
|
46
|
+
zibby template list # see what's available
|
|
47
|
+
zibby template add <name> # add a template later (overwrites = doubles as update)
|
|
38
48
|
```
|
|
39
49
|
|
|
40
50
|
→ Next: [Install](./get-started/install)
|
|
@@ -5,7 +5,9 @@ title: Test automation (legacy)
|
|
|
5
5
|
|
|
6
6
|
# Test automation (legacy)
|
|
7
7
|
|
|
8
|
-
> **The `zibby test` flow predates the workflow engine
|
|
8
|
+
> **Deprecated landing page.** The `zibby test` flow predates the workflow engine. The current canonical pages are [`zibby test` recipe](/recipes/test), [Test memory](/tests/memory), and [CLI Reference → test](/cli-reference#test). This page is kept for users following old links; everything below still works, but **new users should start with [Get Started](/get-started/install)**.
|
|
9
|
+
|
|
10
|
+
> Notably, `zibby init --mem` referenced below is now `zibby init` (memory is set up by default) plus optionally `-t browser-test-automation` to scaffold the test recipe. The current commands are documented in the [CLI Reference](/cli-reference#init).
|
|
9
11
|
|
|
10
12
|
Drive a real browser from a plain-text spec, get a Playwright script back.
|
|
11
13
|
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
---
|
|
2
|
+
sidebar_position: 5
|
|
3
|
+
title: "@zibby/ui-memory"
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# @zibby/ui-memory
|
|
7
|
+
|
|
8
|
+
Version-controlled UI agent memory powered by [Dolt](https://www.dolthub.com/). Learns from every test run — selectors that worked, page-element fingerprints, navigation transitions, timing quirks, recorded insights. Used today by `zibby test`; designed to power any agent that drives a UI.
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
npm install @zibby/ui-memory
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
Current version: **1.1.0**
|
|
15
|
+
|
|
16
|
+
> Renamed from `@zibby/memory` to make the per-domain UI focus explicit. The chat-style memory backend (mem0) is dormant and not documented as a usable feature here.
|
|
17
|
+
|
|
18
|
+
## Why memory
|
|
19
|
+
|
|
20
|
+
Without memory, every test run starts from scratch. The agent has no idea which selectors are stable, which pages have changed, or what workarounds were discovered last week.
|
|
21
|
+
|
|
22
|
+
With `@zibby/ui-memory`:
|
|
23
|
+
|
|
24
|
+
- **Selectors** — the agent prefers selectors with high success / low fail counts
|
|
25
|
+
- **Page model** — known elements, ARIA roles, accessible names
|
|
26
|
+
- **Navigation** — known page-to-page transitions (which click produced which URL)
|
|
27
|
+
- **Test history** — pass/fail trends per spec, full timing
|
|
28
|
+
- **Insights** — categorized free-form notes the agent reads + writes (`selector_tip | timing | navigation | workaround | flaky | general`)
|
|
29
|
+
|
|
30
|
+
Critically, memory is keyed **per domain**, not per spec. A selector that one spec learned for `myapp.com/login` is available to every other spec hitting the same site.
|
|
31
|
+
|
|
32
|
+
## Setup
|
|
33
|
+
|
|
34
|
+
Most users get this automatically via `zibby init` — Dolt is bundled, the DB is initialized, and `zibby test` writes to it. The notes below are for direct package consumption.
|
|
35
|
+
|
|
36
|
+
### 1. Install Dolt (if not bundled)
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
# macOS
|
|
40
|
+
brew install dolt
|
|
41
|
+
|
|
42
|
+
# Linux
|
|
43
|
+
sudo bash -c 'curl -L https://github.com/dolthub/dolt/releases/latest/download/install.sh | bash'
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
### 2. Initialize the database
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
zibby memory init
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Creates `.zibby/memory/.dolt/` with the schema for runs, selectors, page model, navigation, and insights.
|
|
53
|
+
|
|
54
|
+
### 3. Enable memory in your workflow
|
|
55
|
+
|
|
56
|
+
Add `SKILLS.MEMORY` to any node that should have memory access:
|
|
57
|
+
|
|
58
|
+
```javascript
|
|
59
|
+
import { SKILLS } from '@zibby/core';
|
|
60
|
+
|
|
61
|
+
export const executeLiveNode = {
|
|
62
|
+
name: 'execute_live',
|
|
63
|
+
skills: [SKILLS.BROWSER, SKILLS.MEMORY],
|
|
64
|
+
// ...
|
|
65
|
+
};
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
The built-in `execute_live` node already has memory enabled.
|
|
69
|
+
|
|
70
|
+
## How it works
|
|
71
|
+
|
|
72
|
+
### Before each run
|
|
73
|
+
|
|
74
|
+
The runner auto-pulls from the configured remote (if any) and loads the relevant slice into the agent's context — selectors for the page, recent failures, applicable insights.
|
|
75
|
+
|
|
76
|
+
### During the run
|
|
77
|
+
|
|
78
|
+
The agent has 5 MCP tools auto-exposed:
|
|
79
|
+
|
|
80
|
+
| Tool | What it does |
|
|
81
|
+
|---|---|
|
|
82
|
+
| `memory_get_test_history` | Recent runs — filter by spec-path substring; returns pass/fail/timing |
|
|
83
|
+
| `memory_get_selectors` | Known selectors per page with success/fail counts |
|
|
84
|
+
| `memory_get_page_model` | Page elements (URL, ARIA role, accessible name, best selector) |
|
|
85
|
+
| `memory_get_navigation` | Known transitions (from URL → to URL via what trigger) |
|
|
86
|
+
| `memory_save_insight` | Save observation: `selector_tip | timing | navigation | workaround | flaky | general` |
|
|
87
|
+
|
|
88
|
+
### After the run
|
|
89
|
+
|
|
90
|
+
The result handler persists the new state — selectors used (with success/failure deltas), page-model updates, new navigation transitions discovered, and any insights the agent saved.
|
|
91
|
+
|
|
92
|
+
> **The agent is required to call `memory_save_insight` at least once at the end of every run.** This is in the memory skill's prompt fragment. Without insights, memory degrades to cached selectors and run rows; with them it compounds.
|
|
93
|
+
|
|
94
|
+
### Version control
|
|
95
|
+
|
|
96
|
+
Every persist creates a Dolt commit. You can:
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
cd .zibby/memory
|
|
100
|
+
dolt log
|
|
101
|
+
dolt diff HEAD~1 HEAD
|
|
102
|
+
dolt branch experiment
|
|
103
|
+
dolt checkout experiment
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## Database schema
|
|
107
|
+
|
|
108
|
+
### `test_runs`
|
|
109
|
+
|
|
110
|
+
| Column | Type | Description |
|
|
111
|
+
|---|---|---|
|
|
112
|
+
| `session_id` | VARCHAR | Unique session identifier |
|
|
113
|
+
| `spec_path` | VARCHAR | Path to the test spec file |
|
|
114
|
+
| `passed` | BOOLEAN | Whether the test passed |
|
|
115
|
+
| `duration_ms` | INT | Total execution time |
|
|
116
|
+
| `agent_type` | VARCHAR | Which agent ran the test |
|
|
117
|
+
| `tokens_input` / `tokens_output` / `tokens_cache_*` | INT | LLM token usage (drives `zibby memory cost`) |
|
|
118
|
+
| `created_at` | DATETIME | Timestamp |
|
|
119
|
+
|
|
120
|
+
### `selectors`
|
|
121
|
+
|
|
122
|
+
| Column | Type | Description |
|
|
123
|
+
|---|---|---|
|
|
124
|
+
| `page_url` | VARCHAR | URL where this selector was used |
|
|
125
|
+
| `selector` | VARCHAR | The CSS/XPath selector string |
|
|
126
|
+
| `stable_id` | VARCHAR | Zibby stable ID (if available) |
|
|
127
|
+
| `success_count` | INT | Times this selector worked |
|
|
128
|
+
| `fail_count` | INT | Times this selector failed |
|
|
129
|
+
| `last_used` | DATETIME | Last usage timestamp |
|
|
130
|
+
|
|
131
|
+
### `page_model`
|
|
132
|
+
|
|
133
|
+
| Column | Type | Description |
|
|
134
|
+
|---|---|---|
|
|
135
|
+
| `url` | VARCHAR | Page URL |
|
|
136
|
+
| `element_role` | VARCHAR | ARIA role |
|
|
137
|
+
| `element_name` | VARCHAR | Accessible name |
|
|
138
|
+
| `selector` | VARCHAR | Best known selector |
|
|
139
|
+
| `updated_at` | DATETIME | Last update |
|
|
140
|
+
|
|
141
|
+
### `navigation`
|
|
142
|
+
|
|
143
|
+
| Column | Type | Description |
|
|
144
|
+
|---|---|---|
|
|
145
|
+
| `from_url` | VARCHAR | Source page URL |
|
|
146
|
+
| `to_url` | VARCHAR | Destination page URL |
|
|
147
|
+
| `trigger` | VARCHAR | What caused the navigation (click, submit, etc.) |
|
|
148
|
+
| `count` | INT | Times this transition was observed |
|
|
149
|
+
|
|
150
|
+
### `insights`
|
|
151
|
+
|
|
152
|
+
| Column | Type | Description |
|
|
153
|
+
|---|---|---|
|
|
154
|
+
| `category` | ENUM | `selector_tip`, `timing`, `navigation`, `workaround`, `flaky`, `general` |
|
|
155
|
+
| `content` | TEXT | The insight text |
|
|
156
|
+
| `spec_path` | VARCHAR | Related spec |
|
|
157
|
+
| `session_id` | VARCHAR | Session that created it |
|
|
158
|
+
| `created_at` | DATETIME | Timestamp |
|
|
159
|
+
|
|
160
|
+
## Team sync
|
|
161
|
+
|
|
162
|
+
Memory is local-first. Opt into a shared remote so teammates' learnings flow back:
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
# BYO — your S3 / GCS / DoltHub repo / file path
|
|
166
|
+
zibby memory remote add aws://my-bucket/team/proj/main
|
|
167
|
+
zibby memory remote add gs://bucket/team/proj/main
|
|
168
|
+
zibby memory remote add https://www.dolthub.com/repositories/<owner>/<repo>
|
|
169
|
+
zibby memory remote add file:///abs/path/to/local-shared
|
|
170
|
+
|
|
171
|
+
# OR Zibby-managed S3 (no plumbing; signed-in users only)
|
|
172
|
+
zibby memory remote use --hosted
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Once configured:
|
|
176
|
+
|
|
177
|
+
- `zibby test` auto-pulls before runs and auto-pushes after passing runs
|
|
178
|
+
- `zibby memory pull` / `zibby memory push` for manual override
|
|
179
|
+
- `zibby memory remote info` to inspect, `zibby memory remote remove` to disconnect
|
|
180
|
+
|
|
181
|
+
To wire teammates in automatically, set `memorySync.remote` in `.zibby.config.mjs`:
|
|
182
|
+
|
|
183
|
+
```js
|
|
184
|
+
export default {
|
|
185
|
+
agent: { claude: { model: 'auto' } },
|
|
186
|
+
memorySync: {
|
|
187
|
+
remote: 'hosted', // or 'aws://my-bucket/team/proj/main' or null
|
|
188
|
+
},
|
|
189
|
+
};
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
`zibby init` reads this — when set to `'hosted'` and the user isn't signed in, init prompts for `zibby login` but never blocks. After login, the remote is wired and the next `zibby test` pulls.
|
|
193
|
+
|
|
194
|
+
### Hosted vs BYO
|
|
195
|
+
|
|
196
|
+
| | Hosted (`--hosted`) | BYO |
|
|
197
|
+
|---|---|---|
|
|
198
|
+
| Setup | `zibby memory remote use --hosted` | Provision bucket / IAM / KMS |
|
|
199
|
+
| Storage | Zibby-managed AWS account | Your account |
|
|
200
|
+
| Access | Anyone with project access on Zibby | Whoever your IAM grants |
|
|
201
|
+
| Compliance / data residency | Limited regions | Wherever you want |
|
|
202
|
+
| Cost | Included in plan | Your S3 bill |
|
|
203
|
+
|
|
204
|
+
## Middleware integration
|
|
205
|
+
|
|
206
|
+
Memory provides automatic middleware that injects history into the node context:
|
|
207
|
+
|
|
208
|
+
```javascript
|
|
209
|
+
import { createMemoryMiddleware } from '@zibby/ui-memory';
|
|
210
|
+
|
|
211
|
+
const middleware = createMemoryMiddleware();
|
|
212
|
+
const graph = new WorkflowGraph({ middleware: [middleware] });
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
The memory skill registers this middleware automatically when `SKILLS.MEMORY` is declared on a node.
|
|
216
|
+
|
|
217
|
+
## CLI commands
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
zibby memory init # initialize the DB
|
|
221
|
+
zibby memory stats # row counts, last commit, per-spec breakdown
|
|
222
|
+
zibby memory cost # real LLM token spend per spec / per domain
|
|
223
|
+
zibby memory compact # prune old runs + Dolt GC
|
|
224
|
+
zibby memory reset -f # wipe (destructive)
|
|
225
|
+
zibby memory pull / push # manual sync (auto on test start/end if remote configured)
|
|
226
|
+
zibby memory remote add <url> # BYO remote
|
|
227
|
+
zibby memory remote use --hosted# Zibby-managed S3
|
|
228
|
+
zibby memory remote info # show config
|
|
229
|
+
zibby memory remote remove # drop the remote
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
## Exports
|
|
233
|
+
|
|
234
|
+
```javascript
|
|
235
|
+
import {
|
|
236
|
+
createMemoryMiddleware,
|
|
237
|
+
memoryEndRun,
|
|
238
|
+
memorySyncPush,
|
|
239
|
+
} from '@zibby/ui-memory';
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
## See also
|
|
243
|
+
|
|
244
|
+
- [`zibby test` recipe](../recipes/test) — the primary consumer of memory
|
|
245
|
+
- [Test memory deep dive](../tests/memory) — usage-oriented walkthrough
|
package/docs/recipes/test.md
CHANGED
|
@@ -77,6 +77,42 @@ zibby test test-specs/login.txt # headed (default — see the brows
|
|
|
77
77
|
zibby test test-specs/login.txt --headless # headless mode (for CI)
|
|
78
78
|
```
|
|
79
79
|
|
|
80
|
+
**Cloud-stored test cases (run a saved execution):**
|
|
81
|
+
```bash
|
|
82
|
+
zibby test --sources <id1>,<id2> --execution <executionId>
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Test memory
|
|
86
|
+
|
|
87
|
+
The recipe ships with a learning loop: every run reads from and writes to a local Dolt DB at `.zibby/memory/.dolt/` (cross-spec, **per-domain**). Selectors that worked, page-element fingerprints, navigation transitions, and free-form insights are all persisted — the agent's 100th run on a site is sharper and cheaper than its first.
|
|
88
|
+
|
|
89
|
+
When `zibby test` runs and the DB exists, the agent gets 5 MCP tools auto-exposed:
|
|
90
|
+
|
|
91
|
+
- `memory_get_test_history` — recent runs (pass/fail/timing)
|
|
92
|
+
- `memory_get_selectors` — known selectors with stability metrics
|
|
93
|
+
- `memory_get_page_model` — page elements / roles / accessible names
|
|
94
|
+
- `memory_get_navigation` — known page-to-page transitions
|
|
95
|
+
- `memory_save_insight` — save observations (categories: `selector_tip | timing | navigation | workaround | flaky | general`). **Required at least once per run.**
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
zibby memory stats # what's in the DB
|
|
99
|
+
zibby memory cost # real LLM token spend per spec / per domain
|
|
100
|
+
zibby memory compact # prune old runs + GC
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
**Team sync.** Point the DB at a remote and teammates' learnings flow back to you on the next test run:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
zibby memory remote add aws://my-bucket/team/proj/main # BYO (S3 / GCS / DoltHub / file:///)
|
|
107
|
+
zibby memory remote use --hosted # OR Zibby-managed S3 (signed-in only)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Or commit `memorySync.remote: 'hosted'` (or an `aws://` URL) into `.zibby.config.mjs` and `zibby init` auto-wires it for every teammate.
|
|
111
|
+
|
|
112
|
+
Auto-pull on test start, auto-push on test pass. Failing runs don't pollute team memory.
|
|
113
|
+
|
|
114
|
+
→ Full guide: [Test memory](../tests/memory). Schema and SDK: [`@zibby/ui-memory`](../packages/ui-memory).
|
|
115
|
+
|
|
80
116
|
## Forking the recipe
|
|
81
117
|
|
|
82
118
|
If the built-in recipe doesn't fit your case, scaffold a custom workflow and copy the structure:
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
---
|
|
2
|
+
sidebar_position: 1
|
|
3
|
+
title: Test memory
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Test memory
|
|
7
|
+
|
|
8
|
+
Zibby's `zibby test` runner has a local-first **memory database** at `.zibby/memory/.dolt/` that learns from every run — selectors that worked, page-element fingerprints, navigation transitions, timing quirks, recorded insights. Cross-spec via per-domain keying, optionally synced across the team.
|
|
9
|
+
|
|
10
|
+
The package powering this is [`@zibby/ui-memory`](../packages/ui-memory) — read that for the schema and SDK. This page is a **usage walkthrough**.
|
|
11
|
+
|
|
12
|
+
## What's in the box
|
|
13
|
+
|
|
14
|
+
- **Local DB** — `.zibby/memory/.dolt/` (Dolt SQL, version-controlled like git)
|
|
15
|
+
- **5 MCP tools** auto-exposed when `zibby test` runs and the DB exists
|
|
16
|
+
- **Auto-pull / auto-push** before and after each test run when a remote is configured
|
|
17
|
+
- **CLI** — `zibby memory <verb>` for stats, cost, compact, reset, pull, push, remote management
|
|
18
|
+
|
|
19
|
+
## Why per-domain keying matters
|
|
20
|
+
|
|
21
|
+
Memory is keyed on the **domain**, not the spec file. Selectors learned by `login.txt` for `myapp.com/login` are immediately available when `checkout.txt` lands on `myapp.com/checkout` and asks "what's a stable selector for the email field on this site?"
|
|
22
|
+
|
|
23
|
+
In practice this means a multi-spec suite gets sharper with every run — the agent's Nth run on a domain is meaningfully cheaper and more reliable than its first.
|
|
24
|
+
|
|
25
|
+
## The 5 MCP tools
|
|
26
|
+
|
|
27
|
+
When `zibby test` runs and `.zibby/memory/.dolt/` exists, the agent gets these auto-exposed:
|
|
28
|
+
|
|
29
|
+
| Tool | What it does |
|
|
30
|
+
|---|---|
|
|
31
|
+
| `memory_get_test_history` | Recent runs (filter by spec-path substring) — pass/fail and timing |
|
|
32
|
+
| `memory_get_selectors` | Known selectors per page with success/fail counts |
|
|
33
|
+
| `memory_get_page_model` | Page elements (URL, ARIA role, accessible name, best-known selector) |
|
|
34
|
+
| `memory_get_navigation` | Known transitions (from URL → to URL via what trigger) |
|
|
35
|
+
| `memory_save_insight` | Save an observation: `selector_tip | timing | navigation | workaround | flaky | general` |
|
|
36
|
+
|
|
37
|
+
> **The agent must call `memory_save_insight` at least once per run.** It's required by the memory skill's prompt fragment. Without insights, only the structural data (selectors / runs) compounds; insights are how the system learns "the site shows a banner on Tuesdays" or "this button needs a 200ms wait after focus".
|
|
38
|
+
|
|
39
|
+
## CLI commands
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
zibby memory init # initialize (auto-runs on first `zibby test -m`)
|
|
43
|
+
zibby memory stats # row counts, last commit, per-spec breakdown
|
|
44
|
+
zibby memory cost # real LLM token spend per spec / per domain (input/output/cache)
|
|
45
|
+
zibby memory compact # prune old runs + Dolt GC (--max-runs 50, --max-age 90d)
|
|
46
|
+
zibby memory reset -f # wipe the DB (destructive — confirm)
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
`compact` is the maintenance lever once the DB grows. `cost` is the most-asked-after view — it answers "which spec is the expensive one?" with real numbers.
|
|
50
|
+
|
|
51
|
+
## Team sync
|
|
52
|
+
|
|
53
|
+
Memory is local-first by default. Opt into a shared remote so teammates' learnings flow back to you:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
# Option A: bring your own bucket
|
|
57
|
+
zibby memory remote add aws://my-bucket/team/proj/main # S3
|
|
58
|
+
zibby memory remote add gs://bucket/team/proj/main # GCS
|
|
59
|
+
zibby memory remote add https://www.dolthub.com/repositories/<owner>/<repo>
|
|
60
|
+
zibby memory remote add file:///abs/path/to/local-shared # filesystem
|
|
61
|
+
|
|
62
|
+
# Option B: Zibby-managed S3 (signed-in users only)
|
|
63
|
+
zibby memory remote use --hosted
|
|
64
|
+
|
|
65
|
+
# Inspect / disconnect
|
|
66
|
+
zibby memory remote info
|
|
67
|
+
zibby memory remote remove [name] # default: origin
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Once a remote is configured:
|
|
71
|
+
|
|
72
|
+
- `zibby test` auto-pulls before each run
|
|
73
|
+
- `zibby test` auto-pushes after each **passing** run (failing runs don't pollute team memory)
|
|
74
|
+
- `zibby memory pull` / `zibby memory push` for manual override
|
|
75
|
+
|
|
76
|
+
### Auto-wire teammates with `memorySync.remote`
|
|
77
|
+
|
|
78
|
+
Drop this into `.zibby.config.mjs` and commit it:
|
|
79
|
+
|
|
80
|
+
```js
|
|
81
|
+
export default {
|
|
82
|
+
agent: { claude: { model: 'auto' } },
|
|
83
|
+
memorySync: {
|
|
84
|
+
remote: 'hosted', // or 'aws://my-bucket/team/proj/main' or null
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Now when a teammate clones the repo and runs `zibby init`, the CLI reads `memorySync.remote` and auto-configures the matching remote. For `'hosted'`, init prompts for `zibby login` if they're not signed in but never blocks the init itself.
|
|
90
|
+
|
|
91
|
+
### Hosted vs BYO at a glance
|
|
92
|
+
|
|
93
|
+
| | Hosted (`--hosted`) | BYO |
|
|
94
|
+
|---|---|---|
|
|
95
|
+
| Setup time | One command | Provision bucket + IAM (+ KMS if you want) |
|
|
96
|
+
| Where data lives | Zibby-managed AWS account | Your account |
|
|
97
|
+
| Access | Anyone with Zibby project access | Whoever your IAM grants |
|
|
98
|
+
| Compliance / data residency | Limited regions | Wherever you want |
|
|
99
|
+
| Cost | Included in plan | Your S3 bill |
|
|
100
|
+
|
|
101
|
+
If you have any data-residency requirement or a regulated workload, prefer BYO. Otherwise hosted is the path of least resistance.
|
|
102
|
+
|
|
103
|
+
## Run-level controls
|
|
104
|
+
|
|
105
|
+
`zibby test` exposes one memory-relevant flag:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
zibby test test-specs/login.txt -m # enable test memory (auto-init if needed)
|
|
109
|
+
zibby test test-specs/login.txt --no-sync # don't push to cloud (does not affect memory remote)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Memory is independent of the cloud-results sync (`--sync` / `--no-sync` controls run upload to the Zibby dashboard; memory sync is its own remote).
|
|
113
|
+
|
|
114
|
+
## Inspecting the DB by hand
|
|
115
|
+
|
|
116
|
+
It's just Dolt:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
cd .zibby/memory
|
|
120
|
+
dolt log
|
|
121
|
+
dolt sql -q "SELECT spec_path, passed, duration_ms FROM test_runs ORDER BY created_at DESC LIMIT 20"
|
|
122
|
+
dolt diff HEAD~1 HEAD
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Branching works too — `dolt branch experiment` to try out a memory mutation in isolation.
|
|
126
|
+
|
|
127
|
+
## See also
|
|
128
|
+
|
|
129
|
+
- [`@zibby/ui-memory` package](../packages/ui-memory) — schema, SDK, middleware
|
|
130
|
+
- [`zibby test` recipe](../recipes/test) — the primary consumer
|
|
131
|
+
- [CLI reference: memory](../cli-reference#memory) — full command list
|