@quantod/qq 1.1.19 → 1.3.0

package/src/resources/guide.md

@@ -21,7 +21,7 @@ Stages are just labels. The pipeline has no built-in notion of "first" or "last"
 Items have:
 - `id` — the deduplication key. Pushing an item with an existing id fails.
 - `stage` — the current state
-- `payload` — item data; append-only string by default; YAML by convention.
+- `payload` — item data; a structured JSON object (or array). Merged with json_patch by default; replaced when `replace: true`.
 - `seq` — the claim token, returned by `claim`, required to `release`
 - `priority` — float, higher = claimed first (default 0.0)
 
@@ -34,12 +34,12 @@ make_pipeline        → create a pipeline, get its name, once per setup
 push (N times)       → load items into a stage
 claim                → atomically lock one item (returns seq + payload)
   ... process the claimed item ...
-release              → unlock item, move to next stage, append payload
+release              → unlock item, move to next stage, merge payload
 status               → monitor progress
 unstick              → recover stuck/lost items
 ```
 
-**Orchestrator** creates the pipeline, pushes items, spawns subagents, monitors with `status`, calls `batch_read` to collect results, calls `unstick` after.
+**Orchestrator** creates the pipeline, pushes items, spawns subagents, monitors with `status`, calls `query` to collect results, calls `unstick` after.
 
 **Subagent** claims one item, processes it, releases to the next stage, repeats until "no items to claim" — then stops and reports back. Never process an item before claiming it: another agent could claim and modify it concurrently.
 
@@ -50,9 +50,9 @@ unstick              → recover stuck/lost items
 Orchestrator example:
 ```
 make_pipeline → "0529_a3f9c2"   (store this as Q)
-push Q/pending "url-1" payload: "url: https://a.com"
-push Q/pending "url-2" payload: "url: https://b.com"
-push Q/pending "url-3" payload: "url: https://c.com"
+push Q/pending "url-1" payload: {url: "https://a.com"}
+push Q/pending "url-2" payload: {url: "https://b.com"}
+push Q/pending "url-3" payload: {url: "https://c.com"}
 ```
 
 Spawn subagents with instructions like:
@@ -98,8 +98,7 @@ Parameters:
 - `pipeline` (string) — pipeline name
 - `stage` (string, optional) — destination stage. Supports sub-paths: `enriched/shortlisted`. Defaults to empty string if omitted.
 - `id` (string, optional) — stable unique identifier. Omit to use auto-generated seq number.
-- `payload` (string, optional) — payload string
-- `payloadFormat` (`yaml` | `json` | `text`, optional, default `yaml`) — `yaml`: validate as YAML. `json`: validate as JSON and minify to one line (appended payloads form valid JSONL). `text`: no validation.
+- `payload` (object, optional) — structured payload object. Stored as SQLite JSONB. Any JSON-serializable object or array.
 - `priority` (number, optional) — float, default 0.0. Higher = claimed first. Only set when meaningful.
 
 Returns:
@@ -111,8 +110,6 @@ id: url-1
 
 Errors:
 - `error: item <id> already in the pipeline` — duplicate id; item already exists
-- `error: payload is not valid YAML: <detail>` — `payloadFormat` is `yaml` (default) and payload fails YAML parse
-- `error: payload is not valid JSON: <detail>` — `payloadFormat` is `json` and payload fails JSON parse
 
 ---
 
@@ -124,7 +121,7 @@ Parameters:
 - `pipeline` (string)
 - `stage` (string) — supports glob wildcards: `enriched/*`, `*`. Omit to claim from any stage.
 - `id` (string, optional) — claim a specific item by id. Fails if not found or already claimed.
-- `strategy` (`priority` | `fifo` | `lifo` | `random`, optional, default `priority`) — order in which unclaimed items are selected. `priority`: highest priority first, then lowest seq (default). `fifo`: lowest seq first, ignores priority. `lifo`: highest seq first, ignores priority. `random`: random selection, ignores priority. Ignored when claiming by `id`.
+- `sort_order` (`priority` | `fifo` | `lifo` | `random`, optional, default `priority`) — order in which unclaimed items are selected. `priority`: highest priority first, then lowest seq (default). `fifo`: lowest seq first, ignores priority. `lifo`: highest seq first, ignores priority. `random`: random selection, ignores priority. Ignored when claiming by `id`.
 
 Returns:
 ```yaml
@@ -135,7 +132,7 @@ priority: 0
 claimed: true
 created: 2025-05-28T14:30:00.000Z
 last_modified: 2025-05-28T14:30:00.000Z
-payload: |
+payload:
   url: https://example.com/product/42
 ```
 
@@ -156,9 +153,8 @@ Parameters:
 - `pipeline` (string)
 - `seq` (number) — the claim token from `claim`
 - `target` (string, optional) — destination stage. **Always specify when moving the item.** Omit to release back to the same stage.
-- `payload` (string, optional) — payload string to append (or replace)
-- `payloadFormat` (`yaml` | `json` | `text`, optional, default `yaml`) — same as `push`
-- `replace` (boolean, optional) — replace payload entirely instead of appending. Use when treating payload as a structured object.
+- `payload` (object, optional) — structured payload object to merge into the existing payload
+- `replace` (boolean, optional) — replace payload entirely instead of merging. Use when you want to discard the previous payload.
 - `priority` (number, optional) — update the item's priority
 
 Returns:
@@ -169,10 +165,8 @@ ok: true
 Errors:
 - `error: claim on seq <n> expired` — seq no longer exists; discard your work
 - `error: item seq <n> not claimed` — items should be claimed before releasing - likely bug
-- `error: payload is not valid YAML: <detail>` — `payloadFormat` is `yaml` (default) and payload fails YAML parse
-- `error: payload is not valid JSON: <detail>` — `payloadFormat` is `json` and payload fails JSON parse
 
-**Payload is append-only history by default.** Each stage appends its output — resulting duplicate YAML fields are intentional. Use `replace: true` only when treating payload as a structured object you own.
+**Payload merge uses json_patch (RFC 7396) by default.** When you pass a `payload` object, it is merged into the existing payload using RFC 7396 semantics: keys present in the patch are set on the target (recursively for nested objects); keys set to `null` in the patch are **removed** from the target; keys absent from the patch are left unchanged. Omitting `payload` leaves the existing payload unchanged. Use `replace: true` to discard the existing payload entirely.
 
 ---
 
@@ -189,47 +183,36 @@ description: Scrape product pages and extract pricing
 stats:
   /*: [23, 3]
   done: [7, 0]
-  failed: [6, 0]
-  pending: [10, 3]
-```
-
-Each value is `[total, claimed]` — total item count and how many are currently claimed (being processed). `/*` is the pipeline-level aggregate across all stages. `description` is `null` if none was set.
-
-Aggregate vs. direct entries: a key ending in `/*` is an aggregate — it sums all items in that prefix and its sub-stages. A key without `/*` is a direct count — only items whose stage exactly matches that key. When a stage has sub-stages, both appear:
-
-```yaml
-stats:
-  /*: [23, 3]
-  done: [7, 0]
-  enriched: [3, 1]        # items directly in 'enriched'
-  enriched/*: [10, 2]     # aggregate: enriched + enriched/shortlisted
+  enriched: [3, 1]
+  enriched/*: [10, 2]
   enriched/shortlisted: [7, 1]
   failed: [6, 0]
   pending: [10, 3]
 ```
 
-A stage that has no sub-stages appears only as a direct entry (no `/*` variant).
+Each value is `[total, claimed]` — total item count and how many are currently claimed (being processed). `/*` is the pipeline-level aggregate across all stages. A key ending in `*` is an aggregate — it sums all items in that prefix and its sub-stages.
 
 Errors:
 - `error: pipeline <name> not found`
 
 ---
 
-### batch_read
+### query
 
 Reads items without claiming them. For inspection, data collection, and monitoring.
 
 Parameters:
 - `pipeline` (string)
 - `stage` (string, optional) — supports globs. Defaults to `*`
-- `includePayload` (boolean, optional) — include payload in results
-- `claimed` (boolean, optional) — filter by claimed status. Default: unclaimed
+- `payload_fields` (`"*"` | string[], optional) — which payload fields to return. `"*"` returns the full payload; an array of dot-notation paths (e.g. `["status", "meta.author"]`) returns only those fields; omit to return no payload. **Note: this controls payload projection, not item filtering.**
+- `payload_filter` (object, optional) — MongoDB-style filter on payload fields. Keys are dot-notation paths into the payload. Values are either a literal (equality match) or an operator object. Supported operators: `$eq`, `$ne`, `$gt`, `$gte`, `$lt`, `$lte`, `$in` (array), `$exists` (boolean). Example: `{ "status": "active", "score": { "$gt": 80 } }`.
+- `claimed` (boolean, optional) — filter by claimed status
 - `ids` (string[], optional)
-- `createdAfter`, `createdBefore`, `modifiedAfter` (string | number) — epoch ms, ISO 8601 (`"2025-05-28"`, `"2025-05-28T14:30:00Z"`), or relative (`"-2h"`, `"-1d"`, `"-30m"`, `"-1w"`)
+- `created_after`, `created_before`, `modified_after` (string | number) — epoch ms, ISO 8601 (`"2025-05-28"`, `"2025-05-28T14:30:00Z"`), or relative (`"-2h"`, `"-1d"`, `"-30m"`, `"-1w"`)
 - `limit`, `offset` (number)
-- `strategy` (`priority` | `fifo` | `lifo` | `random`, optional, default `fifo`) — sort order. Same semantics as `claim`: `priority` highest first then lowest seq, `fifo` lowest seq first, `lifo` highest seq first, `random` random order.
+- `sort_order` (`priority` | `fifo` | `lifo` | `random`, optional, default `fifo`) — sort order. Same semantics as `claim`: `priority` highest first then lowest seq, `fifo` lowest seq first, `lifo` highest seq first, `random` random order.
 
-Returns a YAML list of items. `payload` is omitted unless `includePayload: true`. Returns `[]` when nothing matches.
+Returns a YAML list of items. `payload` is included only when `select` is provided. Returns `[]` when nothing matches.
 
 ```yaml
 - id: url-1
@@ -239,6 +222,9 @@ Returns a YAML list of items. `payload` is omitted unless `includePayload: true`
   priority: 0
   created: 2025-05-28T14:30:00.000Z
   last_modified: 2025-05-28T14:30:01.000Z
+  payload:
+    url: https://example.com
+    score: 92
 - id: url-2
   stage: done
   claimed: false
@@ -246,6 +232,9 @@ Returns a YAML list of items. `payload` is omitted unless `includePayload: true`
   priority: 0
   created: 2025-05-28T14:30:00.001Z
   last_modified: 2025-05-28T14:30:02.000Z
+  payload:
+    url: https://other.com
+    score: 47
 ```
 
 ---
@@ -257,7 +246,7 @@ Releases all stuck claimed items back to their stage. Use when agents have crash
 Parameters:
 - `pipeline` (string)
 - `stage` (string, optional) — limit to a specific stage. Defaults to `*`
-- `ids`, `createdAfter`, `createdBefore`, `modifiedAfter` — optional filters
+- `ids`, `created_after`, `created_before`, `modified_after` — optional filters
 
 Returns:
 ```yaml
@@ -296,41 +285,42 @@ Returns a code snippet to inline at the top of your script. The snippet uses onl
 QQ.push(pipeline, stage, id?, payload?, opts?)       → Promise<{ id: string } | { error: string }>
 QQ.claim(pipeline, stage, opts?)                     → Promise<Item | { error: string }>
 QQ.release(pipeline, seq, opts?)                     → Promise<{ ok: true } | { error: string }>
-QQ.batch_read(pipeline, stage, opts?)                → Promise<Item[] | { error: string }>
+QQ.query(pipeline, stage, opts?)                     → Promise<Item[] | { error: string }>
 QQ.status(pipeline)                                  → Promise<{ description: string|null, stats: Record<string, [number,number]> } | { error: string }>
 ```
 
-- `push` opts: `priority` (number), `payloadFormat` (`"yaml"` | `"json"` | `"text"`)
-- `claim` opts: `id` (string — claim a specific item), `strategy` (`"priority"` | `"fifo"` | `"lifo"` | `"random"`)
-- `release` opts: `target` (string), `payload` (string), `replace` (boolean), `priority` (number), `payloadFormat`
-- `batch_read` opts: `includePayload` (boolean), `claimed` (boolean), `ids` (string, comma-separated), `createdAfter`, `createdBefore`, `modifiedAfter` (epoch ms, ISO 8601, or relative e.g. `"-2h"`), `limit`, `offset` (number), `strategy` (`"priority"` | `"fifo"` | `"lifo"` | `"random"`)
+- `push` opts: `priority` (number)
+- `claim` opts: `id` (string — claim a specific item), `sort_order` (`"priority"` | `"fifo"` | `"lifo"` | `"random"`)
+- `release` opts: `target` (string), `payload` (object), `replace` (boolean), `priority` (number)
+- `query` opts: `payload_fields` (`"*"` or string[] of dot-notation paths), `payload_filter` (object — MongoDB-style payload filter), `claimed` (boolean), `ids` (string, comma-separated), `created_after`, `created_before`, `modified_after` (epoch ms, ISO 8601, or relative e.g. `"-2h"`), `limit`, `offset` (number), `sort_order` (`"priority"` | `"fifo"` | `"lifo"` | `"random"`)
 
 All functions return `{ error: string }` on failure — never throw (except `BridgeError` for network connection failures).
 
-Item shape (returned by `claim` and `batch_read`):
+Item shape (returned by `claim` and `query`):
 ```
 { id: string, stage: string, seq: number, priority: number, claimed: boolean,
-  created: number, last_modified: number, payload?: string | null }
+  created: number, last_modified: number, payload?: object | null }
 ```
 
-`claim` always includes `payload`. `batch_read` omits `payload` unless `includePayload: true`.
+`claim` always includes `payload`. `query` omits `payload` unless `select` is provided.
+`payload` is a parsed JavaScript object (or null) — never a string.
 
 **Python signatures:**
 ```
-qq_push(pipeline, stage, id=None, payload=None, **opts)  → { 'id': str } | { 'error': str }
-qq_claim(pipeline, stage, **opts)                        → dict | { 'error': str }
-qq_release(pipeline, seq, **opts)                        → { 'ok': True } | { 'error': str }
-qq_batch_read(pipeline, stage, **opts)                   → list[dict] | { 'error': str }
-qq_status(pipeline)                                      → { 'description': str|None, 'stats': dict } | { 'error': str }
+qq_push(pipeline, stage, id=None, payload=None, **opts)                          → { 'id': str } | { 'error': str }
+qq_claim(pipeline, stage, **opts)                                                 → dict | { 'error': str }
+qq_release(pipeline, seq, **opts)                                                 → { 'ok': True } | { 'error': str }
+qq_query(pipeline, stage, payload_filter=None, payload_fields=None, **opts)      → list[dict] | { 'error': str }
+qq_status(pipeline)                                                               → { 'description': str|None, 'stats': dict } | { 'error': str }
 ```
 
 All Python functions return `{ 'error': str }` on failure — never raise (except `BridgeError` for connection failures). Error messages are identical to those in the MCP tool reference above.
 
 ---
 
-### load_file
+### upload_file
 
-Loads multiple items from a file on the host filesystem into a pipeline. Each record in the file becomes one pipeline item.
+Uploads multiple items from a file on the host filesystem into a pipeline. Each record in the file becomes one pipeline item.
 
 **Supported formats:** JSONL (`.jsonl`, `.ndjson`), JSON array (`.json`), YAML array (`.yaml`, `.yml`), CSV (`.csv`). Format is detected from the file extension.
 
@@ -340,8 +330,8 @@ Parameters:
 - `path` (string) — path to the file. Supports `$VAR` expansion.
 - `pipeline` (string) — pipeline to push records into
 - `stage` (string, optional) — default stage for records that do not specify their own. Defaults to empty string.
-- `deleteAfter` (boolean, optional) — delete the file after loading. Default: true.
-- `duplicates` (`ignore` | `append` | `replace`, optional, default `ignore`) — how to handle records whose `id` already exists in the pipeline. `ignore`: skip silently (existing item unchanged). `append`: append the new record's payload to the existing item's payload and bump `seq` (invalidates any live claim). `replace`: replace the existing item's payload entirely and bump `seq`.
+- `delete_after` (boolean, optional) — delete the file after loading. Default: true.
+- `duplicates` (`ignore` | `append` | `replace`, optional, default `ignore`) — how to handle records whose `id` already exists in the pipeline. `ignore`: skip silently (existing item unchanged). `append`: merge the new record's payload into the existing payload using json_patch, bump `seq` (invalidates any live claim). `replace`: replace the existing item's payload entirely and bump `seq`.
 
 **⚠ Avoid `append` and `replace` in production pipelines.** Both modes write to items outside the normal `claim → process → release` cycle, bypassing QQ's concurrency control. Any agent currently holding a claim on an affected item will find its `seq` token expired and lose its work. `replace` is especially destructive — it silently discards the existing payload with no recovery path. The correct way to update an item's payload is to `claim` it, process it, and `release` it with new content. Use `append`/`replace` only for offline data preparation before a pipeline has active workers.
 
@@ -349,14 +339,18 @@ Parameters:
 ```yaml
 id: item-1
 stage: pending
-payload: "any string"
+payload:
+  url: https://example.com
+  score: 42
 priority: 0.0
 ```
 - `id` — omit for auto-generated id
 - `stage` — overrides the `stage` parameter for this record
-- `payload` — treated as an opaque string; non-string values are YAML-serialized
+- `payload` — structured object. String values are parsed as YAML for backward compatibility.
 - `priority` — float, default 0.0
 
+**CSV format**: for CSV files, all columns except `id`, `stage`, and `priority` become the payload object. A CSV row `id,url,title` with values `a,https://x.com,Hello` produces payload `{url: "https://x.com", title: "Hello"}`.
+
 **Returns** YAML with a summary of counts and a per-item outcome list:
 ```yaml
 summary:
@@ -384,7 +378,7 @@ Reads and returns the raw content of a file on the host filesystem. Use this to
 
 Parameters:
 - `path` (string) — path to the file. Supports `$VAR` expansion.
-- `deleteAfter` (boolean, optional) — delete the file after reading. Default: true.
+- `delete_after` (boolean, optional) — delete the file after reading. Default: true.
 
 Returns the file content as plain text.
 
@@ -393,24 +387,22 @@ Errors:
 
 ---
 
-### batch_write
+### upload_data
 
-Same as `load_file` but accepts data inline instead of reading from a file. Use when the records are already in memory or generated on the fly.
+Same as `upload_file` but accepts data inline instead of reading from a file. Use when the records are already in memory or generated on the fly.
 
 Parameters:
 - `data` (string) — the raw record data
 - `format` (`jsonl` | `json` | `yaml` | `csv`) — data format (explicit, no file extension to infer from)
 - `pipeline` (string)
 - `stage` (string, optional) — default stage. Defaults to empty string.
-- `duplicates` (`ignore` | `append` | `replace`, optional, default `ignore`) — same semantics as `load_file`. The same warning applies: prefer `claim → process → release` over `append`/`replace` whenever workers may be active.
+- `duplicates` (`ignore` | `append` | `replace`, optional, default `ignore`) — same semantics as `upload_file`. The same warning applies: prefer `claim → process → release` over `append`/`replace` whenever workers may be active.
 
-Returns the same shape as `load_file` (`summary` + `items`), without the `deleteAfter` behaviour.
+Returns the same shape as `upload_file` (`summary` + `items`), without the `deleteAfter` behaviour.
 
 Errors:
 - `error: <message>` — malformed data. Entire operation aborted; no records are pushed.
 
----
-
 ## SDK Usage
 
 Use the SDK when you need to run code that processes pipeline items programmatically — loops, transforms, bulk operations. It is faster and more efficient than calling MCP tools one at a time for batch work.
@@ -432,25 +424,31 @@ Use the SDK when you need to run code that processes pipeline items programmatic
 
 const pipeline = '0529_a3f9c2';
 
-// Bulk push — payload is a YAML string
+// Bulk push — payload is a plain object
 for (const url of urls) {
-  await QQ.push(pipeline, 'pending', url, `url: ${url}`);
+  await QQ.push(pipeline, 'pending', url, { url });
 }
 
 // Claim loop — check result.error, not null
 while (true) {
   const item = await QQ.claim(pipeline, 'pending');
   if (item.error) break;                             // "no items to claim" or other error
-  const result = await processItem(item.payload);
-  await QQ.release(pipeline, item.seq, { target: 'done', payload: result });
+  const result = await processItem(item.payload);    // item.payload is already a parsed object
+  await QQ.release(pipeline, item.seq, { target: 'done', payload: { result } });
 }
 
 // Collect results with payloads
-const done = await QQ.batch_read(pipeline, 'done', { includePayload: true });
+const done = await QQ.query(pipeline, 'done', { payload_fields: '*' });
 for (const item of done) {
-  console.log(item.id, item.payload);
+  console.log(item.id, item.payload);  // item.payload is a parsed object
 }
 
+// Filter by payload fields and select only specific fields
+const highScores = await QQ.query(pipeline, 'done', {
+  payload_filter: { score: { $gt: 80 }, status: 'active' },
+  payload_fields: ['url', 'score'],
+});
+
 // Monitor progress
 const s = await QQ.status(pipeline);
 console.log(`total=${s.stats['/*'][0]} pending=${s.stats.pending?.[0] ?? 0}`);
@@ -466,34 +464,32 @@ while (true) {
   if (item.error) break;
   try {
     const result = await processItem(item.payload);
-    await QQ.release(pipeline, item.seq, { target: 'done', payload: `result: ${result}` });
+    await QQ.release(pipeline, item.seq, { target: 'done', payload: { result } });
   } catch (e) {
-    await QQ.release(pipeline, item.seq, { target: 'failed', payload: `error: ${e.message}` });
+    await QQ.release(pipeline, item.seq, { target: 'failed', payload: { error: e.message } });
   }
 }
 ```
 
-### Structured JSON payloads
+### Working with structured payloads
 
-Use `payloadFormat: "json"` when payload is a JSON object. Appended payloads form valid JSONL; `replace: true` treats it as a single mutable object:
+Payloads are plain objects. `release` merges them by default (json_patch); `replace: true` overwrites:
 
 ```javascript
-// Push with JSON payload
-await QQ.push(pipeline, 'pending', 'item-1', JSON.stringify({ url: 'https://a.com', score: 0 }),
-  { payloadFormat: 'json' });
+// Push with a structured payload object
+await QQ.push(pipeline, 'pending', 'item-1', { url: 'https://a.com', score: 0 });
 
-// Update a structured payload: parse → modify → replace
+// Claim returns payload as a parsed object — no JSON.parse needed
 const item = await QQ.claim(pipeline, 'pending');
 if (!item.error) {
-  const data = JSON.parse(item.payload);
-  data.score = await scoreItem(data.url);
-  await QQ.release(pipeline, item.seq, {
-    target: 'done',
-    payload: JSON.stringify(data),
-    payloadFormat: 'json',
-    replace: true,
-  });
+  const score = await scoreItem(item.payload.url);
+  // Merge new fields into existing payload (json_patch)
+  await QQ.release(pipeline, item.seq, { target: 'done', payload: { score } });
+  // Result: { url: 'https://a.com', score: <value> }
 }
+
+// Replace entire payload (discard existing)
+await QQ.release(pipeline, item.seq, { payload: { rebuilt: true }, replace: true });
 ```
 
 ### Python example
@@ -503,21 +499,26 @@ if (!item.error) {
 
 pipeline = "0529_a3f9c2"
 
-# Bulk push
+# Bulk push — payload is a dict
 for url in urls:
-    qq_push(pipeline, "pending", url, f"url: {url}")
+    qq_push(pipeline, "pending", url, {"url": url})
 
-# Claim loop
+# Claim loop — item["payload"] is already a parsed dict
 while True:
     item = qq_claim(pipeline, "pending")
     if "error" in item:  # "no items to claim" or other error
         break
-    result = process_item(item["payload"])
-    qq_release(pipeline, item["seq"], target="done", payload=result)
+    result = process_item(item["payload"])   # payload is a dict
+    qq_release(pipeline, item["seq"], target="done", payload={"result": result})
 
-items = qq_batch_read(pipeline, "done", includePayload=True)
+items = qq_query(pipeline, "done", payload_fields="*")
 for item in items:
-    print(item["id"], item["payload"])
+    print(item["id"], item["payload"])  # payload is a dict
+
+# Filter by payload fields; select only specific fields
+high_scores = qq_query(pipeline, "done",
+    payload_filter={"score": {"$gt": 80}, "status": "active"},
+    payload_fields=["url", "score"])
 
 # Monitor
 s = qq_status(pipeline)
@@ -535,39 +536,41 @@ This happens when the QQ MCP server restarted (e.g. Claude Desktop was restarted
 
 ---
 
-## YAML Payload Conventions
+## Structured Payload and Merge Semantics
 
-Payload is a YAML string appended to existing content by default. The result is a YAML document where fields may appear multiple times — this is intentional. Each stage's output appears in sequence, creating a trace of the item's journey.
+Payloads are structured JSON objects (or arrays), stored as SQLite JSONB. Plain text is not allowed.
+
+**Default merge — json_patch (RFC 7396):** When `release` or `upload_data append` receives a payload, it is merged into the existing payload using RFC 7396 semantics:
+- Keys in the patch with a non-null value are set on the target (recursively for nested objects)
+- Keys in the patch set to `null` are **removed** from the target
+- Keys absent from the patch are left unchanged
 
-```yaml
-# After push:
-url: https://example.com/product/42
-
-# After first subagent appends:
-url: https://example.com/product/42
-name: Widget Pro
-price: 29.99
-available: true
-
-# After second subagent appends (review):
-url: https://example.com/product/42
-name: Widget Pro
-price: 29.99
-available: true
-review_reason: price seems anomalous
 ```
+# Existing:   { url: "https://x.com", score: 0, tags: ["a"], draft: true }
+# Patch:      { score: 42, extra: "new", draft: null }
+# Result:     { url: "https://x.com", score: 42, tags: ["a"], extra: "new" }
+#             (draft removed because patch set it to null)
+```
+
+**Replace:** Pass `replace: true` to discard the existing payload entirely and store only the new value.
+
+**No payload:** Omit `payload` from `release` to leave the existing payload unchanged (useful when only moving stage or updating priority).
 
-Use `replace: true` only when you own the payload as a structured object (e.g., accumulating messages in a chat pipeline). To update a structured payload: claim, parse with yaml.load, modify, release with replace.
+**Null payload:** An item pushed with no `payload` argument stores SQL NULL — `claim` and `query` return it as `null`.
+
+**CLI and SDK string input:** The CLI (`--payload`) and `payloadFormat` option accept YAML or JSON strings that are parsed before storage. The `text` format has been removed — all inputs must be valid YAML or JSON.
 
 ---
 
 ## Globs, Filters, Priority
 
-**Stage globs**: `*` matches any single segment, `enriched/*` matches sub-stages. Use in `claim` and `batch_read` to work across multiple stages.
+**Stage globs**: `*` matches any single segment, `enriched/*` matches sub-stages. Use in `claim` and `query` to work across multiple stages.
 
 **Priority**: float, default 0.0. Higher value = claimed first within the same stage. Use only when meaningful (e.g., score, confidence). Most pipelines don't need priority.
 
-**Filters in batch_read / unstick**: `claimed`, `ids`, `createdAfter`, `createdBefore`, `modifiedAfter` (epoch ms, ISO 8601, or relative: `"-2h"`, `"-1d"`, `"-30m"`, `"-1w"`), `limit`, `offset`. **batch_read** also accepts `strategy` (`priority` | `fifo` | `lifo` | `random`, default `fifo`).
+**Filters in query / unstick**: `claimed`, `ids`, `created_after`, `created_before`, `modified_after` (epoch ms, ISO 8601, or relative: `"-2h"`, `"-1d"`, `"-30m"`, `"-1w"`), `limit`, `offset`. **query** also accepts `sort_order` (`priority` | `fifo` | `lifo` | `random`, default `fifo`), `payload_filter` (MongoDB-style payload filter), and `payload_fields` (`"*"` or array of dot-notation field paths).
+
+**Payload filter operators** (`filter` in `query`): `{ "field": value }` (equality), `{ "field": { "$gt": n } }`, `$gte`, `$lt`, `$lte`, `$ne`, `{ "field": { "$in": [a, b] } }`, `{ "field": { "$exists": true } }`. Nested fields use dot notation: `"meta.author"`. All conditions are ANDed together.
 
 ---
 

package/src/sdk-templates/javascript.ts

@@ -13,7 +13,7 @@ const QQ = (() => {
     push:        (pipeline, stage, id, payload, opts = {}) => _req('POST', \`/pipelines/\${pipeline}/push\`, { stage, id, payload, ...opts }),
     claim:       (pipeline, stage, opts = {})               => _req('POST', \`/pipelines/\${pipeline}/claim\`, { stage, ...opts }),
     release:     (pipeline, seq, opts = {})                 => _req('POST', \`/pipelines/\${pipeline}/release/\${seq}\`, opts),
-    batch_read:  (pipeline, stage, opts = {})               => _req('GET',  \`/pipelines/\${pipeline}/items?\${new URLSearchParams({ stage, ...opts })}\`),
+    query:       (pipeline, stage, { payload_filter, payload_fields, ...opts } = {}) => { const p = { stage, ...opts }; if (payload_filter != null) p.payload_filter = JSON.stringify(payload_filter); if (payload_fields != null) p.payload_fields = Array.isArray(payload_fields) ? payload_fields.join(',') : payload_fields; return _req('GET', \`/pipelines/\${pipeline}/items?\${new URLSearchParams(p)}\`); },
     status:      (pipeline)                                 => _req('GET',  \`/pipelines/\${pipeline}/status\`),
   };
 })();`;

package/src/sdk-templates/python.ts

@@ -19,8 +19,11 @@ def qq_claim(pipeline, stage, **opts):
     return _bridge_req('POST', f'/pipelines/{pipeline}/claim', {'stage': stage, **opts})
 def qq_release(pipeline, seq, **opts):
     return _bridge_req('POST', f'/pipelines/{pipeline}/release/{seq}', opts)
-def qq_batch_read(pipeline, stage, **opts):
-    qs = urllib.parse.urlencode({'stage': stage, **opts})
+def qq_query(pipeline, stage, payload_filter=None, payload_fields=None, **opts):
+    params = {'stage': stage, **opts}
+    if payload_filter is not None: params['payload_filter'] = _json.dumps(payload_filter)
+    if payload_fields is not None: params['payload_fields'] = ','.join(payload_fields) if isinstance(payload_fields, list) else payload_fields
+    qs = urllib.parse.urlencode(params)
     return _bridge_req('GET', f'/pipelines/{pipeline}/items?{qs}')
 def qq_status(pipeline):
     return _bridge_req('GET', f'/pipelines/{pipeline}/status')`;