@quantod/qq 1.1.19 → 1.2.0

package/src/commands.ts

@@ -8,22 +8,27 @@ import { openDb, inTransaction } from './db.js';
 
 export * as yaml from 'js-yaml';
 
-export type PayloadFormat = 'yaml' | 'json' | 'text';
-
-function processPayload(payload: string | undefined, format: PayloadFormat = 'yaml'): string | undefined {
-  if (!payload) return payload;
-  switch (format) {
-    case 'yaml':
-      try { return dump(load(payload, { json: true })).trimEnd(); }
-      catch (e) { throw new QQError(`payload is not valid YAML: ${e instanceof Error ? e.message : String(e)}`); }
-    case 'json':
-      try { return JSON.stringify(JSON.parse(payload)); }
-      catch (e) { throw new QQError(`payload is not valid JSON: ${e instanceof Error ? e.message : String(e)}`); }
-    case 'text':
-      return payload;
+export type PayloadFormat = 'yaml' | 'json';
+export type PayloadSelect = '*' | string[];
+
+function parsePayload(input: string | object | undefined, format: PayloadFormat = 'yaml'): unknown {
+  if (input === undefined || input === null) return undefined;
+  if (typeof input !== 'string') return input;
+  if (format !== 'yaml' && format !== 'json')
+    throw new QQError(`unsupported payload format: "${String(format)}" — use 'yaml' or 'json'`);
+  if (format === 'json') {
+    try { return JSON.parse(input); }
+    catch (e) { throw new QQError(`payload is not valid JSON: ${e instanceof Error ? e.message : String(e)}`); }
+  } else {
+    try { return load(input, { json: true }); }
+    catch (e) { throw new QQError(`payload is not valid YAML: ${e instanceof Error ? e.message : String(e)}`); }
   }
 }
 
+function toJsonArg(val: unknown): string | null {
+  return val === undefined || val === null ? null : JSON.stringify(val);
+}
+
 // ── types ──────────────────────────────────────────────────────────────────────
 
 export class QQError extends Error {
@@ -41,7 +46,7 @@ export interface Item {
   priority: number;
   created: string;
   last_modified: string;
-  payload?: string | null;
+  payload?: unknown;
 }
 
 export interface FilterOptions {
@@ -52,11 +57,100 @@ export interface FilterOptions {
   modified_after?: string | number;
   limit?: number;
   offset?: number;
+  filter?: Record<string, unknown>;
+  select?: PayloadSelect;
 }
 
 
 // ── internal helpers ───────────────────────────────────────────────────────────
 
+// SELECT that converts JSONB payload back to JSON text for parsing
+const MSG_SELECT = 'SELECT *, json(payload) AS payload_text FROM messages';
+
+function toSqlParam(v: unknown): string | number | null {
+  if (v === null || v === undefined) return null;
+  if (typeof v === 'boolean') return v ? 1 : 0;
+  if (typeof v === 'number') return v;
+  if (typeof v === 'string') return v;
+  return JSON.stringify(v);
+}
+
+function sanitizeJsonPath(key: string): string {
+  if (!/^[a-zA-Z0-9_.:-]+$/.test(key))
+    throw new QQError(`invalid filter key "${key}": use alphanumeric characters, dots, underscores, or hyphens`);
+  return `$.${key}`;
+}
+
+function buildPayloadFilter(filter: Record<string, unknown>): { conditions: string[]; params: (string | number | null)[] } {
+  const conditions: string[] = [];
+  const params: (string | number | null)[] = [];
+
+  for (const [key, value] of Object.entries(filter)) {
+    const path = sanitizeJsonPath(key);
+    const extract = `json_extract(payload, '${path}')`;
+    const numExtract = `CAST(${extract} AS REAL)`;
+
+    if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
+      for (const [op, opVal] of Object.entries(value as Record<string, unknown>)) {
+        switch (op) {
+          case '$eq':   conditions.push(`${extract} = ?`);     params.push(toSqlParam(opVal)); break;
+          case '$ne':   conditions.push(`${extract} != ?`);    params.push(toSqlParam(opVal)); break;
+          case '$gt':   conditions.push(`${numExtract} > ?`);  params.push(toSqlParam(opVal)); break;
+          case '$gte':  conditions.push(`${numExtract} >= ?`); params.push(toSqlParam(opVal)); break;
+          case '$lt':   conditions.push(`${numExtract} < ?`);  params.push(toSqlParam(opVal)); break;
+          case '$lte':  conditions.push(`${numExtract} <= ?`); params.push(toSqlParam(opVal)); break;
+          case '$in':
+            if (!Array.isArray(opVal) || opVal.length === 0)
+              throw new QQError(`$in requires a non-empty array`);
+            conditions.push(`${extract} IN (${(opVal as unknown[]).map(() => '?').join(', ')})`);
+            params.push(...(opVal as unknown[]).map(toSqlParam));
+            break;
+          case '$exists':
+            conditions.push(`${extract} IS ${opVal ? 'NOT ' : ''}NULL`);
+            break;
+          default:
+            throw new QQError(`unsupported filter operator: ${op}`);
+        }
+      }
+    } else if (value === null) {
+      conditions.push(`${extract} IS NULL`);
+    } else {
+      conditions.push(`${extract} = ?`);
+      params.push(toSqlParam(value));
+    }
+  }
+
+  return { conditions, params };
+}
+
+function projectPayload(payload: unknown, select: string[]): unknown {
+  if (payload === null || payload === undefined || typeof payload !== 'object' || Array.isArray(payload))
+    return payload;
+  const source = payload as Record<string, unknown>;
+  const result: Record<string, unknown> = {};
+
+  for (const path of select) {
+    const parts = path.split('.');
+    let src: unknown = source;
+    let dst = result;
+    let ok = true;
+
+    for (let i = 0; i < parts.length - 1; i++) {
+      const part = parts[i];
+      if (typeof src !== 'object' || src === null) { ok = false; break; }
+      src = (src as Record<string, unknown>)[part];
+      if (!(part in dst) || typeof dst[part] !== 'object') dst[part] = {};
+      dst = dst[part] as Record<string, unknown>;
+    }
+
+    if (ok && typeof src === 'object' && src !== null) {
+      const lastPart = parts[parts.length - 1];
+      dst[lastPart] = (src as Record<string, unknown>)[lastPart];
+    }
+  }
+
+  return result;
+}
 
 function toGlob(pattern: string): string {
   return pattern.replace(/\*/g, '%').replace(/\?/g, '_');
@@ -72,7 +166,7 @@ function nextSeq(db: Database.Database, pipeline: string): number {
   return (row.m ?? 0) + 1;
 }
 
-function rowToItem(row: Record<string, unknown>, includePayload: boolean): Item {
+function rowToItem(row: Record<string, unknown>, select?: PayloadSelect): Item {
   const msg: Item = {
     id: row.id as string,
     stage: row.subqueue as string,
@@ -82,7 +176,11 @@ function rowToItem(row: Record<string, unknown>, includePayload: boolean): Item
     created: new Date(row.created as number).toISOString(),
     last_modified: new Date(row.last_modified as number).toISOString(),
   };
-  if (includePayload) msg.payload = (row.payload as string | null) ?? null;
+  if (select !== undefined) {
+    const pt = row.payload_text as string | null;
+    const parsed = pt != null ? JSON.parse(pt) : null;
+    msg.payload = select === '*' ? parsed : projectPayload(parsed, select);
+  }
   return msg;
 }
 
@@ -137,13 +235,13 @@ export function deletePipeline(pipeline: string): void {
 // ── push ──────────────────────────────────────────────────────────────────────
 
 export interface PushOptions {
-  payload?: string;
+  payload?: string | object;
   priority?: number;
   payloadFormat?: PayloadFormat;
 }
 
 export function push(pipeline: string, stage = '', id?: string, { payload, priority = 0.0, payloadFormat = 'yaml' }: PushOptions = {}): string {
-  const processedPayload = processPayload(payload, payloadFormat);
+  const parsedPayload = parsePayload(payload, payloadFormat);
   const db = openDb();
   try {
     return inTransaction(db, () => {
@@ -154,8 +252,8 @@ export function push(pipeline: string, stage = '', id?: string, { payload, prior
       if (existing) throw new QQError(`item ${actualId} already in the pipeline`);
       const now = Date.now();
       db.prepare(
-        'INSERT INTO messages (pipeline, id, subqueue, claimed, seq, payload, priority, created, last_modified) VALUES (?, ?, ?, 0, ?, ?, ?, ?, ?)'
-      ).run(pipeline, actualId, stage, seq, processedPayload ?? null, priority, now, now);
+        'INSERT INTO messages (pipeline, id, subqueue, claimed, seq, payload, priority, created, last_modified) VALUES (?, ?, ?, 0, ?, jsonb(?), ?, ?, ?)'
+      ).run(pipeline, actualId, stage, seq, toJsonArg(parsedPayload), priority, now, now);
       return actualId;
     });
   } finally {
@@ -163,7 +261,7 @@ export function push(pipeline: string, stage = '', id?: string, { payload, prior
   }
 }
 
-// ── batchWrite / loadFile ─────────────────────────────────────────────────────
+// ── uploadData / uploadFile ───────────────────────────────────────────────────
 
 export type DuplicateMode = 'ignore' | 'append' | 'replace';
 export type BatchWriteFormat = 'jsonl' | 'json' | 'yaml' | 'csv';
@@ -223,7 +321,27 @@ function parseRecords(data: string, format: BatchWriteFormat): Record<string, un
   }
 }
 
-export function batchWrite(
+// For CSV: the whole row (minus reserved fields) becomes the payload.
+// For other formats: use the rec.payload field; string values are parsed as YAML for backward compat.
+const CSV_RESERVED = new Set(['id', 'stage', 'priority']);
+
+function extractRecordPayload(rec: Record<string, unknown>, format: BatchWriteFormat): unknown {
+  if (format === 'csv') {
+    const result: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(rec)) {
+      if (!CSV_RESERVED.has(k)) result[k] = v;
+    }
+    return Object.keys(result).length > 0 ? result : undefined;
+  }
+  const p = rec.payload;
+  if (p === null || p === undefined) return undefined;
+  if (typeof p === 'string') {
+    try { return load(p, { json: true }); } catch { return { _text: p }; }
+  }
+  return p;
+}
+
+export function uploadData(
   data: string,
   format: BatchWriteFormat,
   pipeline: string,
@@ -239,21 +357,19 @@ export function batchWrite(
       const recStage = typeof rec.stage === 'string' ? rec.stage : defaultStage;
       const recId = rec.id !== undefined ? String(rec.id) : undefined;
       const recPriority = rec.priority !== undefined ? Number(rec.priority) : 0.0;
-      const recPayload = rec.payload != null
-        ? (typeof rec.payload === 'string' ? rec.payload : dump(rec.payload))
-        : undefined;
+      const recPayload = extractRecordPayload(rec, format);
 
       try {
         const result = inTransaction(db, (): LoadFileResult => {
           const seq = nextSeq(db, pipeline);
           const actualId = recId ?? String(seq);
-          const existing = db.prepare('SELECT * FROM messages WHERE pipeline = ? AND id = ?').get(pipeline, actualId) as Record<string, unknown> | undefined;
+          const existing = db.prepare('SELECT id FROM messages WHERE pipeline = ? AND id = ?').get(pipeline, actualId);
           const now = Date.now();
 
           if (!existing) {
             db.prepare(
-              'INSERT INTO messages (pipeline, id, subqueue, claimed, seq, payload, priority, created, last_modified) VALUES (?, ?, ?, 0, ?, ?, ?, ?, ?)'
-            ).run(pipeline, actualId, recStage, seq, recPayload ?? null, recPriority, now, now);
+              'INSERT INTO messages (pipeline, id, subqueue, claimed, seq, payload, priority, created, last_modified) VALUES (?, ?, ?, 0, ?, jsonb(?), ?, ?, ?)'
+            ).run(pipeline, actualId, recStage, seq, toJsonArg(recPayload), recPriority, now, now);
             return { id: actualId, status: 'new' };
           }
 
@@ -261,19 +377,23 @@ export function batchWrite(
             return { id: actualId, status: 'duplicate' };
           }
 
-          let newPayload: string | null;
-          if (duplicates === 'append') {
-            const existingPayload = (existing.payload as string | null)?.trimEnd() ?? null;
-            newPayload = existingPayload && recPayload
-              ? `${existingPayload}\n${recPayload}`
-              : (recPayload ?? existingPayload);
+          if (duplicates === 'replace') {
+            db.prepare(
+              'UPDATE messages SET claimed=0, seq=?, subqueue=?, payload=jsonb(?), last_modified=? WHERE pipeline=? AND id=?'
+            ).run(seq, recStage, toJsonArg(recPayload), now, pipeline, actualId);
           } else {
-            newPayload = recPayload ?? null;
+            // 'append' → json_patch merge
+            if (recPayload !== undefined) {
+              const jsonNew = JSON.stringify(recPayload);
+              db.prepare(
+                'UPDATE messages SET claimed=0, seq=?, subqueue=?, payload=CASE WHEN payload IS NULL THEN jsonb(?) ELSE jsonb(json_patch(payload, ?)) END, last_modified=? WHERE pipeline=? AND id=?'
+              ).run(seq, recStage, jsonNew, jsonNew, now, pipeline, actualId);
+            } else {
+              db.prepare(
+                'UPDATE messages SET claimed=0, seq=?, subqueue=?, last_modified=? WHERE pipeline=? AND id=?'
+              ).run(seq, recStage, now, pipeline, actualId);
+            }
           }
-
-          db.prepare(
-            'UPDATE messages SET claimed = 0, seq = ?, subqueue = ?, payload = ?, last_modified = ? WHERE pipeline = ? AND id = ?'
-          ).run(seq, recStage, newPayload, now, pipeline, actualId);
           return { id: actualId, status: 'duplicate' };
         });
         results.push(result);
@@ -288,14 +408,14 @@ export function batchWrite(
   return results;
 }
 
-export function loadFile(
+export function uploadFile(
   rawPath: string,
   pipeline: string,
   { stage, deleteAfter = true, duplicates }: LoadFileOptions = {}
 ): LoadFileResult[] {
   const filePath = expandEnvVars(rawPath);
   const content = readFileSync(filePath, 'utf8');
-  const results = batchWrite(content, formatFromPath(filePath), pipeline, { stage, duplicates });
+  const results = uploadData(content, formatFromPath(filePath), pipeline, { stage, duplicates });
   if (deleteAfter && results.every(r => r.status !== 'error')) {
     try { unlinkSync(filePath); } catch { /* best-effort */ }
   }
@@ -335,13 +455,13 @@ export function claim(pipeline: string, stage: string, id?: string, { strategy =
 
       if (id !== undefined) {
         row = db.prepare(
-          'SELECT * FROM messages WHERE pipeline = ? AND id = ?'
+          `${MSG_SELECT} WHERE pipeline = ? AND id = ?`
         ).get(pipeline, id) as Record<string, unknown> | undefined;
         if (!row) throw new QQError(`item ${id} not found`);
         if (row.claimed) throw new QQError(`item ${id} already claimed`);
       } else {
         row = db.prepare(
-          `SELECT * FROM messages WHERE pipeline = ? AND subqueue LIKE ? AND claimed = 0 ORDER BY ${CLAIM_ORDER[strategy]} LIMIT 1`
+          `${MSG_SELECT} WHERE pipeline = ? AND subqueue LIKE ? AND claimed = 0 ORDER BY ${CLAIM_ORDER[strategy]} LIMIT 1`
         ).get(pipeline, toGlob(stage)) as Record<string, unknown> | undefined;
         if (!row) throw new QQError('no items to claim');
       }
@@ -351,7 +471,7 @@ export function claim(pipeline: string, stage: string, id?: string, { strategy =
         'UPDATE messages SET claimed = 1, last_modified = ? WHERE pipeline = ? AND id = ?'
       ).run(now, pipeline, row.id as string);
 
-      return rowToItem({ ...row, claimed: 1 }, true);
+      return rowToItem({ ...row, claimed: 1 }, '*');
     });
   } finally {
     db.close();
@@ -362,39 +482,45 @@ export function claim(pipeline: string, stage: string, id?: string, { strategy =
 
 export interface ReleaseOptions {
   target?: string;
-  payload?: string;
+  payload?: string | object;
   replace?: boolean;
   priority?: number;
   payloadFormat?: PayloadFormat;
 }
 
 export function release(pipeline: string, seq: number, { target, payload, replace, priority, payloadFormat = 'yaml' }: ReleaseOptions = {}): void {
-  const processedPayload = processPayload(payload, payloadFormat);
+  const parsedPayload = parsePayload(payload, payloadFormat);
   const db = openDb();
   try {
     inTransaction(db, () => {
       assertPipelineExists(db, pipeline);
-      const row = db.prepare('SELECT * FROM messages WHERE pipeline = ? AND seq = ?').get(pipeline, seq) as Record<string, unknown> | undefined;
+      const row = db.prepare(
+        'SELECT claimed, subqueue, priority FROM messages WHERE pipeline = ? AND seq = ?'
+      ).get(pipeline, seq) as { claimed: number; subqueue: string; priority: number } | undefined;
       if (!row) throw new QQError(`claim on seq ${seq} expired`);
       if (!row.claimed) throw new QQError(`item seq ${seq} not claimed`);
 
       const newSeq = nextSeq(db, pipeline);
       const now = Date.now();
-      const newStage = target ?? (row.subqueue as string);
-      const newPriority = priority ?? (row.priority as number) ?? 0.0;
-
-      let newPayload: string | null;
-      if (processedPayload !== undefined) {
-        newPayload = replace
-          ? processedPayload
-          : [(row.payload as string | null)?.trimEnd() ?? null, processedPayload].filter(Boolean).join('\n');
+      const newStage = target ?? row.subqueue;
+      const newPriority = priority ?? row.priority ?? 0.0;
+
+      if (parsedPayload !== undefined) {
+        if (replace) {
+          db.prepare(
+            'UPDATE messages SET claimed=0, seq=?, subqueue=?, payload=jsonb(?), priority=?, last_modified=? WHERE pipeline=? AND seq=?'
+          ).run(newSeq, newStage, toJsonArg(parsedPayload), newPriority, now, pipeline, seq);
+        } else {
+          const jsonNew = JSON.stringify(parsedPayload);
+          db.prepare(
+            'UPDATE messages SET claimed=0, seq=?, subqueue=?, payload=CASE WHEN payload IS NULL THEN jsonb(?) ELSE jsonb(json_patch(payload, ?)) END, priority=?, last_modified=? WHERE pipeline=? AND seq=?'
+          ).run(newSeq, newStage, jsonNew, jsonNew, newPriority, now, pipeline, seq);
+        }
       } else {
-        newPayload = (row.payload as string | null) ?? null;
+        db.prepare(
+          'UPDATE messages SET claimed=0, seq=?, subqueue=?, priority=?, last_modified=? WHERE pipeline=? AND seq=?'
+        ).run(newSeq, newStage, newPriority, now, pipeline, seq);
       }
-
-      db.prepare(
-        'UPDATE messages SET claimed = 0, seq = ?, subqueue = ?, payload = ?, priority = ?, last_modified = ? WHERE pipeline = ? AND seq = ?'
-      ).run(newSeq, newStage, newPayload, newPriority, now, pipeline, seq);
     });
   } finally {
     db.close();
@@ -439,9 +565,9 @@ function buildFilterConditions(pipeline: string, stage: string, filters: FilterO
   return { conditions, params };
 }
 
-// ── batchRead ─────────────────────────────────────────────────────────────────
+// ── query ─────────────────────────────────────────────────────────────────────
 
-export function batchRead(pipeline: string, stage = '*', filters: FilterOptions = {}, includePayload = true, strategy: ClaimStrategy = 'fifo'): Item[] {
+export function query(pipeline: string, stage = '*', filters: FilterOptions = {}, strategy: ClaimStrategy = 'fifo'): Item[] {
   const db = openDb();
   try {
     assertPipelineExists(db, pipeline);
@@ -452,7 +578,13 @@ export function batchRead(pipeline: string, stage = '*', filters: FilterOptions
       params.push(filters.claimed ? 1 : 0);
     }
 
-    let sql = `SELECT * FROM messages WHERE ${conditions.join(' AND ')} ORDER BY ${CLAIM_ORDER[strategy]}`;
+    if (filters.filter) {
+      const { conditions: payloadConditions, params: payloadParams } = buildPayloadFilter(filters.filter);
+      conditions.push(...payloadConditions);
+      params.push(...payloadParams);
+    }
+
+    let sql = `${MSG_SELECT} WHERE ${conditions.join(' AND ')} ORDER BY ${CLAIM_ORDER[strategy]}`;
 
     if (filters.limit !== undefined) {
       sql += ' LIMIT ?';
@@ -466,7 +598,7 @@ export function batchRead(pipeline: string, stage = '*', filters: FilterOptions
     }
 
     const rows = db.prepare(sql).all(...params) as Record<string, unknown>[];
-    return rows.map(r => rowToItem(r, includePayload));
+    return rows.map(r => rowToItem(r, filters.select));
   } finally {
     db.close();
   }

package/src/db.ts

@@ -1,6 +1,7 @@
 import Database from 'better-sqlite3';
 import { mkdirSync } from 'node:fs';
 import { join } from 'node:path';
+import { load } from 'js-yaml';
 
 const DDL = `
 CREATE TABLE IF NOT EXISTS pipelines (
@@ -15,7 +16,7 @@ CREATE TABLE IF NOT EXISTS messages (
   subqueue      TEXT NOT NULL,
   claimed       INTEGER NOT NULL DEFAULT 0,
   seq           INTEGER NOT NULL,
-  payload       TEXT,
+  payload       BLOB,
   created       INTEGER NOT NULL,
   last_modified INTEGER NOT NULL,
   priority      REAL NOT NULL DEFAULT 0.0,
@@ -30,6 +31,57 @@ function dbDir(): string {
   return join(process.env.HOME!, '.claude', 'qq');
 }
 
+function migratePayloadToJsonb(db: Database.Database): void {
+  const cols = db.prepare("PRAGMA table_info(messages)").all() as { name: string; type: string }[];
+  const payloadCol = cols.find(c => c.name === 'payload');
+  if (!payloadCol || payloadCol.type !== 'TEXT') return;
+
+  const rows = db.prepare('SELECT pipeline, id, payload FROM messages').all() as {
+    pipeline: string; id: string; payload: string | null;
+  }[];
+
+  db.transaction(() => {
+    db.prepare(`
+      CREATE TABLE messages_new (
+        id            TEXT NOT NULL,
+        pipeline      TEXT NOT NULL REFERENCES pipelines(id) ON DELETE CASCADE,
+        subqueue      TEXT NOT NULL,
+        claimed       INTEGER NOT NULL DEFAULT 0,
+        seq           INTEGER NOT NULL,
+        payload       BLOB,
+        created       INTEGER NOT NULL,
+        last_modified INTEGER NOT NULL,
+        priority      REAL NOT NULL DEFAULT 0.0,
+        PRIMARY KEY (pipeline, id)
+      )
+    `).run();
+
+    db.prepare(`
+      INSERT INTO messages_new (id, pipeline, subqueue, claimed, seq, payload, created, last_modified, priority)
+      SELECT id, pipeline, subqueue, claimed, seq, NULL, created, last_modified, priority
+      FROM messages
+    `).run();
+
+    const update = db.prepare('UPDATE messages_new SET payload = jsonb(?) WHERE pipeline = ? AND id = ?');
+    for (const row of rows) {
+      if (row.payload === null) continue;
+      let parsed: unknown;
+      try { parsed = JSON.parse(row.payload); }
+      catch {
+        try { parsed = load(row.payload, { json: true }); }
+        catch { parsed = { _text: row.payload }; }
+      }
+      update.run(JSON.stringify(parsed), row.pipeline, row.id);
+    }
+
+    db.prepare('DROP TABLE messages').run();
+    db.prepare('ALTER TABLE messages_new RENAME TO messages').run();
+    db.prepare('CREATE UNIQUE INDEX idx_seq_unique ON messages (pipeline, seq)').run();
+    db.prepare('CREATE INDEX idx_claim ON messages (pipeline, subqueue, claimed, seq)').run();
+    db.prepare('CREATE INDEX idx_claim_pri ON messages (pipeline, subqueue, claimed, priority, seq)').run();
+  })();
+}
+
 export function openDb(): Database.Database {
   const dir = dbDir();
   mkdirSync(dir, { recursive: true });
@@ -38,11 +90,14 @@ export function openDb(): Database.Database {
   db.pragma('busy_timeout = 5000');
   db.pragma('foreign_keys = ON');
   db.exec(DDL);
-  // Migration: add description column if missing (existing DBs pre-date this column)
-  const cols = db.prepare("PRAGMA table_info(pipelines)").all() as { name: string }[];
-  if (!cols.some(c => c.name === 'description')) {
+
+  const pipCols = db.prepare("PRAGMA table_info(pipelines)").all() as { name: string }[];
+  if (!pipCols.some(c => c.name === 'description')) {
     db.exec('ALTER TABLE pipelines ADD COLUMN description TEXT');
   }
+
+  migratePayloadToJsonb(db);
+
   return db;
 }
 

package/src/http.ts

@@ -4,7 +4,8 @@ import { createServer } from 'node:net';
 import { randomBytes } from 'node:crypto';
 import {
   makePipeline, deletePipeline, push, claim, release,
-  batchRead, status, unstick, loadFile, batchWrite, QQError,
+  query, status, unstick, uploadFile, uploadData, QQError,
+  type PayloadSelect, type ClaimStrategy,
 } from './commands.js';
 
 export interface HttpServer {
@@ -65,8 +66,8 @@ export async function startHttp(fixedPort?: number): Promise<HttpServer> {
   });
 
   app.post('/pipelines/:pipeline/push', (req, res) => {
-    const { stage, id, payload, priority, payloadFormat } = req.body ?? {};
-    wrap(res, () => ({ id: push(req.params.pipeline, stage, id, { payload, priority, payloadFormat }) }));
+    const { stage, id, payload, priority } = req.body ?? {};
+    wrap(res, () => ({ id: push(req.params.pipeline, stage, id, { payload, priority }) }));
   });
 
   app.post('/pipelines/:pipeline/claim', (req, res) => {
@@ -76,8 +77,8 @@ export async function startHttp(fixedPort?: number): Promise<HttpServer> {
 
   app.post('/pipelines/:pipeline/release/:seq', (req, res) => {
     const seq = parseInt(req.params.seq, 10);
-    const { target, payload, replace, priority, payloadFormat } = req.body ?? {};
-    wrap(res, () => { release(req.params.pipeline, seq, { target, payload, replace, priority, payloadFormat }); });
+    const { target, payload, replace, priority } = req.body ?? {};
+    wrap(res, () => { release(req.params.pipeline, seq, { target, payload, replace, priority }); });
   });
 
   app.get('/pipelines/:pipeline/status', (req, res) => {
@@ -85,38 +86,51 @@ export async function startHttp(fixedPort?: number): Promise<HttpServer> {
   });
 
   app.get('/pipelines/:pipeline/items', (req, res) => {
-    const { stage = '*', includePayload, claimed, ids, createdAfter, createdBefore, modifiedAfter, limit, offset, strategy } = req.query;
-    wrap(res, () => batchRead(
-      req.params.pipeline,
-      String(stage),
-      {
-        claimed: claimed !== undefined ? String(claimed).toLowerCase() === 'true' : undefined,
-        ids: ids ? String(ids).split(',') : undefined,
-        created_after: createdAfter ? String(createdAfter) : undefined,
-        created_before: createdBefore ? String(createdBefore) : undefined,
-        modified_after: modifiedAfter ? String(modifiedAfter) : undefined,
-        limit: limit ? Number(limit) : undefined,
-        offset: offset ? Number(offset) : undefined,
-      },
-      String(includePayload).toLowerCase() === 'true',
-      strategy ? String(strategy) as import('./commands.js').ClaimStrategy : undefined,
-    ));
+    const { stage = '*', filter: filterStr, select: selectStr, claimed, ids, createdAfter, createdBefore, modifiedAfter, limit, offset, strategy } = req.query;
+    wrap(res, () => {
+      let filter: Record<string, unknown> | undefined;
+      if (filterStr) {
+        try { filter = JSON.parse(String(filterStr)); }
+        catch { throw new QQError('filter must be valid JSON'); }
+      }
+      let select: PayloadSelect | undefined;
+      if (selectStr) {
+        const s = String(selectStr);
+        select = s === '*' ? '*' : s.split(',').map(f => f.trim()).filter(Boolean);
+      }
+      return query(
+        req.params.pipeline,
+        String(stage),
+        {
+          claimed: claimed !== undefined ? String(claimed).toLowerCase() === 'true' : undefined,
+          ids: ids ? String(ids).split(',') : undefined,
+          created_after: createdAfter ? String(createdAfter) : undefined,
+          created_before: createdBefore ? String(createdBefore) : undefined,
+          modified_after: modifiedAfter ? String(modifiedAfter) : undefined,
+          limit: limit ? Number(limit) : undefined,
+          offset: offset ? Number(offset) : undefined,
+          filter,
+          select,
+        },
+        strategy ? String(strategy) as ClaimStrategy : undefined,
+      );
+    });
   });
 
-  app.post('/pipelines/:pipeline/load', (req, res) => {
+  app.post('/pipelines/:pipeline/upload-file', (req, res) => {
     const { path: filePath, stage, deleteAfter, duplicates } = req.body ?? {};
     wrap(res, () => {
-      const results = loadFile(filePath, req.params.pipeline, { stage, deleteAfter, duplicates });
+      const results = uploadFile(filePath, req.params.pipeline, { stage, deleteAfter, duplicates });
       const counts = { new: 0, duplicate: 0, error: 0 };
       for (const r of results) counts[r.status]++;
       return { summary: counts, items: results.map(r => ({ [r.id]: r.message ? `${r.status}: ${r.message}` : r.status })) };
     });
   });
 
-  app.post('/pipelines/:pipeline/batch-write', (req, res) => {
+  app.post('/pipelines/:pipeline/upload-data', (req, res) => {
     const { data, format, stage, duplicates } = req.body ?? {};
     wrap(res, () => {
-      const results = batchWrite(data, format, req.params.pipeline, { stage, duplicates });
+      const results = uploadData(data, format, req.params.pipeline, { stage, duplicates });
       const counts = { new: 0, duplicate: 0, error: 0 };
       for (const r of results) counts[r.status]++;
       return { summary: counts, items: results.map(r => ({ [r.id]: r.message ? `${r.status}: ${r.message}` : r.status })) };