@quantod/qq 1.1.18 → 1.2.0

package/src/http.ts

@@ -4,7 +4,8 @@ import { createServer } from 'node:net';
 import { randomBytes } from 'node:crypto';
 import {
   makePipeline, deletePipeline, push, claim, release,
-  batchRead, status, unstick, loadFile, batchWrite, QQError,
+  query, status, unstick, uploadFile, uploadData, QQError,
+  type PayloadSelect, type ClaimStrategy,
 } from './commands.js';
 
 export interface HttpServer {
@@ -65,8 +66,8 @@ export async function startHttp(fixedPort?: number): Promise<HttpServer> {
   });
 
   app.post('/pipelines/:pipeline/push', (req, res) => {
-    const { stage, id, payload, priority, payloadFormat } = req.body ?? {};
-    wrap(res, () => ({ id: push(req.params.pipeline, stage, id, { payload, priority, payloadFormat }) }));
+    const { stage, id, payload, priority } = req.body ?? {};
+    wrap(res, () => ({ id: push(req.params.pipeline, stage, id, { payload, priority }) }));
   });
 
   app.post('/pipelines/:pipeline/claim', (req, res) => {
@@ -76,8 +77,8 @@ export async function startHttp(fixedPort?: number): Promise<HttpServer> {
 
   app.post('/pipelines/:pipeline/release/:seq', (req, res) => {
     const seq = parseInt(req.params.seq, 10);
-    const { target, payload, replace, priority, payloadFormat } = req.body ?? {};
-    wrap(res, () => { release(req.params.pipeline, seq, { target, payload, replace, priority, payloadFormat }); });
+    const { target, payload, replace, priority } = req.body ?? {};
+    wrap(res, () => { release(req.params.pipeline, seq, { target, payload, replace, priority }); });
   });
 
   app.get('/pipelines/:pipeline/status', (req, res) => {
@@ -85,38 +86,51 @@ export async function startHttp(fixedPort?: number): Promise<HttpServer> {
   });
 
   app.get('/pipelines/:pipeline/items', (req, res) => {
-    const { stage = '*', includePayload, claimed, ids, createdAfter, createdBefore, modifiedAfter, limit, offset, strategy } = req.query;
-    wrap(res, () => batchRead(
-      req.params.pipeline,
-      String(stage),
-      {
-        claimed: claimed !== undefined ? String(claimed).toLowerCase() === 'true' : undefined,
-        ids: ids ? String(ids).split(',') : undefined,
-        created_after: createdAfter ? String(createdAfter) : undefined,
-        created_before: createdBefore ? String(createdBefore) : undefined,
-        modified_after: modifiedAfter ? String(modifiedAfter) : undefined,
-        limit: limit ? Number(limit) : undefined,
-        offset: offset ? Number(offset) : undefined,
-      },
-      String(includePayload).toLowerCase() === 'true',
-      strategy ? String(strategy) as import('./commands.js').ClaimStrategy : undefined,
-    ));
+    const { stage = '*', filter: filterStr, select: selectStr, claimed, ids, createdAfter, createdBefore, modifiedAfter, limit, offset, strategy } = req.query;
+    wrap(res, () => {
+      let filter: Record<string, unknown> | undefined;
+      if (filterStr) {
+        try { filter = JSON.parse(String(filterStr)); }
+        catch { throw new QQError('filter must be valid JSON'); }
+      }
+      let select: PayloadSelect | undefined;
+      if (selectStr) {
+        const s = String(selectStr);
+        select = s === '*' ? '*' : s.split(',').map(f => f.trim()).filter(Boolean);
+      }
+      return query(
+        req.params.pipeline,
+        String(stage),
+        {
+          claimed: claimed !== undefined ? String(claimed).toLowerCase() === 'true' : undefined,
+          ids: ids ? String(ids).split(',') : undefined,
+          created_after: createdAfter ? String(createdAfter) : undefined,
+          created_before: createdBefore ? String(createdBefore) : undefined,
+          modified_after: modifiedAfter ? String(modifiedAfter) : undefined,
+          limit: limit ? Number(limit) : undefined,
+          offset: offset ? Number(offset) : undefined,
+          filter,
+          select,
+        },
+        strategy ? String(strategy) as ClaimStrategy : undefined,
+      );
+    });
   });
 
-  app.post('/pipelines/:pipeline/load', (req, res) => {
+  app.post('/pipelines/:pipeline/upload-file', (req, res) => {
     const { path: filePath, stage, deleteAfter, duplicates } = req.body ?? {};
     wrap(res, () => {
-      const results = loadFile(filePath, req.params.pipeline, { stage, deleteAfter, duplicates });
+      const results = uploadFile(filePath, req.params.pipeline, { stage, deleteAfter, duplicates });
       const counts = { new: 0, duplicate: 0, error: 0 };
       for (const r of results) counts[r.status]++;
       return { summary: counts, items: results.map(r => ({ [r.id]: r.message ? `${r.status}: ${r.message}` : r.status })) };
     });
   });
 
-  app.post('/pipelines/:pipeline/batch-write', (req, res) => {
+  app.post('/pipelines/:pipeline/upload-data', (req, res) => {
     const { data, format, stage, duplicates } = req.body ?? {};
     wrap(res, () => {
-      const results = batchWrite(data, format, req.params.pipeline, { stage, duplicates });
+      const results = uploadData(data, format, req.params.pipeline, { stage, duplicates });
       const counts = { new: 0, duplicate: 0, error: 0 };
       for (const r of results) counts[r.status]++;
       return { summary: counts, items: results.map(r => ({ [r.id]: r.message ? `${r.status}: ${r.message}` : r.status })) };

package/src/index.ts

@@ -4,8 +4,8 @@ import { join } from 'node:path';
 import { dump } from 'js-yaml';
 import {
   makePipeline, listPipelines, deletePipeline, push, claim, release,
-  batchRead, status, unstick, loadFile, batchWrite, backup, QQError,
-  type FilterOptions, type PushOptions, type ClaimOptions, type ClaimStrategy, type PayloadFormat, type DuplicateMode, type BatchWriteFormat,
+  query, status, unstick, uploadFile, uploadData, backup, QQError,
+  type FilterOptions, type PushOptions, type ClaimOptions, type ClaimStrategy, type PayloadFormat, type DuplicateMode, type BatchWriteFormat, type PayloadSelect,
 } from './commands.js';
 
 function out(data: unknown): void {
@@ -112,7 +112,7 @@ program
 program
   .command('push <path> [id]')
   .option('--priority <n>', 'item priority (higher = claimed first)', parseFloatArg, 0.0)
-  .option('--payload-format <yaml|json|text>', 'payload validation: yaml (default), json (minified), text (none)')
+  .option('--payload-format <yaml|json>', 'payload parsing format: yaml (default) or json')
   .action((path, id, opts) => {
     try {
       const { pipeline, stage = '' } = parsePath(path);
@@ -143,7 +143,7 @@ program
   .option('--target <stage>', 'move item to this stage (default: stay in current stage)')
   .option('--replace')
   .option('--priority <n>', 'update item priority', parseFloatArg)
-  .option('--payload-format <yaml|json|text>', 'payload validation: yaml (default), json (minified), text (none)')
+  .option('--payload-format <yaml|json>', 'payload parsing format: yaml (default) or json')
   .action((pipeline, seqStr, opts) => {
     try {
       const seq = parseInt(seqStr, 10);
@@ -154,15 +154,24 @@ program
     } catch (e) { e instanceof QQError ? fail(e.message) : fail(String(e)); }
   });
 
-addFilterOptions(program.command('batch-read <path>'))
-  .option('--include-payload', 'include payload in output')
+addFilterOptions(program.command('query <path>'))
+  .option('--filter <json>', 'MongoDB-style payload filter as JSON')
+  .option('--select <fields>', 'payload fields: * for all, or comma-separated paths (e.g. status,meta.author)')
   .option('--strategy <priority|fifo|lifo|random>', 'sort order (default: fifo)', 'fifo')
   .action((path, opts) => {
     try {
       const { pipeline, stage = '*' } = parsePath(path);
       const filters = collectFilters(opts);
       if (filters.claimed === undefined) filters.claimed = false;
-      out(batchRead(pipeline, stage, filters, opts.includePayload ?? false, opts.strategy as ClaimStrategy));
+      if (opts.filter) {
+        try { filters.filter = JSON.parse(opts.filter as string); }
+        catch { fail('--filter must be valid JSON'); }
+      }
+      if (opts.select) {
+        const s = opts.select as string;
+        filters.select = s === '*' ? '*' : s.split(',').map((f: string) => f.trim()).filter(Boolean) as PayloadSelect;
+      }
+      out(query(pipeline, stage, filters, opts.strategy as ClaimStrategy));
     }
     catch (e) { fail(e instanceof Error ? e.message : String(e)); }
   });
@@ -186,14 +195,14 @@ addReapFilterOptions(program.command('unstick <path>'))
   });
 
 program
-  .command('load-file <path> <pipeline>')
+  .command('upload-file <path> <pipeline>')
   .description('Load multiple items from a file into a pipeline (JSONL, JSON array, YAML array, CSV)')
   .option('--stage <stage>', 'default stage for records that do not specify one')
   .option('--delete-after', 'delete the file after loading')
   .option('--duplicates <ignore|append|replace>', 'how to handle duplicate ids: ignore (default), append payload, or replace payload', 'ignore')
   .action((filePath, pipeline, opts) => {
     try {
-      const results = loadFile(filePath, pipeline, { stage: opts.stage, deleteAfter: opts.deleteAfter ?? false, duplicates: opts.duplicates as DuplicateMode });
+      const results = uploadFile(filePath, pipeline, { stage: opts.stage, deleteAfter: opts.deleteAfter ?? false, duplicates: opts.duplicates as DuplicateMode });
       const counts = { new: 0, duplicate: 0, error: 0 };
       for (const r of results) counts[r.status]++;
       const items = results.map(r => ({ [r.id]: r.message ? `${r.status}: ${r.message}` : r.status }));
@@ -202,7 +211,7 @@ program
   });
 
 program
-  .command('batch-write <pipeline>')
+  .command('upload-data <pipeline>')
   .description('Write multiple items from stdin into a pipeline (JSONL, JSON array, YAML array, CSV)')
   .requiredOption('--format <jsonl|json|yaml|csv>', 'data format')
   .option('--stage <stage>', 'default stage for records that do not specify one')
@@ -211,7 +220,7 @@ program
     try {
       const data = readStdin();
       if (!data) fail('no data on stdin');
-      const results = batchWrite(data, opts.format as BatchWriteFormat, pipeline, { stage: opts.stage, duplicates: opts.duplicates as DuplicateMode });
+      const results = uploadData(data, opts.format as BatchWriteFormat, pipeline, { stage: opts.stage, duplicates: opts.duplicates as DuplicateMode });
       const counts = { new: 0, duplicate: 0, error: 0 };
       for (const r of results) counts[r.status]++;
       const items = results.map(r => ({ [r.id]: r.message ? `${r.status}: ${r.message}` : r.status }));

package/src/mcp.ts

@@ -7,7 +7,7 @@ import { z } from 'zod';
 import { dump } from 'js-yaml';
 import {
   makePipeline, deletePipeline, push, claim, release,
-  batchRead, status, unstick, loadFile, batchWrite, readFile,
+  query, status, unstick, uploadFile, uploadData, readFile,
 } from './commands.js';
 import { startHttp } from './http.js';
 import { renderJavaScriptSdk } from './sdk-templates/javascript.js';
@@ -81,12 +81,11 @@ export async function startMcp(fixedPort?: number): Promise<void> {
       pipeline: z.string(),
       stage: z.string().optional(),
       id: z.string().optional(),
-      payload: z.string().optional(),
+      payload: z.record(z.unknown()).optional(),
       priority: z.number().optional(),
-      payloadFormat: z.enum(['yaml', 'json', 'text']).optional(),
     },
-  }, ({ pipeline, stage, id, payload, priority, payloadFormat }) => {
-    try { return ok(`id: ${push(pipeline, stage, id, { payload, priority, payloadFormat })}`); }
+  }, ({ pipeline, stage, id, payload, priority }) => {
+    try { return ok(`id: ${push(pipeline, stage, id, { payload, priority })}`); }
     catch (e) { return err(e); }
   });
 
@@ -109,13 +108,12 @@ export async function startMcp(fixedPort?: number): Promise<void> {
       pipeline: z.string(),
       seq: z.number(),
       target: z.string().optional(),
-      payload: z.string().optional(),
+      payload: z.record(z.unknown()).optional(),
       replace: z.boolean().optional(),
       priority: z.number().optional(),
-      payloadFormat: z.enum(['yaml', 'json', 'text']).optional(),
     },
-  }, ({ pipeline, seq, target, payload, replace, priority, payloadFormat }) => {
-    try { release(pipeline, seq, { target, payload, replace, priority, payloadFormat }); return ok('ok: true'); }
+  }, ({ pipeline, seq, target, payload, replace, priority }) => {
+    try { release(pipeline, seq, { target, payload, replace, priority }); return ok('ok: true'); }
     catch (e) { return err(e); }
   });
 
@@ -127,12 +125,13 @@ export async function startMcp(fixedPort?: number): Promise<void> {
     catch (e) { return err(e); }
   });
 
-  server.registerTool('batch_read', {
-    description: 'Read items from a pipeline stage without claiming. Read guide before using.',
+  server.registerTool('query', {
+    description: 'Query items from a pipeline stage without claiming. Read guide before using.',
     inputSchema: {
       pipeline: z.string(),
       stage: z.string().optional(),
-      includePayload: z.boolean().optional(),
+      filter: z.record(z.unknown()).optional(),
+      select: z.union([z.literal('*'), z.array(z.string())]).optional(),
       claimed: z.boolean().optional(),
       ids: z.array(z.string()).optional(),
       createdAfter: z.union([z.number(), z.string()]).optional(),
@@ -142,12 +141,12 @@ export async function startMcp(fixedPort?: number): Promise<void> {
       offset: z.number().optional(),
       strategy: z.enum(['priority', 'fifo', 'lifo', 'random']).optional(),
     },
-  }, ({ pipeline, stage, includePayload, claimed, ids, createdAfter, createdBefore, modifiedAfter, limit, offset, strategy }) => {
+  }, ({ pipeline, stage, filter, select, claimed, ids, createdAfter, createdBefore, modifiedAfter, limit, offset, strategy }) => {
     try {
-      const items = batchRead(pipeline, stage ?? '*', {
+      const items = query(pipeline, stage ?? '*', {
         claimed, ids, created_after: createdAfter, created_before: createdBefore,
-        modified_after: modifiedAfter, limit, offset,
-      }, includePayload ?? false, strategy);
+        modified_after: modifiedAfter, limit, offset, filter, select,
+      }, strategy);
       return ok(dump(items, { lineWidth: -1 }));
     } catch (e) { return err(e); }
   });
@@ -191,8 +190,8 @@ export async function startMcp(fixedPort?: number): Promise<void> {
     return ok(snippet);
   });
 
-  server.registerTool('load_file', {
-    description: 'Load multiple items from a file into a pipeline. Supports JSONL, JSON array, YAML array, CSV. Read guide before using.',
+  server.registerTool('upload_file', {
+    description: 'Upload items from a file into a pipeline. Supports JSONL, JSON array, YAML array, CSV. Read guide before using.',
     inputSchema: {
       path: z.string(),
       pipeline: z.string(),
@@ -202,7 +201,7 @@ export async function startMcp(fixedPort?: number): Promise<void> {
     },
   }, ({ path: filePath, pipeline, stage, deleteAfter, duplicates }) => {
     try {
-      const results = loadFile(filePath, pipeline, { stage, deleteAfter, duplicates });
+      const results = uploadFile(filePath, pipeline, { stage, deleteAfter, duplicates });
       const counts = { new: 0, duplicate: 0, error: 0 };
       for (const r of results) counts[r.status]++;
       const items = results.map(r => ({ [r.id]: r.message ? `${r.status}: ${r.message}` : r.status }));
@@ -210,8 +209,8 @@ export async function startMcp(fixedPort?: number): Promise<void> {
     } catch (e) { return err(e); }
   });
 
-  server.registerTool('batch_write', {
-    description: 'Write multiple items from inline data into a pipeline. Supports JSONL, JSON array, YAML array, CSV. Read guide before using.',
+  server.registerTool('upload_data', {
+    description: 'Upload multiple items from inline data into a pipeline. Supports JSONL, JSON array, YAML array, CSV. Read guide before using.',
     inputSchema: {
       data: z.string(),
       format: z.enum(['jsonl', 'json', 'yaml', 'csv']),
@@ -221,7 +220,7 @@ export async function startMcp(fixedPort?: number): Promise<void> {
     },
   }, ({ data, format, pipeline, stage, duplicates }) => {
     try {
-      const results = batchWrite(data, format, pipeline, { stage, duplicates });
+      const results = uploadData(data, format, pipeline, { stage, duplicates });
       const counts = { new: 0, duplicate: 0, error: 0 };
       for (const r of results) counts[r.status]++;
       const items = results.map(r => ({ [r.id]: r.message ? `${r.status}: ${r.message}` : r.status }));

package/src/resources/chrome.md

@@ -6,7 +6,7 @@ Read the `guide` resource first — this document covers Chrome-specific pattern
 
 ## How to get data out of Chrome
 
-Chrome devtools and connectors usually struggle to get large data packages from web pages in Chrome. Solution: download + load_file. The browser's download API has no mixed content restrictions. Trigger a download from the HTTPS page, then call the `load_file` MCP tool to ingest the file. Don't route large data through the context.
+Chrome devtools and connectors usually struggle to get large data packages from web pages in Chrome. Solution: download + upload_file. The browser's download API has no mixed content restrictions. Trigger a download from the HTTPS page, then call the `upload_file` MCP tool to ingest the file. Don't route large data through the context.
 
 **In the injected script (HTTPS page):**
 ```javascript
@@ -19,9 +19,9 @@ a.download = 'qq-data.json';
 a.click();
 ```
 
-**Then call load_file:**
+**Then call upload_file:**
 ```
-load_file(
+upload_file(
   path: "$HOME/Downloads/qq-data.json",
   pipeline: "my_pypeline",
   stage: "inbox",