@quantod/qq 1.1.19 → 1.3.0

package/src/index.ts

@@ -4,8 +4,8 @@ import { join } from 'node:path';
 import { dump } from 'js-yaml';
 import {
   makePipeline, listPipelines, deletePipeline, push, claim, release,
-  batchRead, status, unstick, loadFile, batchWrite, backup, QQError,
-  type FilterOptions, type PushOptions, type ClaimOptions, type ClaimStrategy, type PayloadFormat, type DuplicateMode, type BatchWriteFormat,
+  query, status, unstick, uploadFile, uploadData, backup, QQError,
+  type FilterOptions, type PushOptions, type ClaimOptions, type ClaimStrategy, type PayloadFormat, type DuplicateMode, type BatchWriteFormat, type PayloadSelect,
 } from './commands.js';
 
 function out(data: unknown): void {
@@ -112,7 +112,7 @@ program
 program
   .command('push <path> [id]')
   .option('--priority <n>', 'item priority (higher = claimed first)', parseFloatArg, 0.0)
-  .option('--payload-format <yaml|json|text>', 'payload validation: yaml (default), json (minified), text (none)')
+  .option('--payload-format <yaml|json>', 'payload parsing format: yaml (default) or json')
   .action((path, id, opts) => {
     try {
       const { pipeline, stage = '' } = parsePath(path);
@@ -123,11 +123,11 @@ program
 
 program
   .command('claim <path> [id]')
-  .option('--strategy <priority|fifo|lifo|random>', 'claim strategy (default: priority)', 'priority')
+  .option('--sort-order <priority|fifo|lifo|random>', 'claim order (default: priority)', 'priority')
   .action((path, id, opts) => {
     try {
       const { pipeline, stage = '*' } = parsePath(path);
-      const claimOpts: ClaimOptions = { strategy: opts.strategy as ClaimStrategy };
+      const claimOpts: ClaimOptions = { strategy: opts.sortOrder as ClaimStrategy };
       const item = claim(pipeline, stage, id, claimOpts);
       if (item === null) {
         process.stdout.write('No items to claim\n');
@@ -143,7 +143,7 @@ program
   .option('--target <stage>', 'move item to this stage (default: stay in current stage)')
   .option('--replace')
   .option('--priority <n>', 'update item priority', parseFloatArg)
-  .option('--payload-format <yaml|json|text>', 'payload validation: yaml (default), json (minified), text (none)')
+  .option('--payload-format <yaml|json>', 'payload parsing format: yaml (default) or json')
   .action((pipeline, seqStr, opts) => {
     try {
       const seq = parseInt(seqStr, 10);
@@ -154,15 +154,24 @@ program
     } catch (e) { e instanceof QQError ? fail(e.message) : fail(String(e)); }
   });
 
-addFilterOptions(program.command('batch-read <path>'))
-  .option('--include-payload', 'include payload in output')
-  .option('--strategy <priority|fifo|lifo|random>', 'sort order (default: fifo)', 'fifo')
+addFilterOptions(program.command('query <path>'))
+  .option('--payload-filter <json>', 'MongoDB-style payload filter as JSON')
+  .option('--payload-fields <fields>', 'payload fields to return: * for all, or comma-separated paths (e.g. status,meta.author)')
+  .option('--sort-order <priority|fifo|lifo|random>', 'sort order (default: fifo)', 'fifo')
   .action((path, opts) => {
     try {
       const { pipeline, stage = '*' } = parsePath(path);
       const filters = collectFilters(opts);
       if (filters.claimed === undefined) filters.claimed = false;
-      out(batchRead(pipeline, stage, filters, opts.includePayload ?? false, opts.strategy as ClaimStrategy));
+      if (opts.payloadFilter) {
+        try { filters.filter = JSON.parse(opts.payloadFilter as string); }
+        catch { fail('--payload-filter must be valid JSON'); }
+      }
+      if (opts.payloadFields) {
+        const s = opts.payloadFields as string;
+        filters.select = s === '*' ? '*' : s.split(',').map((f: string) => f.trim()).filter(Boolean) as PayloadSelect;
+      }
+      out(query(pipeline, stage, filters, opts.sortOrder as ClaimStrategy));
     }
     catch (e) { fail(e instanceof Error ? e.message : String(e)); }
   });
@@ -186,14 +195,14 @@ addReapFilterOptions(program.command('unstick <path>'))
   });
 
 program
-  .command('load-file <path> <pipeline>')
+  .command('upload-file <path> <pipeline>')
   .description('Load multiple items from a file into a pipeline (JSONL, JSON array, YAML array, CSV)')
   .option('--stage <stage>', 'default stage for records that do not specify one')
   .option('--delete-after', 'delete the file after loading')
   .option('--duplicates <ignore|append|replace>', 'how to handle duplicate ids: ignore (default), append payload, or replace payload', 'ignore')
   .action((filePath, pipeline, opts) => {
     try {
-      const results = loadFile(filePath, pipeline, { stage: opts.stage, deleteAfter: opts.deleteAfter ?? false, duplicates: opts.duplicates as DuplicateMode });
+      const results = uploadFile(filePath, pipeline, { stage: opts.stage, deleteAfter: opts.deleteAfter ?? false, duplicates: opts.duplicates as DuplicateMode });
       const counts = { new: 0, duplicate: 0, error: 0 };
       for (const r of results) counts[r.status]++;
       const items = results.map(r => ({ [r.id]: r.message ? `${r.status}: ${r.message}` : r.status }));
@@ -202,7 +211,7 @@ program
   });
 
 program
-  .command('batch-write <pipeline>')
+  .command('upload-data <pipeline>')
   .description('Write multiple items from stdin into a pipeline (JSONL, JSON array, YAML array, CSV)')
   .requiredOption('--format <jsonl|json|yaml|csv>', 'data format')
   .option('--stage <stage>', 'default stage for records that do not specify one')
@@ -211,7 +220,7 @@ program
     try {
       const data = readStdin();
       if (!data) fail('no data on stdin');
-      const results = batchWrite(data, opts.format as BatchWriteFormat, pipeline, { stage: opts.stage, duplicates: opts.duplicates as DuplicateMode });
+      const results = uploadData(data, opts.format as BatchWriteFormat, pipeline, { stage: opts.stage, duplicates: opts.duplicates as DuplicateMode });
       const counts = { new: 0, duplicate: 0, error: 0 };
       for (const r of results) counts[r.status]++;
       const items = results.map(r => ({ [r.id]: r.message ? `${r.status}: ${r.message}` : r.status }));

package/src/mcp.ts

@@ -7,7 +7,7 @@ import { z } from 'zod';
 import { dump } from 'js-yaml';
 import {
   makePipeline, deletePipeline, push, claim, release,
-  batchRead, status, unstick, loadFile, batchWrite, readFile,
+  query, status, unstick, uploadFile, uploadData, readFile,
 } from './commands.js';
 import { startHttp } from './http.js';
 import { renderJavaScriptSdk } from './sdk-templates/javascript.js';
@@ -81,12 +81,11 @@ export async function startMcp(fixedPort?: number): Promise<void> {
       pipeline: z.string(),
       stage: z.string().optional(),
       id: z.string().optional(),
-      payload: z.string().optional(),
+      payload: z.record(z.unknown()).optional(),
       priority: z.number().optional(),
-      payloadFormat: z.enum(['yaml', 'json', 'text']).optional(),
     },
-  }, ({ pipeline, stage, id, payload, priority, payloadFormat }) => {
-    try { return ok(`id: ${push(pipeline, stage, id, { payload, priority, payloadFormat })}`); }
+  }, ({ pipeline, stage, id, payload, priority }) => {
+    try { return ok(`id: ${push(pipeline, stage, id, { payload, priority })}`); }
     catch (e) { return err(e); }
   });
 
@@ -96,10 +95,10 @@ export async function startMcp(fixedPort?: number): Promise<void> {
       pipeline: z.string(),
       stage: z.string().default('*'),
       id: z.string().optional(),
-      strategy: z.enum(['priority', 'fifo', 'lifo', 'random']).optional(),
+      sort_order: z.enum(['priority', 'fifo', 'lifo', 'random']).optional(),
     },
-  }, ({ pipeline, stage, id, strategy }) => {
-    try { return ok(dump(claim(pipeline, stage, id, { strategy }), { lineWidth: -1 })); }
+  }, ({ pipeline, stage, id, sort_order }) => {
+    try { return ok(dump(claim(pipeline, stage, id, { strategy: sort_order }), { lineWidth: -1 })); }
     catch (e) { return err(e); }
   });
 
@@ -109,13 +108,12 @@ export async function startMcp(fixedPort?: number): Promise<void> {
       pipeline: z.string(),
       seq: z.number(),
       target: z.string().optional(),
-      payload: z.string().optional(),
+      payload: z.record(z.unknown()).optional(),
       replace: z.boolean().optional(),
       priority: z.number().optional(),
-      payloadFormat: z.enum(['yaml', 'json', 'text']).optional(),
     },
-  }, ({ pipeline, seq, target, payload, replace, priority, payloadFormat }) => {
-    try { release(pipeline, seq, { target, payload, replace, priority, payloadFormat }); return ok('ok: true'); }
+  }, ({ pipeline, seq, target, payload, replace, priority }) => {
+    try { release(pipeline, seq, { target, payload, replace, priority }); return ok('ok: true'); }
     catch (e) { return err(e); }
   });
 
@@ -127,27 +125,28 @@ export async function startMcp(fixedPort?: number): Promise<void> {
     catch (e) { return err(e); }
   });
 
-  server.registerTool('batch_read', {
-    description: 'Read items from a pipeline stage without claiming. Read guide before using.',
+  server.registerTool('query', {
+    description: 'Query items from a pipeline stage without claiming. Read guide before using.',
     inputSchema: {
       pipeline: z.string(),
       stage: z.string().optional(),
-      includePayload: z.boolean().optional(),
+      payload_filter: z.record(z.unknown()).optional(),
+      payload_fields: z.union([z.literal('*'), z.array(z.string())]).optional(),
       claimed: z.boolean().optional(),
       ids: z.array(z.string()).optional(),
-      createdAfter: z.union([z.number(), z.string()]).optional(),
-      createdBefore: z.union([z.number(), z.string()]).optional(),
-      modifiedAfter: z.union([z.number(), z.string()]).optional(),
+      created_after: z.union([z.number(), z.string()]).optional(),
+      created_before: z.union([z.number(), z.string()]).optional(),
+      modified_after: z.union([z.number(), z.string()]).optional(),
       limit: z.number().optional(),
       offset: z.number().optional(),
-      strategy: z.enum(['priority', 'fifo', 'lifo', 'random']).optional(),
+      sort_order: z.enum(['priority', 'fifo', 'lifo', 'random']).optional(),
     },
-  }, ({ pipeline, stage, includePayload, claimed, ids, createdAfter, createdBefore, modifiedAfter, limit, offset, strategy }) => {
+  }, ({ pipeline, stage, payload_filter, payload_fields, claimed, ids, created_after, created_before, modified_after, limit, offset, sort_order }) => {
     try {
-      const items = batchRead(pipeline, stage ?? '*', {
-        claimed, ids, created_after: createdAfter, created_before: createdBefore,
-        modified_after: modifiedAfter, limit, offset,
-      }, includePayload ?? false, strategy);
+      const items = query(pipeline, stage ?? '*', {
+        claimed, ids, created_after, created_before,
+        modified_after, limit, offset, filter: payload_filter, select: payload_fields,
+      }, sort_order);
       return ok(dump(items, { lineWidth: -1 }));
     } catch (e) { return err(e); }
   });
@@ -158,14 +157,14 @@ export async function startMcp(fixedPort?: number): Promise<void> {
       pipeline: z.string(),
       stage: z.string().optional(),
       ids: z.array(z.string()).optional(),
-      createdAfter: z.union([z.number(), z.string()]).optional(),
-      createdBefore: z.union([z.number(), z.string()]).optional(),
-      modifiedAfter: z.union([z.number(), z.string()]).optional(),
+      created_after: z.union([z.number(), z.string()]).optional(),
+      created_before: z.union([z.number(), z.string()]).optional(),
+      modified_after: z.union([z.number(), z.string()]).optional(),
     },
-  }, ({ pipeline, stage, ids, createdAfter, createdBefore, modifiedAfter }) => {
+  }, ({ pipeline, stage, ids, created_after, created_before, modified_after }) => {
     try {
       const n = unstick(pipeline, stage ?? '*', {
-        ids, created_after: createdAfter, created_before: createdBefore, modified_after: modifiedAfter,
+        ids, created_after, created_before, modified_after,
       });
       return ok(`unstuck: ${n}`);
     } catch (e) { return err(e); }
@@ -191,18 +190,18 @@ export async function startMcp(fixedPort?: number): Promise<void> {
     return ok(snippet);
   });
 
-  server.registerTool('load_file', {
-    description: 'Load multiple items from a file into a pipeline. Supports JSONL, JSON array, YAML array, CSV. Read guide before using.',
+  server.registerTool('upload_file', {
+    description: 'Upload items from a file into a pipeline. Supports JSONL, JSON array, YAML array, CSV. Read guide before using.',
     inputSchema: {
       path: z.string(),
       pipeline: z.string(),
       stage: z.string().optional(),
-      deleteAfter: z.boolean().default(true),
+      delete_after: z.boolean().default(true),
       duplicates: z.enum(['ignore', 'append', 'replace']).optional(),
     },
-  }, ({ path: filePath, pipeline, stage, deleteAfter, duplicates }) => {
+  }, ({ path: filePath, pipeline, stage, delete_after, duplicates }) => {
     try {
-      const results = loadFile(filePath, pipeline, { stage, deleteAfter, duplicates });
+      const results = uploadFile(filePath, pipeline, { stage, deleteAfter: delete_after, duplicates });
       const counts = { new: 0, duplicate: 0, error: 0 };
       for (const r of results) counts[r.status]++;
       const items = results.map(r => ({ [r.id]: r.message ? `${r.status}: ${r.message}` : r.status }));
@@ -210,8 +209,8 @@ export async function startMcp(fixedPort?: number): Promise<void> {
     } catch (e) { return err(e); }
   });
 
-  server.registerTool('batch_write', {
-    description: 'Write multiple items from inline data into a pipeline. Supports JSONL, JSON array, YAML array, CSV. Read guide before using.',
+  server.registerTool('upload_data', {
+    description: 'Upload multiple items from inline data into a pipeline. Supports JSONL, JSON array, YAML array, CSV. Read guide before using.',
     inputSchema: {
       data: z.string(),
       format: z.enum(['jsonl', 'json', 'yaml', 'csv']),
@@ -221,7 +220,7 @@ export async function startMcp(fixedPort?: number): Promise<void> {
     },
   }, ({ data, format, pipeline, stage, duplicates }) => {
     try {
-      const results = batchWrite(data, format, pipeline, { stage, duplicates });
+      const results = uploadData(data, format, pipeline, { stage, duplicates });
       const counts = { new: 0, duplicate: 0, error: 0 };
       for (const r of results) counts[r.status]++;
       const items = results.map(r => ({ [r.id]: r.message ? `${r.status}: ${r.message}` : r.status }));
@@ -230,13 +229,13 @@ export async function startMcp(fixedPort?: number): Promise<void> {
   });
 
   server.registerTool('read_file', {
-    description: 'Read and return the content of a file on the host filesystem. Supports $VAR env var expansion in path. Use deleteAfter to consume the file after reading.',
+    description: 'Read and return the content of a file on the host filesystem. Supports $VAR env var expansion in path. Use delete_after to consume the file after reading.',
     inputSchema: {
       path: z.string(),
-      deleteAfter: z.boolean().default(true),
+      delete_after: z.boolean().default(true),
     },
-  }, ({ path: filePath, deleteAfter }) => {
-    try { return ok(readFile(filePath, deleteAfter)); }
+  }, ({ path: filePath, delete_after }) => {
+    try { return ok(readFile(filePath, delete_after)); }
     catch (e) { return err(e); }
   });
 

package/src/resources/chrome.md

@@ -6,7 +6,7 @@ Read the `guide` resource first — this document covers Chrome-specific pattern
 
 ## How to get data out of Chrome
 
-Chrome devtools and connectors usually struggle to get large data packages from web pages in Chrome. Solution: download + load_file. The browser's download API has no mixed content restrictions. Trigger a download from the HTTPS page, then call the `load_file` MCP tool to ingest the file. Don't route large data through the context.
+Chrome devtools and connectors usually struggle to get large data packages from web pages in Chrome. Solution: download + upload_file. The browser's download API has no mixed content restrictions. Trigger a download from the HTTPS page, then call the `upload_file` MCP tool to ingest the file. Don't route large data through the context.
 
 **In the injected script (HTTPS page):**
 ```javascript
@@ -19,9 +19,9 @@ a.download = 'qq-data.json';
 a.click();
 ```
 
-**Then call load_file:**
+**Then call upload_file:**
 ```
-load_file(
+upload_file(
   path: "$HOME/Downloads/qq-data.json",
   pipeline: "my_pypeline",
   stage: "inbox",