@quantod/qq 0.3.7 → 1.0.0

package/src/index.ts

@@ -1,13 +1,12 @@
 import { Command, InvalidArgumentError } from 'commander';
-import { spawnSync } from 'node:child_process';
 import { readFileSync } from 'node:fs';
 import { createInterface } from 'node:readline';
-import { resolve, join } from 'node:path';
+import { join } from 'node:path';
 import { dump } from 'js-yaml';
 import {
-  makePipeline, deletePipeline, push, claim, release,
-  batchRead, stats, unstick, QQError,
-  type FilterOptions, type PushOptions, type ClaimOptions,
+  makePipeline, listPipelines, deletePipeline, push, claim, release,
+  batchRead, status, unstick, loadFile, QQError,
+  type FilterOptions, type PushOptions, type ClaimOptions, type PayloadFormat,
 } from './commands.js';
 
 function out(data: unknown): void {
@@ -49,7 +48,7 @@ function parsePath(arg: string): { pipeline: string; stage?: string } {
 
 function collectFilters(opts: Record<string, unknown>): FilterOptions {
   const f: FilterOptions = {};
-  if (opts['claimed'] !== undefined) f.claimed = opts['claimed'] === '1';
+  if (opts['claimed'] !== undefined) f.claimed = opts['claimed'] === 'true';
   if (opts['ids']) f.ids = (opts['ids'] as string).split(',');
   if (opts['createdAfter']) f.created_after = opts['createdAfter'] as number;
   if (opts['createdBefore']) f.created_before = opts['createdBefore'] as number;
@@ -61,7 +60,7 @@ function collectFilters(opts: Record<string, unknown>): FilterOptions {
 
 function addFilterOptions(cmd: Command): Command {
   return cmd
-    .option('--claimed <0|1>')
+    .option('--claimed <true|false>')
     .option('--ids <id1,id2,...>')
     .option('--created-after <epoch>', '', parseIntArg)
     .option('--created-before <epoch>', '', parseIntArg)
@@ -85,11 +84,25 @@ program.name('qq').description('Persistent pipeline for Claude agent workflows')
 
 program
   .command('make-pipeline [name]')
-  .action((name) => {
-    try { out({ pipeline: makePipeline(name) }); }
+  .option('--description <text>', 'human-readable description of the pipeline')
+  .action((name, opts) => {
+    try { out({ pipeline: makePipeline(name, opts.description) }); }
     catch (e) { fail(e instanceof Error ? e.message : String(e)); }
   });
 
+program
+  .command('list-pipelines')
+  .action(() => {
+    try {
+      const names = listPipelines();
+      if (names.length === 0) {
+        process.stdout.write('(no pipelines)\n');
+      } else {
+        process.stdout.write(names.join('\n') + '\n');
+      }
+    } catch (e) { fail(e instanceof Error ? e.message : String(e)); }
+  });
+
 program
   .command('delete-pipeline <pipeline>')
   .action((pipeline) => {
@@ -110,12 +123,11 @@ program
 program
   .command('push <path> [id]')
   .option('--priority <n>', 'item priority (higher = claimed first)', parseFloatArg, 0.0)
-  .option('--plain-text', 'skip YAML validation of payload')
+  .option('--payload-format <yaml|json|text>', 'payload validation: yaml (default), json (minified), text (none)')
   .action((path, id, opts) => {
     try {
-      const { pipeline, stage } = parsePath(path);
-      if (!stage) fail('expected pipeline/stage, got: ' + path);
-      const o: PushOptions = { payload: readStdin() ?? undefined, priority: opts.priority, plainText: opts.plainText ?? false };
+      const { pipeline, stage = '' } = parsePath(path);
+      const o: PushOptions = { payload: readStdin() ?? undefined, priority: opts.priority, payloadFormat: opts.payloadFormat as PayloadFormat | undefined };
       out({ id: push(pipeline, stage, id, o) });
     } catch (e) { e instanceof QQError ? fail(e.message) : fail(String(e)); }
   });
@@ -142,36 +154,34 @@ program
   .option('--target <stage>', 'move item to this stage (default: stay in current stage)')
   .option('--replace')
   .option('--priority <n>', 'update item priority', parseFloatArg)
-  .option('--plain-text', 'skip YAML validation of payload')
+  .option('--payload-format <yaml|json|text>', 'payload validation: yaml (default), json (minified), text (none)')
   .action((pipeline, seqStr, opts) => {
     try {
       const seq = parseInt(seqStr, 10);
       if (isNaN(seq)) fail('seq must be an integer');
       const payload = readStdin() ?? undefined;
-      release(pipeline, seq, { target: opts.target, payload, replace: opts.replace, priority: opts.priority, plainText: opts.plainText ?? false });
+      release(pipeline, seq, { target: opts.target, payload, replace: opts.replace, priority: opts.priority, payloadFormat: opts.payloadFormat as PayloadFormat | undefined });
       out({ ok: true });
     } catch (e) { e instanceof QQError ? fail(e.message) : fail(String(e)); }
   });
 
 addFilterOptions(program.command('batch-read <path>'))
-  .option('--payload', 'include payload in output')
+  .option('--include-payload', 'include payload in output')
   .action((path, opts) => {
     try {
       const { pipeline, stage = '*' } = parsePath(path);
       const filters = collectFilters(opts);
       if (filters.claimed === undefined) filters.claimed = false;
-      out(batchRead(pipeline, stage, filters, opts.payload ?? false));
+      out(batchRead(pipeline, stage, filters, opts.includePayload ?? false));
     }
     catch (e) { fail(e instanceof Error ? e.message : String(e)); }
   });
 
 program
-  .command('stats [pipeline]')
+  .command('status <pipeline>')
   .action((pipeline) => {
     try {
-      const data = stats(pipeline);
-      process.stdout.write('# pipeline[/stage][/substage][...]: [total, claimed]\n');
-      process.stdout.write(dump(data, { lineWidth: -1, flowLevel: 1 }));
+      out(status(pipeline));
     }
     catch (e) { fail(e instanceof Error ? e.message : String(e)); }
   });
@@ -186,17 +196,26 @@ addReapFilterOptions(program.command('unstick <path>'))
   });
 
 program
-  .command('run <script>')
-  .action((script) => {
-    // dist/index.js → @quantod/qq → @quantod → node_modules
-    const nodeModules = join(__dirname, '../../..');
-    const existing = process.env.NODE_PATH;
-    const nodePath = existing ? `${nodeModules}:${existing}` : nodeModules;
-    const result = spawnSync(process.execPath, [resolve(script)], {
-      stdio: 'inherit',
-      env: { ...process.env, NODE_PATH: nodePath },
-    });
-    process.exit(result.status ?? 1);
+  .command('load-file <path> <pipeline>')
+  .description('Load multiple items from a file into a pipeline (JSONL, JSON array, YAML array, CSV)')
+  .option('--stage <stage>', 'default stage for records that do not specify one')
+  .option('--delete-after', 'delete the file after loading')
+  .action((filePath, pipeline, opts) => {
+    try {
+      const results = loadFile(filePath, pipeline, { stage: opts.stage, deleteAfter: opts.deleteAfter ?? false });
+      for (const r of results) {
+        process.stdout.write(r.message ? `${r.id}: ${r.status}: ${r.message}\n` : `${r.id}: ${r.status}\n`);
+      }
+    } catch (e) { fail(e instanceof Error ? e.message : String(e)); }
+  });
+
+program
+  .command('mcp')
+  .description('Start MCP server (stdio) with embedded HTTP bridge')
+  .option('--port <n>', 'HTTP bridge port (default: random)', (v) => parseInt(v, 10))
+  .action(async (opts) => {
+    const { startMcp } = await import('./mcp.js');
+    await startMcp(opts.port);
   });
 
 program.parse();

package/src/mcp.ts

@@ -0,0 +1,226 @@
+import { McpServer, ResourceTemplate } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { networkInterfaces } from 'node:os';
+import { z } from 'zod';
+import { dump } from 'js-yaml';
+import {
+  makePipeline, deletePipeline, push, claim, release,
+  batchRead, status, unstick, loadFile,
+} from './commands.js';
+import { startHttp } from './http.js';
+import { renderJavaScriptSdk } from './sdk-templates/javascript.js';
+import { renderPythonSdk } from './sdk-templates/python.js';
+
+const VERSION = JSON.parse(
+  readFileSync(join(__dirname, '../package.json'), 'utf8')
+).version as string;
+
+function loadResource(name: string): string {
+  return readFileSync(join(__dirname, 'resources', `${name}.md`), 'utf8');
+}
+
+const INSTRUCTIONS = `QQ is a persistent, crash-safe pipeline for multi-stage agentic workflows. \
+It implements a state machine with concurrency guarantees: items move through stages, \
+claim→release is an atomic transition only one agent holds at a time, and state survives restarts.
+
+Use QQ when work fans out across multiple items or stages: bulk processing, web crawling, \
+research pipelines, batch enrichment, any task where multiple items need to be processed \
+concurrently or sequentially through defined stages.
+
+Before calling any QQ tool, read the \`guide\` resource — it contains the mental model, \
+the complete tool reference, SDK usage, and examples. For Chrome browser scenarios, \
+also read the \`chrome\` resource. When designing a new pipeline, read the \`pipeline_design\` resource first.`;
+
+function getLanIp(): string {
+  const ifaces = networkInterfaces();
+  for (const name of Object.keys(ifaces)) {
+    if (name.startsWith('lo')) continue;
+    for (const iface of ifaces[name] ?? []) {
+      if (iface.family === 'IPv4' && !iface.internal) return iface.address;
+    }
+  }
+  return '127.0.0.1';
+}
+
+function ok(text: string) {
+  return { content: [{ type: 'text' as const, text }] };
+}
+
+function err(e: unknown) {
+  const msg = e instanceof Error ? e.message : String(e);
+  return { content: [{ type: 'text' as const, text: `error: ${msg}` }], isError: true };
+}
+
+export async function startMcp(fixedPort?: number): Promise<void> {
+  const http = await startHttp(fixedPort);
+  const lanIp = getLanIp();
+
+  const server = new McpServer(
+    { name: 'qq', version: VERSION },
+    { instructions: INSTRUCTIONS },
+  );
+
+  // ── Pipeline tools ─────────────────────────────────────────────────────────
+
+  server.registerTool('make_pipeline', {
+    description: 'Create a new pipeline. Read guide before using.',
+    inputSchema: { name: z.string().optional(), description: z.string().optional() },
+  }, ({ name, description }) => {
+    try { return ok(`pipeline: ${makePipeline(name, description)}`); }
+    catch (e) { return err(e); }
+  });
+
+  server.registerTool('push', {
+    description: 'Push an item into a pipeline stage. Read guide before using.',
+    inputSchema: {
+      pipeline: z.string(),
+      stage: z.string().optional(),
+      id: z.string().optional(),
+      payload: z.string().optional(),
+      priority: z.number().optional(),
+      payloadFormat: z.enum(['yaml', 'json', 'text']).optional(),
+    },
+  }, ({ pipeline, stage, id, payload, priority, payloadFormat }) => {
+    try { return ok(`id: ${push(pipeline, stage, id, { payload, priority, payloadFormat })}`); }
+    catch (e) { return err(e); }
+  });
+
+  server.registerTool('claim', {
+    description: 'Claim the next available item from a pipeline stage. Read guide before using.',
+    inputSchema: {
+      pipeline: z.string(),
+      stage: z.string().default('*'),
+      id: z.string().optional(),
+      random: z.boolean().optional(),
+    },
+  }, ({ pipeline, stage, id, random }) => {
+    try { return ok(dump(claim(pipeline, stage, id, { random }), { lineWidth: -1 })); }
+    catch (e) { return err(e); }
+  });
+
+  server.registerTool('release', {
+    description: 'Release a claimed item, optionally moving it to a new stage. Read guide before using.',
+    inputSchema: {
+      pipeline: z.string(),
+      seq: z.number(),
+      target: z.string().optional(),
+      payload: z.string().optional(),
+      replace: z.boolean().optional(),
+      priority: z.number().optional(),
+      payloadFormat: z.enum(['yaml', 'json', 'text']).optional(),
+    },
+  }, ({ pipeline, seq, target, payload, replace, priority, payloadFormat }) => {
+    try { release(pipeline, seq, { target, payload, replace, priority, payloadFormat }); return ok('ok: true'); }
+    catch (e) { return err(e); }
+  });
+
+  server.registerTool('status', {
+    description: 'Get pipeline description and item counts per stage. Read guide before using.',
+    inputSchema: { pipeline: z.string() },
+  }, ({ pipeline }) => {
+    try { return ok(dump(status(pipeline), { lineWidth: -1 })); }
+    catch (e) { return err(e); }
+  });
+
+  server.registerTool('batch_read', {
+    description: 'Read items from a pipeline stage without claiming. Read guide before using.',
+    inputSchema: {
+      pipeline: z.string(),
+      stage: z.string().optional(),
+      includePayload: z.boolean().optional(),
+      claimed: z.boolean().optional(),
+      ids: z.array(z.string()).optional(),
+      createdAfter: z.number().optional(),
+      createdBefore: z.number().optional(),
+      modifiedAfter: z.number().optional(),
+      limit: z.number().optional(),
+      offset: z.number().optional(),
+    },
+  }, ({ pipeline, stage, includePayload, claimed, ids, createdAfter, createdBefore, modifiedAfter, limit, offset }) => {
+    try {
+      const items = batchRead(pipeline, stage ?? '*', {
+        claimed, ids, created_after: createdAfter, created_before: createdBefore,
+        modified_after: modifiedAfter, limit, offset,
+      }, includePayload ?? false);
+      return ok(dump(items, { lineWidth: -1 }));
+    } catch (e) { return err(e); }
+  });
+
+  server.registerTool('unstick', {
+    description: 'Release all stuck claimed items back to their stage. Read guide before using.',
+    inputSchema: {
+      pipeline: z.string(),
+      stage: z.string().optional(),
+      ids: z.array(z.string()).optional(),
+      createdAfter: z.number().optional(),
+      createdBefore: z.number().optional(),
+      modifiedAfter: z.number().optional(),
+    },
+  }, ({ pipeline, stage, ids, createdAfter, createdBefore, modifiedAfter }) => {
+    try {
+      const n = unstick(pipeline, stage ?? '*', {
+        ids, created_after: createdAfter, created_before: createdBefore, modified_after: modifiedAfter,
+      });
+      return ok(`unstuck: ${n}`);
+    } catch (e) { return err(e); }
+  });
+
+  server.registerTool('delete_pipeline', {
+    description: 'Delete a pipeline and all its items. Irreversible. Read guide before using.',
+    inputSchema: { pipeline: z.string() },
+  }, ({ pipeline }) => {
+    try { deletePipeline(pipeline); return ok('ok: true'); }
+    catch (e) { return err(e); }
+  });
+
+  // ── Bridge tools ───────────────────────────────────────────────────────────
+
+  server.registerTool('get_sdk', {
+    description: "Get an SDK snippet for connecting scripts to QQ's HTTP bridge. Read guide before using.",
+    inputSchema: { language: z.enum(['javascript', 'python']) },
+  }, ({ language }) => {
+    const snippet = language === 'python'
+      ? renderPythonSdk(lanIp, http.port, http.token)
+      : renderJavaScriptSdk(lanIp, http.port, http.token);
+    return ok(snippet);
+  });
+
+  server.registerTool('load_file', {
+    description: 'Load multiple items from a file into a pipeline. Supports JSONL, JSON array, YAML array, CSV. Read guide before using.',
+    inputSchema: {
+      path: z.string(),
+      pipeline: z.string(),
+      stage: z.string().optional(),
+      deleteAfter: z.boolean().optional(),
+    },
+  }, ({ path: filePath, pipeline, stage, deleteAfter }) => {
+    try {
+      const results = loadFile(filePath, pipeline, { stage, deleteAfter });
+      const lines = results.map(r => r.message ? `${r.id}: ${r.status}: ${r.message}` : `${r.id}: ${r.status}`);
+      return ok(lines.join('\n'));
+    } catch (e) { return err(e); }
+  });
+
+  // ── Resources ──────────────────────────────────────────────────────────────
+
+  server.registerResource('guide', new ResourceTemplate('qq://guide', { list: undefined }), { mimeType: 'text/markdown' }, async () => ({
+    contents: [{ uri: 'qq://guide', mimeType: 'text/markdown', text: loadResource('guide') }],
+  }));
+
+  server.registerResource('chrome', new ResourceTemplate('qq://chrome', { list: undefined }), { mimeType: 'text/markdown' }, async () => ({
+    contents: [{ uri: 'qq://chrome', mimeType: 'text/markdown', text: loadResource('chrome') }],
+  }));
+
+  server.registerResource('pipeline_design', new ResourceTemplate('qq://pipeline_design', { list: undefined }), { mimeType: 'text/markdown' }, async () => ({
+    contents: [{ uri: 'qq://pipeline_design', mimeType: 'text/markdown', text: loadResource('pipeline_design') }],
+  }));
+
+  // ── Connect ────────────────────────────────────────────────────────────────
+
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+
+  process.stderr.write(`qq MCP server running. HTTP bridge on ${lanIp}:${http.port}\n`);
+}

package/src/resources/chrome.md

@@ -0,0 +1,33 @@
+# QQ Chrome Integration
+
+Read the `guide` resource first — this document covers Chrome-specific patterns only.
+
+---
+
+## How to get data out of Chrome
+
+Chrome devtools and connectors usually struggle to get large data packages from web pages in Chrome. Solution: download + load_file. The browser's download API has no mixed content restrictions. Trigger a download from the HTTPS page, then call the `load_file` MCP tool to ingest the file.
+
+**In the injected script (HTTPS page):**
+```javascript
+// Serialize data and trigger a download — no network request, no mixed content issue
+const data = JSON.stringify({ items: extractedItems });
+const blob = new Blob([data], { type: 'application/json' });
+const a = document.createElement('a');
+a.href = URL.createObjectURL(blob);
+a.download = 'qq-data.json';
+a.click();
+```
+
+**Then call load_file:**
+```
+load_file(
+  path: "$HOME/Downloads/qq-data.json",
+  pipeline: "my_pypeline",
+  stage: "inbox",
+  deleteAfter: true
+)
+```
+
+**Requirement**: Chrome must be set to auto-download to a known directory (no "ask where to save" prompt). Set the `download` attribute on the anchor to control the filename.
+