memory-journal-mcp 7.7.0 → 8.0.0

package/skills/cloudflare/references/queues/api.md

@@ -0,0 +1,211 @@
+# Queues API Reference
+
+## Producer: Send Messages
+
+```typescript
+// Basic send
+await env.MY_QUEUE.send({ url: request.url, timestamp: Date.now() })
+
+// Options: delay (max 43200s), contentType (json|text|bytes|v8)
+await env.MY_QUEUE.send(message, { delaySeconds: 600 })
+await env.MY_QUEUE.send(message, { delaySeconds: 0 }) // Override queue default
+
+// Batch (up to 100 msgs or 256 KB)
+await env.MY_QUEUE.sendBatch([
+  { body: 'msg1' },
+  { body: 'msg2' },
+  { body: 'msg3', options: { delaySeconds: 300 } },
+])
+
+// Non-blocking with ctx.waitUntil - send continues after response
+ctx.waitUntil(env.MY_QUEUE.send({ data: 'async' }))
+
+// Background tasks in queue consumer
+export default {
+  async queue(batch: MessageBatch, env: Env, ctx: ExecutionContext): Promise<void> {
+    for (const msg of batch.messages) {
+      await processMessage(msg.body)
+
+      // Fire-and-forget analytics (doesn't block ack)
+      ctx.waitUntil(env.ANALYTICS_QUEUE.send({ messageId: msg.id, processedAt: Date.now() }))
+
+      msg.ack()
+    }
+  },
+}
+```
+
+## Consumer: Push-based (Worker)
+
+```typescript
+// Type-safe handler with ExportedHandler
+interface Env {
+  MY_QUEUE: Queue
+  DB: D1Database
+}
+
+export default {
+  async queue(batch: MessageBatch<MessageBody>, env: Env, ctx: ExecutionContext): Promise<void> {
+    // batch.queue, batch.messages.length
+    for (const msg of batch.messages) {
+      // msg.id, msg.body, msg.timestamp, msg.attempts
+      try {
+        await processMessage(msg.body)
+        msg.ack()
+      } catch (error) {
+        msg.retry({ delaySeconds: 600 })
+      }
+    }
+  },
+} satisfies ExportedHandler<Env>
+```
+
+**CRITICAL WARNINGS:**
+
+1. **Messages not explicitly ack'd or retry'd will auto-retry indefinitely** until `max_retries` is reached. Always call `msg.ack()` or `msg.retry()` for each message.
+
+2. **Throwing uncaught errors retries the ENTIRE batch**, not just the failed message. Always wrap individual message processing in try/catch and call `msg.retry()` explicitly per message.
+
+```typescript
+// ❌ BAD: Uncaught error retries entire batch
+async queue(batch: MessageBatch): Promise<void> {
+  for (const msg of batch.messages) {
+    await riskyOperation(msg.body); // If this throws, entire batch retries
+    msg.ack();
+  }
+}
+
+// ✅ GOOD: Catch per message, handle individually
+async queue(batch: MessageBatch): Promise<void> {
+  for (const msg of batch.messages) {
+    try {
+      await riskyOperation(msg.body);
+      msg.ack();
+    } catch (error) {
+      msg.retry({ delaySeconds: 60 });
+    }
+  }
+}
+```
+
+## Ack/Retry Precedence Rules
+
+1. **Per-message calls take precedence**: If you call both `msg.ack()` and `msg.retry()`, last call wins
+2. **Batch calls don't override**: `batch.ackAll()` only affects messages without explicit ack/retry
+3. **No action = automatic retry**: Messages with no explicit action retry with configured delay
+
+```typescript
+async queue(batch: MessageBatch): Promise<void> {
+  for (const msg of batch.messages) {
+    msg.ack();        // Message marked for ack
+    msg.retry();      // Overrides ack - message will retry
+  }
+
+  batch.ackAll();     // Only affects messages not explicitly handled above
+}
+```
+
+## Batch Operations
+
+```typescript
+// Acknowledge entire batch
+try {
+  await bulkProcess(batch.messages)
+  batch.ackAll()
+} catch (error) {
+  batch.retryAll({ delaySeconds: 300 })
+}
+```
+
+## Exponential Backoff
+
+```typescript
+async queue(batch: MessageBatch, env: Env): Promise<void> {
+  for (const msg of batch.messages) {
+    try {
+      await processMessage(msg.body);
+      msg.ack();
+    } catch (error) {
+      // 30s, 60s, 120s, 240s, 480s, ... up to 12h max
+      const delay = Math.min(30 * (2 ** msg.attempts), 43200);
+      msg.retry({ delaySeconds: delay });
+    }
+  }
+}
+```
+
+## Multiple Queues, Single Consumer
+
+```typescript
+export default {
+  async queue(batch: MessageBatch, env: Env): Promise<void> {
+    switch (batch.queue) {
+      case 'high-priority':
+        await processUrgent(batch.messages)
+        break
+      case 'low-priority':
+        await processDeferred(batch.messages)
+        break
+      case 'email':
+        await sendEmails(batch.messages)
+        break
+      default:
+        batch.retryAll()
+    }
+  },
+}
+```
+
+## Consumer: Pull-based (HTTP)
+
+```typescript
+// Pull messages
+const response = await fetch(
+  `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/queues/${QUEUE_ID}/messages/pull`,
+  {
+    method: 'POST',
+    headers: { authorization: `Bearer ${API_TOKEN}`, 'content-type': 'application/json' },
+    body: JSON.stringify({ visibility_timeout_ms: 6000, batch_size: 50 }),
+  }
+)
+
+const data = await response.json()
+
+// Acknowledge
+await fetch(
+  `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/queues/${QUEUE_ID}/messages/ack`,
+  {
+    method: 'POST',
+    headers: { authorization: `Bearer ${API_TOKEN}`, 'content-type': 'application/json' },
+    body: JSON.stringify({
+      acks: [{ lease_id: msg.lease_id }],
+      retries: [{ lease_id: msg2.lease_id, delay_seconds: 600 }],
+    }),
+  }
+)
+```
+
+## Interfaces
+
+```typescript
+interface MessageBatch<Body = unknown> {
+  readonly queue: string
+  readonly messages: Message<Body>[]
+  ackAll(): void
+  retryAll(options?: QueueRetryOptions): void
+}
+
+interface Message<Body = unknown> {
+  readonly id: string
+  readonly timestamp: Date
+  readonly body: Body
+  readonly attempts: number
+  ack(): void
+  retry(options?: QueueRetryOptions): void
+}
+
+interface QueueSendOptions {
+  contentType?: 'text' | 'bytes' | 'json' | 'v8'
+  delaySeconds?: number // 0-43200
+}
+```

package/skills/cloudflare/references/queues/configuration.md

@@ -0,0 +1,151 @@
+# Queues Configuration
+
+## Create Queue
+
+```bash
+wrangler queues create my-queue
+wrangler queues create my-queue --retention-period-hours=336  # 14 days
+wrangler queues create my-queue --delivery-delay-secs=300
+```
+
+## Producer Binding
+
+**wrangler.jsonc:**
+
+```jsonc
+{
+  "queues": {
+    "producers": [
+      {
+        "queue": "my-queue-name",
+        "binding": "MY_QUEUE",
+        "delivery_delay": 60, // Optional: default delay in seconds
+      },
+    ],
+  },
+}
+```
+
+## Consumer Configuration (Push-based)
+
+**wrangler.jsonc:**
+
+```jsonc
+{
+  "queues": {
+    "consumers": [
+      {
+        "queue": "my-queue-name",
+        "max_batch_size": 10, // 1-100, default 10
+        "max_batch_timeout": 5, // 0-60s, default 5
+        "max_retries": 3, // default 3, max 100
+        "dead_letter_queue": "my-dlq", // optional
+        "retry_delay": 300, // optional: delay retries in seconds
+      },
+    ],
+  },
+}
+```
+
+## Consumer Configuration (Pull-based)
+
+**wrangler.jsonc:**
+
+```jsonc
+{
+  "queues": {
+    "consumers": [
+      {
+        "queue": "my-queue-name",
+        "type": "http_pull",
+        "visibility_timeout_ms": 5000, // default 30000, max 12h
+        "max_retries": 5,
+        "dead_letter_queue": "my-dlq",
+      },
+    ],
+  },
+}
+```
+
+## TypeScript Types
+
+```typescript
+interface Env {
+  MY_QUEUE: Queue<MessageBody>
+  ANALYTICS_QUEUE: Queue<AnalyticsEvent>
+}
+
+interface MessageBody {
+  id: string
+  action: 'create' | 'update' | 'delete'
+  data: Record<string, any>
+}
+
+export default {
+  async queue(batch: MessageBatch<MessageBody>, env: Env): Promise<void> {
+    for (const msg of batch.messages) {
+      console.log(msg.body.action)
+      msg.ack()
+    }
+  },
+} satisfies ExportedHandler<Env>
+```
+
+## Content Type Selection
+
+Choose content type based on consumer type and data requirements:
+
+| Content Type | Use When                                             | Readable By               | Supports                             | Size     |
+| ------------ | ---------------------------------------------------- | ------------------------- | ------------------------------------ | -------- |
+| `json`       | Pull consumers, dashboard visibility, simple objects | All (push/pull/dashboard) | JSON-serializable types only         | Medium   |
+| `v8`         | Push consumers only, complex JS objects              | Push consumers only       | Date, Map, Set, BigInt, typed arrays | Small    |
+| `text`       | String-only payloads                                 | All                       | Strings only                         | Smallest |
+| `bytes`      | Binary data (images, files)                          | All                       | ArrayBuffer, Uint8Array              | Variable |
+
+**Decision tree:**
+
+1. Need to view in dashboard or use pull consumer? → Use `json`
+2. Need Date, Map, Set, or other V8 types? → Use `v8` (push consumers only)
+3. Just strings? → Use `text`
+4. Binary data? → Use `bytes`
+
+```typescript
+// JSON: Good for simple objects, pull consumers, dashboard visibility
+await env.QUEUE.send({ id: 123, name: 'test' }, { contentType: 'json' })
+
+// V8: Good for Date, Map, Set (push consumers only)
+await env.QUEUE.send(
+  {
+    created: new Date(),
+    tags: new Set(['a', 'b']),
+  },
+  { contentType: 'v8' }
+)
+
+// Text: Simple strings
+await env.QUEUE.send('process-user-123', { contentType: 'text' })
+
+// Bytes: Binary data
+await env.QUEUE.send(imageBuffer, { contentType: 'bytes' })
+```
+
+**Default behavior:** If not specified, Cloudflare auto-selects `json` for JSON-serializable objects and `v8` for complex types.
+
+**IMPORTANT:** `v8` messages cannot be read by pull consumers or viewed in the dashboard. Use `json` if you need visibility or pull-based consumption.
+
+## CLI Commands
+
+```bash
+# Consumer management
+wrangler queues consumer add my-queue my-worker --batch-size=50 --max-retries=5
+wrangler queues consumer http add my-queue
+wrangler queues consumer worker remove my-queue my-worker
+wrangler queues consumer http remove my-queue
+
+# Queue operations
+wrangler queues list
+wrangler queues pause my-queue
+wrangler queues resume my-queue
+wrangler queues purge my-queue
+wrangler queues delete my-queue
+```

package/skills/cloudflare/references/queues/gotchas.md

@@ -0,0 +1,210 @@
+# Queues Gotchas & Troubleshooting
+
+## CRITICAL: Top Production Mistakes
+
+### 1. "Entire Batch Retried After Single Error"
+
+**Problem:** Throwing uncaught error in queue handler retries the entire batch, not just the failed message  
+**Cause:** Uncaught exceptions propagate to the runtime, triggering batch-level retry  
+**Solution:** Always wrap individual message processing in try/catch and call `msg.retry()` explicitly
+
+```typescript
+// ❌ BAD: Throws error, retries entire batch
+async queue(batch: MessageBatch): Promise<void> {
+  for (const msg of batch.messages) {
+    await riskyOperation(msg.body); // If this throws, entire batch retries
+    msg.ack();
+  }
+}
+
+// ✅ GOOD: Catch per message, handle individually
+async queue(batch: MessageBatch): Promise<void> {
+  for (const msg of batch.messages) {
+    try {
+      await riskyOperation(msg.body);
+      msg.ack();
+    } catch (error) {
+      msg.retry({ delaySeconds: 60 });
+    }
+  }
+}
+```
+
+### 2. "Messages Retry Forever"
+
+**Problem:** Messages not explicitly ack'd or retry'd will auto-retry indefinitely  
+**Cause:** Runtime default behavior retries unhandled messages until `max_retries` reached  
+**Solution:** Always call `msg.ack()` or `msg.retry()` for each message. Never leave messages unhandled.
+
+```typescript
+// ❌ BAD: Skipped messages auto-retry forever
+async queue(batch: MessageBatch): Promise<void> {
+  for (const msg of batch.messages) {
+    if (shouldProcess(msg.body)) {
+      await process(msg.body);
+      msg.ack();
+    }
+    // Missing: msg.ack() for skipped messages - they will retry!
+  }
+}
+
+// ✅ GOOD: Explicitly handle all messages
+async queue(batch: MessageBatch): Promise<void> {
+  for (const msg of batch.messages) {
+    if (shouldProcess(msg.body)) {
+      await process(msg.body);
+      msg.ack();
+    } else {
+      msg.ack(); // Explicitly ack even if not processing
+    }
+  }
+}
+```
+
+## Common Errors
+
+### "Duplicate Message Processing"
+
+**Problem:** Same message processed multiple times  
+**Cause:** At-least-once delivery guarantee means duplicates are possible during retries  
+**Solution:** Design consumers to be idempotent by tracking processed message IDs in KV with expiration TTL
+
+```typescript
+async queue(batch: MessageBatch, env: Env): Promise<void> {
+  for (const msg of batch.messages) {
+    const processed = await env.PROCESSED_KV.get(msg.id);
+    if (processed) {
+      msg.ack();
+      continue;
+    }
+
+    await processMessage(msg.body);
+    await env.PROCESSED_KV.put(msg.id, '1', { expirationTtl: 86400 });
+    msg.ack();
+  }
+}
+```
+
+### "Pull Consumer Can't Decode Messages"
+
+**Problem:** Pull consumer or dashboard shows unreadable message bodies  
+**Cause:** Messages sent with `v8` content type are only decodable by Workers push consumers  
+**Solution:** Use `json` content type for pull consumers or dashboard visibility
+
+```typescript
+// Use json for pull consumers
+await env.MY_QUEUE.send(data, { contentType: 'json' })
+
+// Use v8 only for push consumers with complex JS types
+await env.MY_QUEUE.send({ date: new Date(), tags: new Set() }, { contentType: 'v8' })
+```
+
+### "Messages Not Being Delivered"
+
+**Problem:** Messages sent but consumer not processing  
+**Cause:** Queue paused, consumer not configured, or consumer errors  
+**Solution:** Check queue status with `wrangler queues list`, verify consumer configured with `wrangler queues consumer add`, and check logs with `wrangler tail`
+
+### "High Dead Letter Queue Rate"
+
+**Problem:** Many messages ending up in DLQ  
+**Cause:** Consumer repeatedly failing to process messages after max retries  
+**Solution:** Review consumer error logs, check external dependency availability, verify message format matches expectations, or increase retry delay
+
+## Error Classification Patterns
+
+Classify errors to decide whether to retry or DLQ:
+
+```typescript
+async queue(batch: MessageBatch, env: Env): Promise<void> {
+  for (const msg of batch.messages) {
+    try {
+      await processMessage(msg.body);
+      msg.ack();
+    } catch (error) {
+      // Transient errors: retry with backoff
+      if (isRetryable(error)) {
+        const delay = Math.min(30 * (2 ** msg.attempts), 43200);
+        msg.retry({ delaySeconds: delay });
+      }
+      // Permanent errors: ack to avoid infinite retries
+      else {
+        console.error('Permanent error, sending to DLQ:', error);
+        await env.ERROR_LOG.put(msg.id, JSON.stringify({ msg: msg.body, error: String(error) }));
+        msg.ack(); // Prevent further retries
+      }
+    }
+  }
+}
+
+function isRetryable(error: unknown): boolean {
+  if (error instanceof Response) {
+    // Retry: rate limits, timeouts, server errors
+    return error.status === 429 || error.status >= 500;
+  }
+  if (error instanceof Error) {
+    // Don't retry: validation, auth, not found
+    return !error.message.includes('validation') &&
+           !error.message.includes('unauthorized') &&
+           !error.message.includes('not found');
+  }
+  return false; // Unknown errors don't retry
+}
+```
+
+### "CPU Time Exceeded in Consumer"
+
+**Problem:** Consumer fails with CPU time limit exceeded  
+**Cause:** Consumer processing exceeding 30s default CPU time limit  
+**Solution:** Increase CPU limit in wrangler.jsonc: `{ "limits": { "cpu_ms": 300000 } }` (5 minutes max)
+
+## Content Type Decision Guide
+
+**When to use each content type:**
+
+| Content Type     | Use When                                             | Readable By               | Supports                             |
+| ---------------- | ---------------------------------------------------- | ------------------------- | ------------------------------------ |
+| `json` (default) | Pull consumers, dashboard visibility, simple objects | All (push/pull/dashboard) | JSON-serializable types only         |
+| `v8`             | Push consumers only, complex JS objects              | Push consumers only       | Date, Map, Set, BigInt, typed arrays |
+| `text`           | String-only payloads                                 | All                       | Strings only                         |
+| `bytes`          | Binary data (images, files)                          | All                       | ArrayBuffer, Uint8Array              |
+
+**Decision tree:**
+
+1. Need to view in dashboard or use pull consumer? → Use `json`
+2. Need Date, Map, Set, or other V8 types? → Use `v8` (push consumers only)
+3. Just strings? → Use `text`
+4. Binary data? → Use `bytes`
+
+```typescript
+// Dashboard/pull: use json
+await env.QUEUE.send({ id: 123, name: 'test' }, { contentType: 'json' })
+
+// Complex JS types (push only): use v8
+await env.QUEUE.send(
+  {
+    created: new Date(),
+    tags: new Set(['a', 'b']),
+  },
+  { contentType: 'v8' }
+)
+```
+
+## Limits
+
+| Limit                  | Value                     | Notes                                |
+| ---------------------- | ------------------------- | ------------------------------------ |
+| Max queues             | 10,000                    | Per account                          |
+| Message size           | 128 KB                    | Maximum per message                  |
+| Batch size (consumer)  | 100 messages              | Maximum messages per batch           |
+| Batch size (sendBatch) | 100 msgs or 256 KB        | Whichever limit reached first        |
+| Throughput             | 5,000 msgs/sec            | Per queue                            |
+| Retention              | 4-14 days                 | Configurable retention period        |
+| Max backlog            | 25 GB                     | Maximum queue backlog size           |
+| Max delay              | 12 hours (43,200s)        | Maximum message delay                |
+| Max retries            | 100                       | Maximum retry attempts               |
+| CPU time default       | 30s                       | Per consumer invocation              |
+| CPU time max           | 300s (5 min)              | Configurable via `limits.cpu_ms`     |
+| Operations per message | 3 (write + read + delete) | Base cost per message                |
+| Pricing                | $0.40 per 1M operations   | After 1M free operations             |
+| Message charging       | Per 64 KB chunk           | Messages charged in 64 KB increments |