@rune-kit/rune 2.8.0 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (287) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +68 -34
  3. package/agents/adversary.md +27 -0
  4. package/agents/architect.md +19 -29
  5. package/agents/asset-creator.md +18 -4
  6. package/agents/audit.md +25 -4
  7. package/agents/autopsy.md +19 -4
  8. package/agents/ba.md +35 -0
  9. package/agents/brainstorm.md +31 -4
  10. package/agents/browser-pilot.md +21 -4
  11. package/agents/coder.md +21 -29
  12. package/agents/completion-gate.md +20 -4
  13. package/agents/constraint-check.md +18 -4
  14. package/agents/context-engine.md +22 -4
  15. package/agents/context-pack.md +32 -0
  16. package/agents/cook.md +41 -4
  17. package/agents/db.md +19 -4
  18. package/agents/debug.md +33 -4
  19. package/agents/dependency-doctor.md +20 -4
  20. package/agents/deploy.md +27 -4
  21. package/agents/design.md +22 -4
  22. package/agents/doc-processor.md +27 -0
  23. package/agents/docs-seeker.md +19 -4
  24. package/agents/docs.md +31 -0
  25. package/agents/fix.md +37 -4
  26. package/agents/git.md +29 -0
  27. package/agents/hallucination-guard.md +20 -4
  28. package/agents/incident.md +21 -4
  29. package/agents/integrity-check.md +18 -4
  30. package/agents/journal.md +19 -4
  31. package/agents/launch.md +32 -4
  32. package/agents/logic-guardian.md +26 -11
  33. package/agents/marketing.md +23 -4
  34. package/agents/mcp-builder.md +26 -0
  35. package/agents/neural-memory.md +30 -0
  36. package/agents/onboard.md +22 -4
  37. package/agents/perf.md +21 -4
  38. package/agents/plan.md +29 -4
  39. package/agents/preflight.md +22 -4
  40. package/agents/problem-solver.md +20 -4
  41. package/agents/rescue.md +23 -4
  42. package/agents/research.md +19 -4
  43. package/agents/researcher.md +19 -29
  44. package/agents/retro.md +32 -0
  45. package/agents/review-intake.md +20 -4
  46. package/agents/review.md +32 -4
  47. package/agents/reviewer.md +20 -28
  48. package/agents/safeguard.md +19 -4
  49. package/agents/sast.md +18 -4
  50. package/agents/scaffold.md +41 -0
  51. package/agents/scanner.md +19 -28
  52. package/agents/scope-guard.md +18 -4
  53. package/agents/scout.md +23 -4
  54. package/agents/sentinel-env.md +26 -0
  55. package/agents/sentinel.md +33 -4
  56. package/agents/sequential-thinking.md +20 -4
  57. package/agents/session-bridge.md +24 -4
  58. package/agents/skill-forge.md +22 -4
  59. package/agents/skill-router.md +26 -4
  60. package/agents/slides.md +24 -0
  61. package/agents/surgeon.md +19 -4
  62. package/agents/team.md +30 -4
  63. package/agents/test.md +36 -4
  64. package/agents/trend-scout.md +17 -4
  65. package/agents/verification.md +20 -4
  66. package/agents/video-creator.md +20 -4
  67. package/agents/watchdog.md +19 -4
  68. package/agents/worktree.md +17 -4
  69. package/commands/rune.md +168 -168
  70. package/compiler/__tests__/analytics.test.js +370 -0
  71. package/compiler/adapters/openclaw.js +2 -2
  72. package/compiler/analytics.js +385 -0
  73. package/compiler/bin/rune.js +68 -2
  74. package/compiler/dashboard.js +883 -0
  75. package/compiler/transforms/branding.js +1 -1
  76. package/contexts/dev.md +34 -34
  77. package/contexts/research.md +43 -43
  78. package/contexts/review.md +55 -55
  79. package/extensions/ai-ml/PACK.md +88 -88
  80. package/extensions/ai-ml/skills/ai-agents.md +172 -172
  81. package/extensions/ai-ml/skills/code-sandbox.md +187 -187
  82. package/extensions/ai-ml/skills/deep-research.md +146 -146
  83. package/extensions/ai-ml/skills/embedding-search.md +66 -66
  84. package/extensions/ai-ml/skills/fine-tuning-guide.md +74 -74
  85. package/extensions/ai-ml/skills/llm-architect.md +125 -125
  86. package/extensions/ai-ml/skills/llm-integration.md +64 -64
  87. package/extensions/ai-ml/skills/prompt-patterns.md +72 -72
  88. package/extensions/ai-ml/skills/rag-patterns.md +66 -66
  89. package/extensions/ai-ml/skills/web-extraction.md +114 -114
  90. package/extensions/analytics/PACK.md +92 -92
  91. package/extensions/analytics/skills/ab-testing.md +72 -72
  92. package/extensions/analytics/skills/dashboard-patterns.md +83 -83
  93. package/extensions/analytics/skills/data-validation.md +68 -68
  94. package/extensions/analytics/skills/funnel-analysis.md +81 -81
  95. package/extensions/analytics/skills/sql-patterns.md +57 -57
  96. package/extensions/analytics/skills/statistical-analysis.md +79 -79
  97. package/extensions/analytics/skills/tracking-setup.md +71 -71
  98. package/extensions/backend/PACK.md +104 -104
  99. package/extensions/backend/skills/api-patterns.md +84 -84
  100. package/extensions/backend/skills/async-pipeline.md +193 -193
  101. package/extensions/backend/skills/auth-patterns.md +97 -97
  102. package/extensions/backend/skills/background-jobs.md +133 -133
  103. package/extensions/backend/skills/caching-patterns.md +108 -108
  104. package/extensions/backend/skills/cli-generation.md +133 -133
  105. package/extensions/backend/skills/database-patterns.md +87 -87
  106. package/extensions/backend/skills/middleware-patterns.md +104 -104
  107. package/extensions/chrome-ext/PACK.md +93 -93
  108. package/extensions/chrome-ext/skills/cws-preflight.md +143 -143
  109. package/extensions/chrome-ext/skills/cws-publish.md +104 -104
  110. package/extensions/chrome-ext/skills/ext-ai-integration.md +251 -251
  111. package/extensions/chrome-ext/skills/ext-messaging.md +139 -139
  112. package/extensions/chrome-ext/skills/ext-storage.md +133 -133
  113. package/extensions/chrome-ext/skills/mv3-scaffold.md +164 -164
  114. package/extensions/content/PACK.md +96 -96
  115. package/extensions/content/skills/blog-patterns.md +88 -88
  116. package/extensions/content/skills/cms-integration.md +131 -131
  117. package/extensions/content/skills/content-scoring.md +107 -107
  118. package/extensions/content/skills/i18n.md +83 -83
  119. package/extensions/content/skills/mdx-authoring.md +137 -137
  120. package/extensions/content/skills/reference.md +1014 -1014
  121. package/extensions/content/skills/seo-patterns.md +67 -67
  122. package/extensions/content/skills/video-repurpose.md +153 -153
  123. package/extensions/devops/PACK.md +101 -101
  124. package/extensions/devops/skills/chaos-testing.md +67 -67
  125. package/extensions/devops/skills/ci-cd.md +75 -75
  126. package/extensions/devops/skills/docker.md +58 -58
  127. package/extensions/devops/skills/edge-serverless.md +163 -163
  128. package/extensions/devops/skills/infra-as-code.md +158 -158
  129. package/extensions/devops/skills/kubernetes.md +110 -110
  130. package/extensions/devops/skills/monitoring.md +57 -57
  131. package/extensions/devops/skills/server-setup.md +64 -64
  132. package/extensions/devops/skills/ssl-domain.md +42 -42
  133. package/extensions/ecommerce/PACK.md +116 -116
  134. package/extensions/ecommerce/skills/cart-system.md +79 -79
  135. package/extensions/ecommerce/skills/inventory-mgmt.md +102 -102
  136. package/extensions/ecommerce/skills/order-management.md +126 -126
  137. package/extensions/ecommerce/skills/payment-integration.md +472 -472
  138. package/extensions/ecommerce/skills/shopify-dev.md +69 -69
  139. package/extensions/ecommerce/skills/subscription-billing.md +93 -93
  140. package/extensions/ecommerce/skills/tax-compliance.md +117 -117
  141. package/extensions/gamedev/PACK.md +142 -142
  142. package/extensions/gamedev/skills/asset-pipeline.md +74 -74
  143. package/extensions/gamedev/skills/audio-system.md +129 -129
  144. package/extensions/gamedev/skills/camera-system.md +87 -87
  145. package/extensions/gamedev/skills/ecs.md +98 -98
  146. package/extensions/gamedev/skills/game-loops.md +72 -72
  147. package/extensions/gamedev/skills/input-system.md +199 -199
  148. package/extensions/gamedev/skills/multiplayer.md +180 -180
  149. package/extensions/gamedev/skills/particles.md +105 -105
  150. package/extensions/gamedev/skills/physics-engine.md +89 -89
  151. package/extensions/gamedev/skills/scene-management.md +146 -146
  152. package/extensions/gamedev/skills/threejs-patterns.md +90 -90
  153. package/extensions/gamedev/skills/webgl.md +71 -71
  154. package/extensions/mobile/PACK.md +106 -106
  155. package/extensions/mobile/skills/app-store-connect.md +152 -152
  156. package/extensions/mobile/skills/app-store-prep.md +66 -66
  157. package/extensions/mobile/skills/deep-linking.md +109 -109
  158. package/extensions/mobile/skills/flutter.md +60 -60
  159. package/extensions/mobile/skills/ios-build-pipeline.md +142 -142
  160. package/extensions/mobile/skills/native-bridge.md +66 -66
  161. package/extensions/mobile/skills/ota-updates.md +97 -97
  162. package/extensions/mobile/skills/push-notifications.md +111 -111
  163. package/extensions/mobile/skills/react-native.md +82 -82
  164. package/extensions/saas/PACK.md +116 -116
  165. package/extensions/saas/skills/billing-integration.md +200 -200
  166. package/extensions/saas/skills/feature-flags.md +130 -130
  167. package/extensions/saas/skills/multi-tenant.md +103 -103
  168. package/extensions/saas/skills/onboarding-flow.md +139 -139
  169. package/extensions/saas/skills/subscription-flow.md +95 -95
  170. package/extensions/saas/skills/team-management.md +144 -144
  171. package/extensions/security/PACK.md +99 -99
  172. package/extensions/security/skills/api-security.md +140 -140
  173. package/extensions/security/skills/compliance.md +68 -68
  174. package/extensions/security/skills/owasp-audit.md +64 -64
  175. package/extensions/security/skills/pentest-patterns.md +77 -77
  176. package/extensions/security/skills/secret-mgmt.md +65 -65
  177. package/extensions/security/skills/supply-chain.md +65 -65
  178. package/extensions/trading/PACK.md +80 -80
  179. package/extensions/trading/skills/chart-components.md +55 -55
  180. package/extensions/trading/skills/experiment-loop.md +125 -125
  181. package/extensions/trading/skills/fintech-patterns.md +47 -47
  182. package/extensions/trading/skills/indicator-library.md +58 -58
  183. package/extensions/trading/skills/quant-analysis.md +111 -111
  184. package/extensions/trading/skills/realtime-data.md +58 -58
  185. package/extensions/trading/skills/trade-logic.md +104 -104
  186. package/extensions/ui/PACK.md +130 -130
  187. package/extensions/ui/skills/a11y-audit.md +91 -91
  188. package/extensions/ui/skills/animation-patterns.md +127 -106
  189. package/extensions/ui/skills/component-patterns.md +100 -75
  190. package/extensions/ui/skills/design-decision.md +108 -108
  191. package/extensions/ui/skills/design-system.md +68 -68
  192. package/extensions/ui/skills/landing-patterns.md +155 -155
  193. package/extensions/ui/skills/palette-picker.md +173 -173
  194. package/extensions/ui/skills/react-health.md +90 -90
  195. package/extensions/ui/skills/type-system.md +125 -125
  196. package/extensions/ui/skills/web-vitals.md +153 -153
  197. package/extensions/zalo/PACK.md +145 -145
  198. package/extensions/zalo/skills/zalo-oa-mcp.md +317 -317
  199. package/extensions/zalo/skills/zalo-oa-messaging.md +429 -429
  200. package/extensions/zalo/skills/zalo-oa-setup.md +236 -236
  201. package/extensions/zalo/skills/zalo-oa-webhook.md +189 -189
  202. package/extensions/zalo/skills/zalo-personal-messaging.md +194 -194
  203. package/extensions/zalo/skills/zalo-personal-setup.md +153 -153
  204. package/extensions/zalo/skills/zalo-rate-guard.md +219 -219
  205. package/hooks/auto-format/index.cjs +48 -48
  206. package/hooks/context-watch/index.cjs +95 -68
  207. package/hooks/hooks.json +111 -111
  208. package/hooks/metrics-collector/index.cjs +86 -42
  209. package/hooks/post-session-reflect/index.cjs +189 -153
  210. package/hooks/pre-compact/index.cjs +95 -95
  211. package/hooks/run-hook.cmd +1 -1
  212. package/hooks/secrets-scan/index.cjs +100 -100
  213. package/hooks/session-start/index.cjs +71 -65
  214. package/hooks/typecheck/index.cjs +65 -65
  215. package/package.json +63 -63
  216. package/references/ui-pro-max-data/LICENSE-UI-PRO-MAX +21 -21
  217. package/references/ui-pro-max-data/charts.csv +26 -26
  218. package/references/ui-pro-max-data/colors.csv +161 -161
  219. package/references/ui-pro-max-data/styles.csv +68 -68
  220. package/references/ui-pro-max-data/typography.csv +74 -74
  221. package/references/ui-pro-max-data/ui-reasoning.csv +162 -162
  222. package/references/ui-pro-max-data/ux-guidelines.csv +99 -99
  223. package/skills/adversary/SKILL.md +283 -283
  224. package/skills/asset-creator/SKILL.md +157 -157
  225. package/skills/audit/SKILL.md +148 -2
  226. package/skills/autopsy/SKILL.md +335 -259
  227. package/skills/autopsy/references/repo-analysis-patterns.md +113 -0
  228. package/skills/ba/SKILL.md +72 -2
  229. package/skills/brainstorm/SKILL.md +342 -341
  230. package/skills/browser-pilot/SKILL.md +168 -168
  231. package/skills/constraint-check/SKILL.md +165 -165
  232. package/skills/context-engine/SKILL.md +404 -404
  233. package/skills/cook/SKILL.md +917 -834
  234. package/skills/cook/references/output-format.md +33 -0
  235. package/skills/db/SKILL.md +273 -272
  236. package/skills/debug/SKILL.md +465 -443
  237. package/skills/dependency-doctor/SKILL.md +265 -235
  238. package/skills/deploy/SKILL.md +274 -231
  239. package/skills/design/DESIGN-REFERENCE.md +365 -365
  240. package/skills/design/SKILL.md +589 -482
  241. package/skills/doc-processor/SKILL.md +254 -254
  242. package/skills/docs/SKILL.md +374 -373
  243. package/skills/docs-seeker/SKILL.md +177 -177
  244. package/skills/fix/SKILL.md +330 -308
  245. package/skills/git/SKILL.md +339 -339
  246. package/skills/graft/SKILL.md +352 -0
  247. package/skills/graft/references/challenge-framework.md +98 -0
  248. package/skills/graft/references/mode-decision.md +44 -0
  249. package/skills/hallucination-guard/SKILL.md +219 -219
  250. package/skills/incident/SKILL.md +254 -251
  251. package/skills/integrity-check/SKILL.md +169 -169
  252. package/skills/journal/SKILL.md +240 -238
  253. package/skills/launch/SKILL.md +344 -342
  254. package/skills/logic-guardian/SKILL.md +251 -251
  255. package/skills/marketing/SKILL.md +290 -245
  256. package/skills/mcp-builder/SKILL.md +425 -423
  257. package/skills/mcp-builder/references/auto-discovery-pattern.md +169 -0
  258. package/skills/neural-memory/SKILL.md +362 -362
  259. package/skills/onboard/SKILL.md +404 -403
  260. package/skills/perf/SKILL.md +346 -346
  261. package/skills/plan/SKILL.md +433 -370
  262. package/skills/plan/references/feature-map.md +84 -0
  263. package/skills/preflight/SKILL.md +415 -396
  264. package/skills/problem-solver/SKILL.md +380 -284
  265. package/skills/rescue/SKILL.md +474 -450
  266. package/skills/retro/SKILL.md +5 -1
  267. package/skills/review/SKILL.md +612 -535
  268. package/skills/review-intake/SKILL.md +249 -249
  269. package/skills/safeguard/SKILL.md +200 -200
  270. package/skills/sast/SKILL.md +190 -190
  271. package/skills/scaffold/SKILL.md +328 -286
  272. package/skills/scope-guard/SKILL.md +180 -162
  273. package/skills/scout/SKILL.md +263 -263
  274. package/skills/sentinel/SKILL.md +382 -353
  275. package/skills/sentinel-env/SKILL.md +254 -254
  276. package/skills/sequential-thinking/SKILL.md +234 -234
  277. package/skills/session-bridge/SKILL.md +543 -397
  278. package/skills/skill-forge/SKILL.md +581 -539
  279. package/skills/skill-router/{skill.md → SKILL.md} +30 -2
  280. package/skills/surgeon/SKILL.md +215 -215
  281. package/skills/team/SKILL.md +556 -514
  282. package/skills/test/SKILL.md +614 -587
  283. package/skills/trend-scout/SKILL.md +145 -145
  284. package/skills/verification/SKILL.md +326 -325
  285. package/skills/video-creator/SKILL.md +201 -201
  286. package/skills/watchdog/SKILL.md +168 -168
  287. package/skills/worktree/SKILL.md +140 -140
@@ -1,133 +1,133 @@
1
- ---
2
- name: "background-jobs"
3
- pack: "@rune/backend"
4
- description: "Queue-based async processing — BullMQ (Node.js), job patterns, retry strategies, idempotency, dead letter queues, monitoring."
5
- model: sonnet
6
- tools: [Read, Edit, Write, Grep, Glob, Bash]
7
- ---
8
-
9
- # background-jobs
10
-
11
- Queue-based async processing — BullMQ (Node.js), job patterns, retry strategies, idempotency, dead letter queues, monitoring.
12
-
13
- #### Workflow
14
-
15
- **Step 1 — Identify async operations**
16
- Scan route handlers and service functions for operations that: (a) take > 200ms (PDF generation, image resizing, report aggregation), (b) are non-user-facing (email sending, webhook delivery, analytics events), (c) can tolerate eventual consistency (data sync, cache warming, notification dispatch). Flag these as candidates for background jobs. Output a classification: fire-and-forget vs delayed vs scheduled (cron) vs fan-out.
17
-
18
- **Step 2 — Choose queue system**
19
- Node.js: BullMQ (Redis-backed, TypeScript-native, built-in retry/delay/priority/rate-limiting — recommended). Python: Celery + Redis/RabbitMQ broker (mature, distributed workers, beat scheduler for cron). For very simple use cases (single server, low volume): `node-cron` + in-process worker. Avoid in-process queues in production — they die with the process and lose jobs.
20
-
21
- **Step 3 — Implement job with retry strategy**
22
- Emit job producer (enqueue) and worker (processor) as separate files. Retry strategy: exponential backoff with jitter (`attempts: 5, backoff: { type: 'exponential', delay: 1000 }`). Idempotency: every job MUST have an idempotency key — use a deterministic ID from the operation (e.g., `email:welcome:${userId}` not a random UUID). This ensures duplicate enqueues (from retries, double-clicks) process exactly once. Dead letter queue: after max retries, move job to a `{queue-name}:failed` queue for inspection and manual replay — never silently drop.
23
-
24
- **Step 4 — Add monitoring and alerting**
25
- BullMQ Board or Bull Dashboard for visual queue monitoring. Emit metrics: queue depth (jobs waiting), processing rate (jobs/sec), failure rate (failed/total). Alert when: queue depth > threshold (workers not keeping up), failure rate > 5% (systematic error in processor), job age > expected TTL (stuck job). Use BullMQ events (`queue.on('failed', ...)`) to push metrics to Prometheus or Datadog.
26
-
27
- **Step 5 — Handle dead letters**
28
- Emit dead letter inspection endpoint: list failed jobs with error reason, retry count, and last error. Emit replay endpoint: re-enqueue a specific failed job with a fresh retry budget. Purge endpoint: clear dead letter queue after investigation. Add alerting on dead letter queue depth > 0 for critical job types (payment processing, compliance logging).
29
-
30
- #### Example
31
-
32
- ```typescript
33
- // BullMQ setup with TypeScript — producer + worker
34
- import { Queue, Worker, Job } from 'bullmq';
35
-
36
- const connection = { host: REDIS_HOST, port: 6379 };
37
-
38
- // Job type definitions
39
- interface EmailJob { to: string; template: string; data: Record<string, unknown> }
40
- interface PdfJob { reportId: string; userId: string; format: 'pdf' | 'xlsx' }
41
-
42
- // Producers
43
- export const emailQueue = new Queue<EmailJob>('email', { connection });
44
- export const pdfQueue = new Queue<PdfJob>('pdf', { connection });
45
-
46
- // Enqueue with idempotency key (jobId = idempotent identifier)
47
- export const sendWelcomeEmail = (userId: string, email: string) =>
48
- emailQueue.add('welcome', { to: email, template: 'welcome', data: { userId } }, {
49
- jobId: `email:welcome:${userId}`, // prevents duplicate welcome emails
50
- attempts: 3,
51
- backoff: { type: 'exponential', delay: 2_000 },
52
- removeOnComplete: { count: 1000 }, // keep last 1000 completed for audit
53
- removeOnFail: false, // keep all failed for dead letter review
54
- });
55
-
56
- // Scheduled/delayed job
57
- export const sendReminderEmail = (userId: string, delayMs: number) =>
58
- emailQueue.add('reminder', { to: userId, template: 'reminder', data: {} }, {
59
- delay: delayMs,
60
- attempts: 5,
61
- backoff: { type: 'exponential', delay: 5_000 },
62
- });
63
-
64
- // Worker processor with error handling
65
- const emailWorker = new Worker<EmailJob>('email', async (job: Job<EmailJob>) => {
66
- const { to, template, data } = job.data;
67
- // Validate job data — serialized payload may be stale
68
- if (!to || !template) throw new Error(`Invalid job payload: ${JSON.stringify(job.data)}`);
69
- await emailService.send({ to, template, data });
70
- // Return value is stored in job.returnvalue for audit
71
- return { sentAt: new Date().toISOString() };
72
- }, {
73
- connection,
74
- concurrency: 10, // process up to 10 emails in parallel
75
- limiter: { max: 100, duration: 60_000 }, // rate limit: 100/min
76
- });
77
-
78
- emailWorker.on('failed', async (job, err) => {
79
- logger.error({ jobId: job?.id, queue: 'email', error: err.message, attempts: job?.attemptsMade });
80
- if (job?.attemptsMade >= job?.opts.attempts!) {
81
- // max retries exhausted → alert
82
- await alerting.notify(`Dead letter: email job ${job.id} failed after ${job.attemptsMade} attempts`);
83
- }
84
- });
85
-
86
- // Fan-out pattern: one job enqueues many children
87
- const fanOutNotification = async (eventId: string, userIds: string[]) => {
88
- const jobs = userIds.map(userId => ({
89
- name: 'notify',
90
- data: { userId, eventId },
91
- opts: {
92
- jobId: `notify:${eventId}:${userId}`,
93
- attempts: 3,
94
- backoff: { type: 'exponential', delay: 1_000 },
95
- },
96
- }));
97
- await notificationQueue.addBulk(jobs);
98
- };
99
-
100
- // Dead letter inspection API
101
- app.get('/admin/jobs/failed', authenticate, authorize('admin'), async (req, res) => {
102
- const failed = await emailQueue.getFailed(0, 50);
103
- res.json({ count: failed.length, jobs: failed.map(j => ({ id: j.id, data: j.data, reason: j.failedReason, attempts: j.attemptsMade })) });
104
- });
105
-
106
- app.post('/admin/jobs/:id/retry', authenticate, authorize('admin'), async (req, res) => {
107
- const job = await emailQueue.getJob(req.params.id);
108
- if (!job) return res.status(404).json({ error: { code: 'NOT_FOUND' } });
109
- await job.retry();
110
- res.json({ status: 'retried' });
111
- });
112
-
113
- // Celery equivalent (Python) — minimal pattern
114
- # tasks.py
115
- from celery import Celery
116
- from celery.utils.log import get_task_logger
117
-
118
- app = Celery('tasks', broker=REDIS_URL, backend=REDIS_URL)
119
- app.conf.task_acks_late = True # at-least-once delivery
120
- app.conf.task_reject_on_worker_lost = True # requeue on worker crash
121
- logger = get_task_logger(__name__)
122
-
123
- @app.task(bind=True, max_retries=5, default_retry_delay=60)
124
- def send_email(self, to: str, template: str, data: dict) -> dict:
125
- try:
126
- result = email_service.send(to=to, template=template, data=data)
127
- return {'sent_at': result.timestamp.isoformat()}
128
- except TransientError as exc:
129
- raise self.retry(exc=exc, countdown=2 ** self.request.retries * 60)
130
- except PermanentError as exc:
131
- logger.error(f"Permanent failure for {to}: {exc}")
132
- raise # no retry — goes to dead letter
133
- ```
1
+ ---
2
+ name: "background-jobs"
3
+ pack: "@rune/backend"
4
+ description: "Queue-based async processing — BullMQ (Node.js), job patterns, retry strategies, idempotency, dead letter queues, monitoring."
5
+ model: sonnet
6
+ tools: [Read, Edit, Write, Grep, Glob, Bash]
7
+ ---
8
+
9
+ # background-jobs
10
+
11
+ Queue-based async processing — BullMQ (Node.js), job patterns, retry strategies, idempotency, dead letter queues, monitoring.
12
+
13
+ #### Workflow
14
+
15
+ **Step 1 — Identify async operations**
16
+ Scan route handlers and service functions for operations that: (a) take > 200ms (PDF generation, image resizing, report aggregation), (b) are non-user-facing (email sending, webhook delivery, analytics events), (c) can tolerate eventual consistency (data sync, cache warming, notification dispatch). Flag these as candidates for background jobs. Output a classification: fire-and-forget vs delayed vs scheduled (cron) vs fan-out.
17
+
18
+ **Step 2 — Choose queue system**
19
+ Node.js: BullMQ (Redis-backed, TypeScript-native, built-in retry/delay/priority/rate-limiting — recommended). Python: Celery + Redis/RabbitMQ broker (mature, distributed workers, beat scheduler for cron). For very simple use cases (single server, low volume): `node-cron` + in-process worker. Avoid in-process queues in production — they die with the process and lose jobs.
20
+
21
+ **Step 3 — Implement job with retry strategy**
22
+ Emit job producer (enqueue) and worker (processor) as separate files. Retry strategy: exponential backoff with jitter (`attempts: 5, backoff: { type: 'exponential', delay: 1000 }`). Idempotency: every job MUST have an idempotency key — use a deterministic ID from the operation (e.g., `email:welcome:${userId}` not a random UUID). This ensures duplicate enqueues (from retries, double-clicks) process exactly once. Dead letter queue: after max retries, move job to a `{queue-name}:failed` queue for inspection and manual replay — never silently drop.
23
+
24
+ **Step 4 — Add monitoring and alerting**
25
+ BullMQ Board or Bull Dashboard for visual queue monitoring. Emit metrics: queue depth (jobs waiting), processing rate (jobs/sec), failure rate (failed/total). Alert when: queue depth > threshold (workers not keeping up), failure rate > 5% (systematic error in processor), job age > expected TTL (stuck job). Use BullMQ events (`queue.on('failed', ...)`) to push metrics to Prometheus or Datadog.
26
+
27
+ **Step 5 — Handle dead letters**
28
+ Emit dead letter inspection endpoint: list failed jobs with error reason, retry count, and last error. Emit replay endpoint: re-enqueue a specific failed job with a fresh retry budget. Purge endpoint: clear dead letter queue after investigation. Add alerting on dead letter queue depth > 0 for critical job types (payment processing, compliance logging).
29
+
30
+ #### Example
31
+
32
+ ```typescript
33
+ // BullMQ setup with TypeScript — producer + worker
34
+ import { Queue, Worker, Job } from 'bullmq';
35
+
36
+ const connection = { host: REDIS_HOST, port: 6379 };
37
+
38
+ // Job type definitions
39
+ interface EmailJob { to: string; template: string; data: Record<string, unknown> }
40
+ interface PdfJob { reportId: string; userId: string; format: 'pdf' | 'xlsx' }
41
+
42
+ // Producers
43
+ export const emailQueue = new Queue<EmailJob>('email', { connection });
44
+ export const pdfQueue = new Queue<PdfJob>('pdf', { connection });
45
+
46
+ // Enqueue with idempotency key (jobId = idempotent identifier)
47
+ export const sendWelcomeEmail = (userId: string, email: string) =>
48
+ emailQueue.add('welcome', { to: email, template: 'welcome', data: { userId } }, {
49
+ jobId: `email:welcome:${userId}`, // prevents duplicate welcome emails
50
+ attempts: 3,
51
+ backoff: { type: 'exponential', delay: 2_000 },
52
+ removeOnComplete: { count: 1000 }, // keep last 1000 completed for audit
53
+ removeOnFail: false, // keep all failed for dead letter review
54
+ });
55
+
56
+ // Scheduled/delayed job
57
+ export const sendReminderEmail = (userId: string, delayMs: number) =>
58
+ emailQueue.add('reminder', { to: userId, template: 'reminder', data: {} }, {
59
+ delay: delayMs,
60
+ attempts: 5,
61
+ backoff: { type: 'exponential', delay: 5_000 },
62
+ });
63
+
64
+ // Worker processor with error handling
65
+ const emailWorker = new Worker<EmailJob>('email', async (job: Job<EmailJob>) => {
66
+ const { to, template, data } = job.data;
67
+ // Validate job data — serialized payload may be stale
68
+ if (!to || !template) throw new Error(`Invalid job payload: ${JSON.stringify(job.data)}`);
69
+ await emailService.send({ to, template, data });
70
+ // Return value is stored in job.returnvalue for audit
71
+ return { sentAt: new Date().toISOString() };
72
+ }, {
73
+ connection,
74
+ concurrency: 10, // process up to 10 emails in parallel
75
+ limiter: { max: 100, duration: 60_000 }, // rate limit: 100/min
76
+ });
77
+
78
+ emailWorker.on('failed', async (job, err) => {
79
+ logger.error({ jobId: job?.id, queue: 'email', error: err.message, attempts: job?.attemptsMade });
80
+ if (job?.attemptsMade >= job?.opts.attempts!) {
81
+ // max retries exhausted → alert
82
+ await alerting.notify(`Dead letter: email job ${job.id} failed after ${job.attemptsMade} attempts`);
83
+ }
84
+ });
85
+
86
+ // Fan-out pattern: one job enqueues many children
87
+ const fanOutNotification = async (eventId: string, userIds: string[]) => {
88
+ const jobs = userIds.map(userId => ({
89
+ name: 'notify',
90
+ data: { userId, eventId },
91
+ opts: {
92
+ jobId: `notify:${eventId}:${userId}`,
93
+ attempts: 3,
94
+ backoff: { type: 'exponential', delay: 1_000 },
95
+ },
96
+ }));
97
+ await notificationQueue.addBulk(jobs);
98
+ };
99
+
100
+ // Dead letter inspection API
101
+ app.get('/admin/jobs/failed', authenticate, authorize('admin'), async (req, res) => {
102
+ const failed = await emailQueue.getFailed(0, 50);
103
+ res.json({ count: failed.length, jobs: failed.map(j => ({ id: j.id, data: j.data, reason: j.failedReason, attempts: j.attemptsMade })) });
104
+ });
105
+
106
+ app.post('/admin/jobs/:id/retry', authenticate, authorize('admin'), async (req, res) => {
107
+ const job = await emailQueue.getJob(req.params.id);
108
+ if (!job) return res.status(404).json({ error: { code: 'NOT_FOUND' } });
109
+ await job.retry();
110
+ res.json({ status: 'retried' });
111
+ });
112
+
113
+ // Celery equivalent (Python) — minimal pattern
114
+ # tasks.py
115
+ from celery import Celery
116
+ from celery.utils.log import get_task_logger
117
+
118
+ app = Celery('tasks', broker=REDIS_URL, backend=REDIS_URL)
119
+ app.conf.task_acks_late = True # at-least-once delivery
120
+ app.conf.task_reject_on_worker_lost = True # requeue on worker crash
121
+ logger = get_task_logger(__name__)
122
+
123
+ @app.task(bind=True, max_retries=5, default_retry_delay=60)
124
+ def send_email(self, to: str, template: str, data: dict) -> dict:
125
+ try:
126
+ result = email_service.send(to=to, template=template, data=data)
127
+ return {'sent_at': result.timestamp.isoformat()}
128
+ except TransientError as exc:
129
+ raise self.retry(exc=exc, countdown=2 ** self.request.retries * 60)
130
+ except PermanentError as exc:
131
+ logger.error(f"Permanent failure for {to}: {exc}")
132
+ raise # no retry — goes to dead letter
133
+ ```
@@ -1,108 +1,108 @@
1
- ---
2
- name: "caching-patterns"
3
- pack: "@rune/backend"
4
- description: "Caching strategies for backend applications — in-memory LRU, Redis distributed cache, CDN/edge cache, browser cache headers, invalidation, and stampede prevention."
5
- model: sonnet
6
- tools: [Read, Edit, Write, Grep, Glob, Bash]
7
- ---
8
-
9
- # caching-patterns
10
-
11
- Caching strategies for backend applications — in-memory LRU, Redis distributed cache, CDN/edge cache, browser cache headers, invalidation, and stampede prevention.
12
-
13
- #### Workflow
14
-
15
- **Step 1 — Identify cacheable endpoints**
16
- Scan routes for: (a) read-heavy endpoints called frequently with the same inputs (user profile, product catalog, config lookups), (b) expensive computations (aggregations, report generation), (c) external API calls that are rate-limited or slow. Flag endpoints that mutate state as NOT cacheable at the response level (cache the data layer instead). Output a cacheable/non-cacheable classification per endpoint.
17
-
18
- **Step 2 — Select cache layer**
19
- Choose layer based on access pattern: in-memory (node-cache, LRU-cache) for single-process data with sub-millisecond access and low cardinality; Redis for distributed cache shared across multiple server instances or processes; CDN (Cloudflare, Fastly) for public, user-agnostic responses (marketing pages, public API responses); browser cache (`Cache-Control` headers) for static assets and safe GET responses. Hybrid: in-memory L1 + Redis L2 for hot-path data that justifies two-layer lookup.
20
-
21
- **Step 3 — Implement cache pattern**
22
- Cache-aside (most common): application checks cache first, on miss fetches from DB, writes to cache. Write-through: write to cache and DB together on every write (cache always warm, higher write latency). Write-behind (write-back): write to cache immediately, flush to DB asynchronously (lowest write latency, risk of data loss on crash). Read-through: cache sits in front of DB, handles miss transparently (simpler app code, less control). For most web APIs: cache-aside for reads + TTL-based expiry is the correct default.
23
-
24
- **Step 4 — Add invalidation strategy**
25
- TTL-based: set appropriate TTL per data type (user session: match auth token TTL; product catalog: 5–15min; config: 1hr). Event-driven: on mutation, publish event to Redis pub/sub, cache subscribers delete affected keys. Versioned keys: `cache:user:v3:{id}` — bump version in config to invalidate all users atomically. Tag-based: associate keys with tags (`tag:user:123`), delete all keys for a tag on mutation. Stale-while-revalidate: serve stale data immediately, refresh in background — valid for data where slight staleness is acceptable (leaderboards, stats). Emit invalidation hook alongside every write operation.
26
-
27
- **Step 5 — Monitor hit/miss ratio**
28
- Instrument cache calls to emit metrics: hit count, miss count, eviction count, cache size. Redis provides `INFO stats` — parse `keyspace_hits` and `keyspace_misses`. Target hit ratio > 80% for hot-path caches; < 50% indicates wrong key granularity or TTL too short. Alert on sudden hit ratio drop (invalidation bug) or memory > 80% of `maxmemory` (eviction risk).
29
-
30
- #### Example
31
-
32
- ```typescript
33
- // Redis cache-aside middleware for Express/Fastify
34
- import { Redis } from 'ioredis';
35
- const redis = new Redis(REDIS_URL);
36
-
37
- const cacheMiddleware = (ttlSeconds: number, keyFn?: (req) => string) =>
38
- async (req, res, next) => {
39
- const key = keyFn ? keyFn(req) : `cache:${req.method}:${req.originalUrl}`;
40
- const cached = await redis.get(key);
41
- if (cached) {
42
- res.setHeader('X-Cache', 'HIT');
43
- return res.json(JSON.parse(cached));
44
- }
45
- const originalJson = res.json.bind(res);
46
- res.json = (data) => {
47
- // Only cache successful responses
48
- if (res.statusCode < 400) redis.setex(key, ttlSeconds, JSON.stringify(data));
49
- res.setHeader('X-Cache', 'MISS');
50
- return originalJson(data);
51
- };
52
- next();
53
- };
54
-
55
- // Usage: cache product list for 5 minutes
56
- app.get('/products', cacheMiddleware(300), async (req, res) => { /* handler */ });
57
-
58
- // Cache stampede prevention: mutex lock on cache miss
59
- const getWithLock = async <T>(key: string, fetchFn: () => Promise<T>, ttl: number): Promise<T> => {
60
- const cached = await redis.get(key);
61
- if (cached) return JSON.parse(cached);
62
-
63
- const lockKey = `lock:${key}`;
64
- const lock = await redis.set(lockKey, '1', 'EX', 10, 'NX'); // 10s lock
65
- if (!lock) {
66
- // Another process is fetching — wait briefly and retry
67
- await new Promise(r => setTimeout(r, 100));
68
- return getWithLock(key, fetchFn, ttl); // retry (max ~10 cycles within 10s lock)
69
- }
70
-
71
- try {
72
- const data = await fetchFn();
73
- await redis.setex(key, ttl, JSON.stringify(data));
74
- return data;
75
- } finally {
76
- await redis.del(lockKey);
77
- }
78
- };
79
-
80
- // Event-driven invalidation with Redis pub/sub
81
- const invalidateOnMutation = async (userId: string) => {
82
- await redis.del(`cache:user:${userId}`);
83
- await redis.publish('cache:invalidate', JSON.stringify({ type: 'user', id: userId }));
84
- };
85
-
86
- // Cache-Control headers for browser/CDN caching
87
- app.get('/products', (req, res) => {
88
- res.setHeader('Cache-Control', 'public, max-age=300, stale-while-revalidate=60');
89
- // ^ CDN caches 5min, serves stale for extra 60s while revalidating in background
90
- res.json(products);
91
- });
92
-
93
- app.get('/user/profile', authenticate, (req, res) => {
94
- res.setHeader('Cache-Control', 'private, max-age=60'); // user-specific, browser only
95
- res.json(profile);
96
- });
97
-
98
- // In-memory LRU cache for single-process hot data
99
- import LRU from 'lru-cache';
100
- const configCache = new LRU<string, unknown>({ max: 500, ttl: 60_000 }); // 500 entries, 1min TTL
101
-
102
- const getConfig = async (key: string) => {
103
- if (configCache.has(key)) return configCache.get(key);
104
- const value = await db.config.findUnique({ where: { key } });
105
- configCache.set(key, value);
106
- return value;
107
- };
108
- ```
1
+ ---
2
+ name: "caching-patterns"
3
+ pack: "@rune/backend"
4
+ description: "Caching strategies for backend applications — in-memory LRU, Redis distributed cache, CDN/edge cache, browser cache headers, invalidation, and stampede prevention."
5
+ model: sonnet
6
+ tools: [Read, Edit, Write, Grep, Glob, Bash]
7
+ ---
8
+
9
+ # caching-patterns
10
+
11
+ Caching strategies for backend applications — in-memory LRU, Redis distributed cache, CDN/edge cache, browser cache headers, invalidation, and stampede prevention.
12
+
13
+ #### Workflow
14
+
15
+ **Step 1 — Identify cacheable endpoints**
16
+ Scan routes for: (a) read-heavy endpoints called frequently with the same inputs (user profile, product catalog, config lookups), (b) expensive computations (aggregations, report generation), (c) external API calls that are rate-limited or slow. Flag endpoints that mutate state as NOT cacheable at the response level (cache the data layer instead). Output a cacheable/non-cacheable classification per endpoint.
17
+
18
+ **Step 2 — Select cache layer**
19
+ Choose layer based on access pattern: in-memory (node-cache, LRU-cache) for single-process data with sub-millisecond access and low cardinality; Redis for distributed cache shared across multiple server instances or processes; CDN (Cloudflare, Fastly) for public, user-agnostic responses (marketing pages, public API responses); browser cache (`Cache-Control` headers) for static assets and safe GET responses. Hybrid: in-memory L1 + Redis L2 for hot-path data that justifies two-layer lookup.
20
+
21
+ **Step 3 — Implement cache pattern**
22
+ Cache-aside (most common): application checks cache first, on miss fetches from DB, writes to cache. Write-through: write to cache and DB together on every write (cache always warm, higher write latency). Write-behind (write-back): write to cache immediately, flush to DB asynchronously (lowest write latency, risk of data loss on crash). Read-through: cache sits in front of DB, handles miss transparently (simpler app code, less control). For most web APIs: cache-aside for reads + TTL-based expiry is the correct default.
23
+
24
+ **Step 4 — Add invalidation strategy**
25
+ TTL-based: set appropriate TTL per data type (user session: match auth token TTL; product catalog: 5–15min; config: 1hr). Event-driven: on mutation, publish event to Redis pub/sub, cache subscribers delete affected keys. Versioned keys: `cache:user:v3:{id}` — bump version in config to invalidate all users atomically. Tag-based: associate keys with tags (`tag:user:123`), delete all keys for a tag on mutation. Stale-while-revalidate: serve stale data immediately, refresh in background — valid for data where slight staleness is acceptable (leaderboards, stats). Emit invalidation hook alongside every write operation.
26
+
27
+ **Step 5 — Monitor hit/miss ratio**
28
+ Instrument cache calls to emit metrics: hit count, miss count, eviction count, cache size. Redis provides `INFO stats` — parse `keyspace_hits` and `keyspace_misses`. Target hit ratio > 80% for hot-path caches; < 50% indicates wrong key granularity or TTL too short. Alert on sudden hit ratio drop (invalidation bug) or memory > 80% of `maxmemory` (eviction risk).
29
+
30
+ #### Example
31
+
32
+ ```typescript
33
+ // Redis cache-aside middleware for Express/Fastify
34
+ import { Redis } from 'ioredis';
35
+ const redis = new Redis(REDIS_URL);
36
+
37
+ const cacheMiddleware = (ttlSeconds: number, keyFn?: (req) => string) =>
38
+ async (req, res, next) => {
39
+ const key = keyFn ? keyFn(req) : `cache:${req.method}:${req.originalUrl}`;
40
+ const cached = await redis.get(key);
41
+ if (cached) {
42
+ res.setHeader('X-Cache', 'HIT');
43
+ return res.json(JSON.parse(cached));
44
+ }
45
+ const originalJson = res.json.bind(res);
46
+ res.json = (data) => {
47
+ // Only cache successful responses
48
+ if (res.statusCode < 400) redis.setex(key, ttlSeconds, JSON.stringify(data));
49
+ res.setHeader('X-Cache', 'MISS');
50
+ return originalJson(data);
51
+ };
52
+ next();
53
+ };
54
+
55
+ // Usage: cache product list for 5 minutes
56
+ app.get('/products', cacheMiddleware(300), async (req, res) => { /* handler */ });
57
+
58
+ // Cache stampede prevention: mutex lock on cache miss
59
+ const getWithLock = async <T>(key: string, fetchFn: () => Promise<T>, ttl: number): Promise<T> => {
60
+ const cached = await redis.get(key);
61
+ if (cached) return JSON.parse(cached);
62
+
63
+ const lockKey = `lock:${key}`;
64
+ const lock = await redis.set(lockKey, '1', 'EX', 10, 'NX'); // 10s lock
65
+ if (!lock) {
66
+ // Another process is fetching — wait briefly and retry
67
+ await new Promise(r => setTimeout(r, 100));
68
+ return getWithLock(key, fetchFn, ttl); // retry (max ~10 cycles within 10s lock)
69
+ }
70
+
71
+ try {
72
+ const data = await fetchFn();
73
+ await redis.setex(key, ttl, JSON.stringify(data));
74
+ return data;
75
+ } finally {
76
+ await redis.del(lockKey);
77
+ }
78
+ };
79
+
80
+ // Event-driven invalidation with Redis pub/sub
81
+ const invalidateOnMutation = async (userId: string) => {
82
+ await redis.del(`cache:user:${userId}`);
83
+ await redis.publish('cache:invalidate', JSON.stringify({ type: 'user', id: userId }));
84
+ };
85
+
86
+ // Cache-Control headers for browser/CDN caching
87
+ app.get('/products', (req, res) => {
88
+ res.setHeader('Cache-Control', 'public, max-age=300, stale-while-revalidate=60');
89
+ // ^ CDN caches 5min, serves stale for extra 60s while revalidating in background
90
+ res.json(products);
91
+ });
92
+
93
+ app.get('/user/profile', authenticate, (req, res) => {
94
+ res.setHeader('Cache-Control', 'private, max-age=60'); // user-specific, browser only
95
+ res.json(profile);
96
+ });
97
+
98
+ // In-memory LRU cache for single-process hot data
99
+ import LRU from 'lru-cache';
100
+ const configCache = new LRU<string, unknown>({ max: 500, ttl: 60_000 }); // 500 entries, 1min TTL
101
+
102
+ const getConfig = async (key: string) => {
103
+ if (configCache.has(key)) return configCache.get(key);
104
+ const value = await db.config.findUnique({ where: { key } });
105
+ configCache.set(key, value);
106
+ return value;
107
+ };
108
+ ```