@rune-kit/rune 2.10.0 → 2.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (240) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +65 -6
  3. package/commands/rune.md +168 -168
  4. package/compiler/__tests__/detect-invariants.test.js +136 -0
  5. package/compiler/__tests__/doctor-mesh.test.js +229 -0
  6. package/compiler/__tests__/hook-dispatch.test.js +91 -0
  7. package/compiler/__tests__/hooks-antigravity.test.js +118 -0
  8. package/compiler/__tests__/hooks-cursor.test.js +139 -0
  9. package/compiler/__tests__/hooks-install.test.js +305 -0
  10. package/compiler/__tests__/hooks-merge.test.js +204 -0
  11. package/compiler/__tests__/hooks-tiers.test.js +519 -0
  12. package/compiler/__tests__/hooks-windsurf.test.js +115 -0
  13. package/compiler/__tests__/inject-claude-md.test.js +152 -0
  14. package/compiler/__tests__/load-invariants.test.js +408 -0
  15. package/compiler/__tests__/onboard-invariants.test.js +240 -0
  16. package/compiler/adapters/hooks/antigravity.js +140 -0
  17. package/compiler/adapters/hooks/claude.js +166 -0
  18. package/compiler/adapters/hooks/cursor.js +191 -0
  19. package/compiler/adapters/hooks/index.js +82 -0
  20. package/compiler/adapters/hooks/tier-emitter.js +182 -0
  21. package/compiler/adapters/hooks/windsurf.js +202 -0
  22. package/compiler/bin/rune.js +196 -6
  23. package/compiler/commands/hook-dispatch.js +87 -0
  24. package/compiler/commands/hooks/install.js +120 -0
  25. package/compiler/commands/hooks/merge.js +211 -0
  26. package/compiler/commands/hooks/presets.js +116 -0
  27. package/compiler/commands/hooks/status.js +112 -0
  28. package/compiler/commands/hooks/tiers.js +221 -0
  29. package/compiler/commands/hooks/uninstall.js +94 -0
  30. package/compiler/doctor.js +236 -0
  31. package/contexts/dev.md +34 -34
  32. package/contexts/research.md +43 -43
  33. package/contexts/review.md +55 -55
  34. package/extensions/ai-ml/PACK.md +88 -88
  35. package/extensions/ai-ml/skills/ai-agents.md +172 -172
  36. package/extensions/ai-ml/skills/code-sandbox.md +187 -187
  37. package/extensions/ai-ml/skills/deep-research.md +146 -146
  38. package/extensions/ai-ml/skills/embedding-search.md +66 -66
  39. package/extensions/ai-ml/skills/fine-tuning-guide.md +74 -74
  40. package/extensions/ai-ml/skills/llm-architect.md +125 -125
  41. package/extensions/ai-ml/skills/llm-integration.md +64 -64
  42. package/extensions/ai-ml/skills/prompt-patterns.md +72 -72
  43. package/extensions/ai-ml/skills/rag-patterns.md +66 -66
  44. package/extensions/ai-ml/skills/web-extraction.md +114 -114
  45. package/extensions/analytics/PACK.md +92 -92
  46. package/extensions/analytics/skills/ab-testing.md +72 -72
  47. package/extensions/analytics/skills/dashboard-patterns.md +83 -83
  48. package/extensions/analytics/skills/data-validation.md +68 -68
  49. package/extensions/analytics/skills/funnel-analysis.md +81 -81
  50. package/extensions/analytics/skills/sql-patterns.md +57 -57
  51. package/extensions/analytics/skills/statistical-analysis.md +79 -79
  52. package/extensions/analytics/skills/tracking-setup.md +71 -71
  53. package/extensions/backend/PACK.md +104 -104
  54. package/extensions/backend/skills/api-patterns.md +84 -84
  55. package/extensions/backend/skills/async-pipeline.md +193 -193
  56. package/extensions/backend/skills/auth-patterns.md +97 -97
  57. package/extensions/backend/skills/background-jobs.md +133 -133
  58. package/extensions/backend/skills/caching-patterns.md +108 -108
  59. package/extensions/backend/skills/cli-generation.md +133 -133
  60. package/extensions/backend/skills/database-patterns.md +87 -87
  61. package/extensions/backend/skills/middleware-patterns.md +104 -104
  62. package/extensions/chrome-ext/PACK.md +93 -93
  63. package/extensions/chrome-ext/skills/cws-preflight.md +143 -143
  64. package/extensions/chrome-ext/skills/cws-publish.md +104 -104
  65. package/extensions/chrome-ext/skills/ext-ai-integration.md +251 -251
  66. package/extensions/chrome-ext/skills/ext-messaging.md +139 -139
  67. package/extensions/chrome-ext/skills/ext-storage.md +133 -133
  68. package/extensions/chrome-ext/skills/mv3-scaffold.md +164 -164
  69. package/extensions/content/PACK.md +96 -96
  70. package/extensions/content/skills/blog-patterns.md +88 -88
  71. package/extensions/content/skills/cms-integration.md +131 -131
  72. package/extensions/content/skills/content-scoring.md +107 -107
  73. package/extensions/content/skills/i18n.md +83 -83
  74. package/extensions/content/skills/mdx-authoring.md +137 -137
  75. package/extensions/content/skills/reference.md +1014 -1014
  76. package/extensions/content/skills/seo-patterns.md +67 -67
  77. package/extensions/content/skills/video-repurpose.md +153 -153
  78. package/extensions/devops/PACK.md +101 -101
  79. package/extensions/devops/skills/chaos-testing.md +67 -67
  80. package/extensions/devops/skills/ci-cd.md +75 -75
  81. package/extensions/devops/skills/docker.md +58 -58
  82. package/extensions/devops/skills/edge-serverless.md +163 -163
  83. package/extensions/devops/skills/infra-as-code.md +158 -158
  84. package/extensions/devops/skills/kubernetes.md +110 -110
  85. package/extensions/devops/skills/monitoring.md +57 -57
  86. package/extensions/devops/skills/server-setup.md +64 -64
  87. package/extensions/devops/skills/ssl-domain.md +42 -42
  88. package/extensions/ecommerce/PACK.md +116 -116
  89. package/extensions/ecommerce/skills/cart-system.md +79 -79
  90. package/extensions/ecommerce/skills/inventory-mgmt.md +102 -102
  91. package/extensions/ecommerce/skills/order-management.md +126 -126
  92. package/extensions/ecommerce/skills/payment-integration.md +472 -472
  93. package/extensions/ecommerce/skills/shopify-dev.md +69 -69
  94. package/extensions/ecommerce/skills/subscription-billing.md +93 -93
  95. package/extensions/ecommerce/skills/tax-compliance.md +117 -117
  96. package/extensions/gamedev/PACK.md +142 -142
  97. package/extensions/gamedev/skills/asset-pipeline.md +74 -74
  98. package/extensions/gamedev/skills/audio-system.md +129 -129
  99. package/extensions/gamedev/skills/camera-system.md +87 -87
  100. package/extensions/gamedev/skills/ecs.md +98 -98
  101. package/extensions/gamedev/skills/game-loops.md +72 -72
  102. package/extensions/gamedev/skills/input-system.md +199 -199
  103. package/extensions/gamedev/skills/multiplayer.md +180 -180
  104. package/extensions/gamedev/skills/particles.md +105 -105
  105. package/extensions/gamedev/skills/physics-engine.md +89 -89
  106. package/extensions/gamedev/skills/scene-management.md +146 -146
  107. package/extensions/gamedev/skills/threejs-patterns.md +90 -90
  108. package/extensions/gamedev/skills/webgl.md +71 -71
  109. package/extensions/mobile/PACK.md +106 -106
  110. package/extensions/mobile/skills/app-store-connect.md +152 -152
  111. package/extensions/mobile/skills/app-store-prep.md +66 -66
  112. package/extensions/mobile/skills/deep-linking.md +109 -109
  113. package/extensions/mobile/skills/flutter.md +60 -60
  114. package/extensions/mobile/skills/ios-build-pipeline.md +142 -142
  115. package/extensions/mobile/skills/native-bridge.md +66 -66
  116. package/extensions/mobile/skills/ota-updates.md +97 -97
  117. package/extensions/mobile/skills/push-notifications.md +111 -111
  118. package/extensions/mobile/skills/react-native.md +82 -82
  119. package/extensions/saas/PACK.md +116 -116
  120. package/extensions/saas/skills/billing-integration.md +200 -200
  121. package/extensions/saas/skills/feature-flags.md +130 -130
  122. package/extensions/saas/skills/multi-tenant.md +103 -103
  123. package/extensions/saas/skills/onboarding-flow.md +139 -139
  124. package/extensions/saas/skills/subscription-flow.md +95 -95
  125. package/extensions/saas/skills/team-management.md +144 -144
  126. package/extensions/security/PACK.md +99 -99
  127. package/extensions/security/skills/api-security.md +140 -140
  128. package/extensions/security/skills/compliance.md +68 -68
  129. package/extensions/security/skills/owasp-audit.md +64 -64
  130. package/extensions/security/skills/pentest-patterns.md +77 -77
  131. package/extensions/security/skills/secret-mgmt.md +65 -65
  132. package/extensions/security/skills/supply-chain.md +65 -65
  133. package/extensions/trading/PACK.md +80 -80
  134. package/extensions/trading/skills/chart-components.md +55 -55
  135. package/extensions/trading/skills/experiment-loop.md +125 -125
  136. package/extensions/trading/skills/fintech-patterns.md +47 -47
  137. package/extensions/trading/skills/indicator-library.md +58 -58
  138. package/extensions/trading/skills/quant-analysis.md +111 -111
  139. package/extensions/trading/skills/realtime-data.md +58 -58
  140. package/extensions/trading/skills/trade-logic.md +104 -104
  141. package/extensions/ui/PACK.md +130 -130
  142. package/extensions/ui/skills/a11y-audit.md +91 -91
  143. package/extensions/ui/skills/animation-patterns.md +127 -127
  144. package/extensions/ui/skills/component-patterns.md +100 -100
  145. package/extensions/ui/skills/design-decision.md +108 -108
  146. package/extensions/ui/skills/design-system.md +68 -68
  147. package/extensions/ui/skills/landing-patterns.md +155 -155
  148. package/extensions/ui/skills/palette-picker.md +173 -173
  149. package/extensions/ui/skills/react-health.md +90 -90
  150. package/extensions/ui/skills/type-system.md +125 -125
  151. package/extensions/ui/skills/web-vitals.md +153 -153
  152. package/extensions/zalo/PACK.md +145 -145
  153. package/extensions/zalo/skills/zalo-oa-mcp.md +317 -317
  154. package/extensions/zalo/skills/zalo-oa-messaging.md +429 -429
  155. package/extensions/zalo/skills/zalo-oa-setup.md +236 -236
  156. package/extensions/zalo/skills/zalo-oa-webhook.md +189 -189
  157. package/extensions/zalo/skills/zalo-personal-messaging.md +194 -194
  158. package/extensions/zalo/skills/zalo-personal-setup.md +153 -153
  159. package/extensions/zalo/skills/zalo-rate-guard.md +219 -219
  160. package/hooks/auto-format/index.cjs +48 -48
  161. package/hooks/hooks.json +111 -111
  162. package/hooks/post-session-reflect/index.cjs +189 -189
  163. package/hooks/pre-compact/index.cjs +95 -95
  164. package/hooks/run-hook.cmd +1 -1
  165. package/hooks/secrets-scan/index.cjs +100 -100
  166. package/hooks/session-start/index.cjs +71 -71
  167. package/hooks/typecheck/index.cjs +65 -65
  168. package/package.json +63 -63
  169. package/references/ui-pro-max-data/LICENSE-UI-PRO-MAX +21 -21
  170. package/references/ui-pro-max-data/charts.csv +26 -26
  171. package/references/ui-pro-max-data/colors.csv +161 -161
  172. package/references/ui-pro-max-data/styles.csv +68 -68
  173. package/references/ui-pro-max-data/typography.csv +74 -74
  174. package/references/ui-pro-max-data/ui-reasoning.csv +162 -162
  175. package/references/ui-pro-max-data/ux-guidelines.csv +99 -99
  176. package/skills/adversary/SKILL.md +283 -283
  177. package/skills/asset-creator/SKILL.md +157 -157
  178. package/skills/audit/SKILL.md +147 -2
  179. package/skills/autopsy/SKILL.md +335 -335
  180. package/skills/ba/SKILL.md +85 -1
  181. package/skills/brainstorm/SKILL.md +380 -342
  182. package/skills/browser-pilot/SKILL.md +169 -168
  183. package/skills/constraint-check/SKILL.md +165 -165
  184. package/skills/context-engine/SKILL.md +408 -404
  185. package/skills/cook/SKILL.md +917 -863
  186. package/skills/db/SKILL.md +273 -273
  187. package/skills/debug/SKILL.md +465 -465
  188. package/skills/dependency-doctor/SKILL.md +265 -235
  189. package/skills/deploy/SKILL.md +274 -231
  190. package/skills/design/DESIGN-REFERENCE.md +365 -365
  191. package/skills/design/SKILL.md +590 -589
  192. package/skills/doc-processor/SKILL.md +254 -254
  193. package/skills/docs/SKILL.md +374 -374
  194. package/skills/docs-seeker/SKILL.md +178 -177
  195. package/skills/fix/SKILL.md +332 -330
  196. package/skills/git/SKILL.md +339 -339
  197. package/skills/hallucination-guard/SKILL.md +220 -219
  198. package/skills/incident/SKILL.md +254 -253
  199. package/skills/integrity-check/SKILL.md +169 -169
  200. package/skills/journal/SKILL.md +241 -240
  201. package/skills/launch/SKILL.md +344 -344
  202. package/skills/logic-guardian/SKILL.md +269 -251
  203. package/skills/marketing/SKILL.md +351 -289
  204. package/skills/mcp-builder/SKILL.md +425 -425
  205. package/skills/neural-memory/SKILL.md +359 -362
  206. package/skills/onboard/SKILL.md +432 -403
  207. package/skills/onboard/references/invariants-template.md +76 -0
  208. package/skills/onboard/scripts/detect-invariants.js +439 -0
  209. package/skills/onboard/scripts/inject-claude-md.js +150 -0
  210. package/skills/onboard/scripts/onboard-invariants.js +194 -0
  211. package/skills/perf/SKILL.md +347 -346
  212. package/skills/plan/SKILL.md +435 -428
  213. package/skills/preflight/SKILL.md +415 -415
  214. package/skills/problem-solver/SKILL.md +380 -284
  215. package/skills/rescue/SKILL.md +474 -474
  216. package/skills/research/SKILL.md +4 -0
  217. package/skills/retro/SKILL.md +3 -1
  218. package/skills/review/SKILL.md +614 -588
  219. package/skills/review-intake/SKILL.md +249 -249
  220. package/skills/safeguard/SKILL.md +200 -200
  221. package/skills/sast/SKILL.md +190 -190
  222. package/skills/scaffold/SKILL.md +328 -287
  223. package/skills/scope-guard/SKILL.md +183 -180
  224. package/skills/scout/SKILL.md +269 -263
  225. package/skills/sentinel/SKILL.md +384 -381
  226. package/skills/sentinel-env/SKILL.md +254 -254
  227. package/skills/sequential-thinking/SKILL.md +234 -234
  228. package/skills/session-bridge/SKILL.md +595 -543
  229. package/skills/session-bridge/scripts/load-invariants.js +397 -0
  230. package/skills/skill-forge/SKILL.md +581 -581
  231. package/skills/skill-router/SKILL.md +3 -0
  232. package/skills/slides/SKILL.md +19 -0
  233. package/skills/surgeon/SKILL.md +215 -215
  234. package/skills/team/SKILL.md +557 -537
  235. package/skills/test/SKILL.md +620 -614
  236. package/skills/trend-scout/SKILL.md +145 -145
  237. package/skills/verification/SKILL.md +334 -326
  238. package/skills/video-creator/SKILL.md +201 -201
  239. package/skills/watchdog/SKILL.md +168 -168
  240. package/skills/worktree/SKILL.md +140 -140
@@ -1,133 +1,133 @@
1
- ---
2
- name: "background-jobs"
3
- pack: "@rune/backend"
4
- description: "Queue-based async processing — BullMQ (Node.js), job patterns, retry strategies, idempotency, dead letter queues, monitoring."
5
- model: sonnet
6
- tools: [Read, Edit, Write, Grep, Glob, Bash]
7
- ---
8
-
9
- # background-jobs
10
-
11
- Queue-based async processing — BullMQ (Node.js), job patterns, retry strategies, idempotency, dead letter queues, monitoring.
12
-
13
- #### Workflow
14
-
15
- **Step 1 — Identify async operations**
16
- Scan route handlers and service functions for operations that: (a) take > 200ms (PDF generation, image resizing, report aggregation), (b) are non-user-facing (email sending, webhook delivery, analytics events), (c) can tolerate eventual consistency (data sync, cache warming, notification dispatch). Flag these as candidates for background jobs. Output a classification: fire-and-forget vs delayed vs scheduled (cron) vs fan-out.
17
-
18
- **Step 2 — Choose queue system**
19
- Node.js: BullMQ (Redis-backed, TypeScript-native, built-in retry/delay/priority/rate-limiting — recommended). Python: Celery + Redis/RabbitMQ broker (mature, distributed workers, beat scheduler for cron). For very simple use cases (single server, low volume): `node-cron` + in-process worker. Avoid in-process queues in production — they die with the process and lose jobs.
20
-
21
- **Step 3 — Implement job with retry strategy**
22
- Emit job producer (enqueue) and worker (processor) as separate files. Retry strategy: exponential backoff with jitter (`attempts: 5, backoff: { type: 'exponential', delay: 1000 }`). Idempotency: every job MUST have an idempotency key — use a deterministic ID from the operation (e.g., `email:welcome:${userId}` not a random UUID). This ensures duplicate enqueues (from retries, double-clicks) process exactly once. Dead letter queue: after max retries, move job to a `{queue-name}:failed` queue for inspection and manual replay — never silently drop.
23
-
24
- **Step 4 — Add monitoring and alerting**
25
- BullMQ Board or Bull Dashboard for visual queue monitoring. Emit metrics: queue depth (jobs waiting), processing rate (jobs/sec), failure rate (failed/total). Alert when: queue depth > threshold (workers not keeping up), failure rate > 5% (systematic error in processor), job age > expected TTL (stuck job). Use BullMQ events (`queue.on('failed', ...)`) to push metrics to Prometheus or Datadog.
26
-
27
- **Step 5 — Handle dead letters**
28
- Emit dead letter inspection endpoint: list failed jobs with error reason, retry count, and last error. Emit replay endpoint: re-enqueue a specific failed job with a fresh retry budget. Purge endpoint: clear dead letter queue after investigation. Add alerting on dead letter queue depth > 0 for critical job types (payment processing, compliance logging).
29
-
30
- #### Example
31
-
32
- ```typescript
33
- // BullMQ setup with TypeScript — producer + worker
34
- import { Queue, Worker, Job } from 'bullmq';
35
-
36
- const connection = { host: REDIS_HOST, port: 6379 };
37
-
38
- // Job type definitions
39
- interface EmailJob { to: string; template: string; data: Record<string, unknown> }
40
- interface PdfJob { reportId: string; userId: string; format: 'pdf' | 'xlsx' }
41
-
42
- // Producers
43
- export const emailQueue = new Queue<EmailJob>('email', { connection });
44
- export const pdfQueue = new Queue<PdfJob>('pdf', { connection });
45
-
46
- // Enqueue with idempotency key (jobId = idempotent identifier)
47
- export const sendWelcomeEmail = (userId: string, email: string) =>
48
- emailQueue.add('welcome', { to: email, template: 'welcome', data: { userId } }, {
49
- jobId: `email:welcome:${userId}`, // prevents duplicate welcome emails
50
- attempts: 3,
51
- backoff: { type: 'exponential', delay: 2_000 },
52
- removeOnComplete: { count: 1000 }, // keep last 1000 completed for audit
53
- removeOnFail: false, // keep all failed for dead letter review
54
- });
55
-
56
- // Scheduled/delayed job
57
- export const sendReminderEmail = (userId: string, delayMs: number) =>
58
- emailQueue.add('reminder', { to: userId, template: 'reminder', data: {} }, {
59
- delay: delayMs,
60
- attempts: 5,
61
- backoff: { type: 'exponential', delay: 5_000 },
62
- });
63
-
64
- // Worker processor with error handling
65
- const emailWorker = new Worker<EmailJob>('email', async (job: Job<EmailJob>) => {
66
- const { to, template, data } = job.data;
67
- // Validate job data — serialized payload may be stale
68
- if (!to || !template) throw new Error(`Invalid job payload: ${JSON.stringify(job.data)}`);
69
- await emailService.send({ to, template, data });
70
- // Return value is stored in job.returnvalue for audit
71
- return { sentAt: new Date().toISOString() };
72
- }, {
73
- connection,
74
- concurrency: 10, // process up to 10 emails in parallel
75
- limiter: { max: 100, duration: 60_000 }, // rate limit: 100/min
76
- });
77
-
78
- emailWorker.on('failed', async (job, err) => {
79
- logger.error({ jobId: job?.id, queue: 'email', error: err.message, attempts: job?.attemptsMade });
80
- if (job?.attemptsMade >= job?.opts.attempts!) {
81
- // max retries exhausted → alert
82
- await alerting.notify(`Dead letter: email job ${job.id} failed after ${job.attemptsMade} attempts`);
83
- }
84
- });
85
-
86
- // Fan-out pattern: one job enqueues many children
87
- const fanOutNotification = async (eventId: string, userIds: string[]) => {
88
- const jobs = userIds.map(userId => ({
89
- name: 'notify',
90
- data: { userId, eventId },
91
- opts: {
92
- jobId: `notify:${eventId}:${userId}`,
93
- attempts: 3,
94
- backoff: { type: 'exponential', delay: 1_000 },
95
- },
96
- }));
97
- await notificationQueue.addBulk(jobs);
98
- };
99
-
100
- // Dead letter inspection API
101
- app.get('/admin/jobs/failed', authenticate, authorize('admin'), async (req, res) => {
102
- const failed = await emailQueue.getFailed(0, 50);
103
- res.json({ count: failed.length, jobs: failed.map(j => ({ id: j.id, data: j.data, reason: j.failedReason, attempts: j.attemptsMade })) });
104
- });
105
-
106
- app.post('/admin/jobs/:id/retry', authenticate, authorize('admin'), async (req, res) => {
107
- const job = await emailQueue.getJob(req.params.id);
108
- if (!job) return res.status(404).json({ error: { code: 'NOT_FOUND' } });
109
- await job.retry();
110
- res.json({ status: 'retried' });
111
- });
112
-
113
- // Celery equivalent (Python) — minimal pattern
114
- # tasks.py
115
- from celery import Celery
116
- from celery.utils.log import get_task_logger
117
-
118
- app = Celery('tasks', broker=REDIS_URL, backend=REDIS_URL)
119
- app.conf.task_acks_late = True # at-least-once delivery
120
- app.conf.task_reject_on_worker_lost = True # requeue on worker crash
121
- logger = get_task_logger(__name__)
122
-
123
- @app.task(bind=True, max_retries=5, default_retry_delay=60)
124
- def send_email(self, to: str, template: str, data: dict) -> dict:
125
- try:
126
- result = email_service.send(to=to, template=template, data=data)
127
- return {'sent_at': result.timestamp.isoformat()}
128
- except TransientError as exc:
129
- raise self.retry(exc=exc, countdown=2 ** self.request.retries * 60)
130
- except PermanentError as exc:
131
- logger.error(f"Permanent failure for {to}: {exc}")
132
- raise # no retry — goes to dead letter
133
- ```
1
+ ---
2
+ name: "background-jobs"
3
+ pack: "@rune/backend"
4
+ description: "Queue-based async processing — BullMQ (Node.js), job patterns, retry strategies, idempotency, dead letter queues, monitoring."
5
+ model: sonnet
6
+ tools: [Read, Edit, Write, Grep, Glob, Bash]
7
+ ---
8
+
9
+ # background-jobs
10
+
11
+ Queue-based async processing — BullMQ (Node.js), job patterns, retry strategies, idempotency, dead letter queues, monitoring.
12
+
13
+ #### Workflow
14
+
15
+ **Step 1 — Identify async operations**
16
+ Scan route handlers and service functions for operations that: (a) take > 200ms (PDF generation, image resizing, report aggregation), (b) are non-user-facing (email sending, webhook delivery, analytics events), (c) can tolerate eventual consistency (data sync, cache warming, notification dispatch). Flag these as candidates for background jobs. Output a classification: fire-and-forget vs delayed vs scheduled (cron) vs fan-out.
17
+
18
+ **Step 2 — Choose queue system**
19
+ Node.js: BullMQ (Redis-backed, TypeScript-native, built-in retry/delay/priority/rate-limiting — recommended). Python: Celery + Redis/RabbitMQ broker (mature, distributed workers, beat scheduler for cron). For very simple use cases (single server, low volume): `node-cron` + in-process worker. Avoid in-process queues in production — they die with the process and lose jobs.
20
+
21
+ **Step 3 — Implement job with retry strategy**
22
+ Emit job producer (enqueue) and worker (processor) as separate files. Retry strategy: exponential backoff with jitter (`attempts: 5, backoff: { type: 'exponential', delay: 1000 }`). Idempotency: every job MUST have an idempotency key — use a deterministic ID from the operation (e.g., `email:welcome:${userId}` not a random UUID). This ensures duplicate enqueues (from retries, double-clicks) process exactly once. Dead letter queue: after max retries, move job to a `{queue-name}:failed` queue for inspection and manual replay — never silently drop.
23
+
24
+ **Step 4 — Add monitoring and alerting**
25
+ BullMQ Board or Bull Dashboard for visual queue monitoring. Emit metrics: queue depth (jobs waiting), processing rate (jobs/sec), failure rate (failed/total). Alert when: queue depth > threshold (workers not keeping up), failure rate > 5% (systematic error in processor), job age > expected TTL (stuck job). Use BullMQ events (`queue.on('failed', ...)`) to push metrics to Prometheus or Datadog.
26
+
27
+ **Step 5 — Handle dead letters**
28
+ Emit dead letter inspection endpoint: list failed jobs with error reason, retry count, and last error. Emit replay endpoint: re-enqueue a specific failed job with a fresh retry budget. Purge endpoint: clear dead letter queue after investigation. Add alerting on dead letter queue depth > 0 for critical job types (payment processing, compliance logging).
29
+
30
+ #### Example
31
+
32
+ ```typescript
33
+ // BullMQ setup with TypeScript — producer + worker
34
+ import { Queue, Worker, Job } from 'bullmq';
35
+
36
+ const connection = { host: REDIS_HOST, port: 6379 };
37
+
38
+ // Job type definitions
39
+ interface EmailJob { to: string; template: string; data: Record<string, unknown> }
40
+ interface PdfJob { reportId: string; userId: string; format: 'pdf' | 'xlsx' }
41
+
42
+ // Producers
43
+ export const emailQueue = new Queue<EmailJob>('email', { connection });
44
+ export const pdfQueue = new Queue<PdfJob>('pdf', { connection });
45
+
46
+ // Enqueue with idempotency key (jobId = idempotent identifier)
47
+ export const sendWelcomeEmail = (userId: string, email: string) =>
48
+ emailQueue.add('welcome', { to: email, template: 'welcome', data: { userId } }, {
49
+ jobId: `email:welcome:${userId}`, // prevents duplicate welcome emails
50
+ attempts: 3,
51
+ backoff: { type: 'exponential', delay: 2_000 },
52
+ removeOnComplete: { count: 1000 }, // keep last 1000 completed for audit
53
+ removeOnFail: false, // keep all failed for dead letter review
54
+ });
55
+
56
+ // Scheduled/delayed job
57
+ export const sendReminderEmail = (userId: string, delayMs: number) =>
58
+ emailQueue.add('reminder', { to: userId, template: 'reminder', data: {} }, {
59
+ delay: delayMs,
60
+ attempts: 5,
61
+ backoff: { type: 'exponential', delay: 5_000 },
62
+ });
63
+
64
+ // Worker processor with error handling
65
+ const emailWorker = new Worker<EmailJob>('email', async (job: Job<EmailJob>) => {
66
+ const { to, template, data } = job.data;
67
+ // Validate job data — serialized payload may be stale
68
+ if (!to || !template) throw new Error(`Invalid job payload: ${JSON.stringify(job.data)}`);
69
+ await emailService.send({ to, template, data });
70
+ // Return value is stored in job.returnvalue for audit
71
+ return { sentAt: new Date().toISOString() };
72
+ }, {
73
+ connection,
74
+ concurrency: 10, // process up to 10 emails in parallel
75
+ limiter: { max: 100, duration: 60_000 }, // rate limit: 100/min
76
+ });
77
+
78
+ emailWorker.on('failed', async (job, err) => {
79
+ logger.error({ jobId: job?.id, queue: 'email', error: err.message, attempts: job?.attemptsMade });
80
+ if (job?.attemptsMade >= job?.opts.attempts!) {
81
+ // max retries exhausted → alert
82
+ await alerting.notify(`Dead letter: email job ${job.id} failed after ${job.attemptsMade} attempts`);
83
+ }
84
+ });
85
+
86
+ // Fan-out pattern: one job enqueues many children
87
+ const fanOutNotification = async (eventId: string, userIds: string[]) => {
88
+ const jobs = userIds.map(userId => ({
89
+ name: 'notify',
90
+ data: { userId, eventId },
91
+ opts: {
92
+ jobId: `notify:${eventId}:${userId}`,
93
+ attempts: 3,
94
+ backoff: { type: 'exponential', delay: 1_000 },
95
+ },
96
+ }));
97
+ await notificationQueue.addBulk(jobs);
98
+ };
99
+
100
+ // Dead letter inspection API
101
+ app.get('/admin/jobs/failed', authenticate, authorize('admin'), async (req, res) => {
102
+ const failed = await emailQueue.getFailed(0, 50);
103
+ res.json({ count: failed.length, jobs: failed.map(j => ({ id: j.id, data: j.data, reason: j.failedReason, attempts: j.attemptsMade })) });
104
+ });
105
+
106
+ app.post('/admin/jobs/:id/retry', authenticate, authorize('admin'), async (req, res) => {
107
+ const job = await emailQueue.getJob(req.params.id);
108
+ if (!job) return res.status(404).json({ error: { code: 'NOT_FOUND' } });
109
+ await job.retry();
110
+ res.json({ status: 'retried' });
111
+ });
112
+
113
+ // Celery equivalent (Python) — minimal pattern
114
+ # tasks.py
115
+ from celery import Celery
116
+ from celery.utils.log import get_task_logger
117
+
118
+ app = Celery('tasks', broker=REDIS_URL, backend=REDIS_URL)
119
+ app.conf.task_acks_late = True # at-least-once delivery
120
+ app.conf.task_reject_on_worker_lost = True # requeue on worker crash
121
+ logger = get_task_logger(__name__)
122
+
123
+ @app.task(bind=True, max_retries=5, default_retry_delay=60)
124
+ def send_email(self, to: str, template: str, data: dict) -> dict:
125
+ try:
126
+ result = email_service.send(to=to, template=template, data=data)
127
+ return {'sent_at': result.timestamp.isoformat()}
128
+ except TransientError as exc:
129
+ raise self.retry(exc=exc, countdown=2 ** self.request.retries * 60)
130
+ except PermanentError as exc:
131
+ logger.error(f"Permanent failure for {to}: {exc}")
132
+ raise # no retry — goes to dead letter
133
+ ```
@@ -1,108 +1,108 @@
1
- ---
2
- name: "caching-patterns"
3
- pack: "@rune/backend"
4
- description: "Caching strategies for backend applications — in-memory LRU, Redis distributed cache, CDN/edge cache, browser cache headers, invalidation, and stampede prevention."
5
- model: sonnet
6
- tools: [Read, Edit, Write, Grep, Glob, Bash]
7
- ---
8
-
9
- # caching-patterns
10
-
11
- Caching strategies for backend applications — in-memory LRU, Redis distributed cache, CDN/edge cache, browser cache headers, invalidation, and stampede prevention.
12
-
13
- #### Workflow
14
-
15
- **Step 1 — Identify cacheable endpoints**
16
- Scan routes for: (a) read-heavy endpoints called frequently with the same inputs (user profile, product catalog, config lookups), (b) expensive computations (aggregations, report generation), (c) external API calls that are rate-limited or slow. Flag endpoints that mutate state as NOT cacheable at the response level (cache the data layer instead). Output a cacheable/non-cacheable classification per endpoint.
17
-
18
- **Step 2 — Select cache layer**
19
- Choose layer based on access pattern: in-memory (node-cache, LRU-cache) for single-process data with sub-millisecond access and low cardinality; Redis for distributed cache shared across multiple server instances or processes; CDN (Cloudflare, Fastly) for public, user-agnostic responses (marketing pages, public API responses); browser cache (`Cache-Control` headers) for static assets and safe GET responses. Hybrid: in-memory L1 + Redis L2 for hot-path data that justifies two-layer lookup.
20
-
21
- **Step 3 — Implement cache pattern**
22
- Cache-aside (most common): application checks cache first, on miss fetches from DB, writes to cache. Write-through: write to cache and DB together on every write (cache always warm, higher write latency). Write-behind (write-back): write to cache immediately, flush to DB asynchronously (lowest write latency, risk of data loss on crash). Read-through: cache sits in front of DB, handles miss transparently (simpler app code, less control). For most web APIs: cache-aside for reads + TTL-based expiry is the correct default.
23
-
24
- **Step 4 — Add invalidation strategy**
25
- TTL-based: set appropriate TTL per data type (user session: match auth token TTL; product catalog: 5–15min; config: 1hr). Event-driven: on mutation, publish event to Redis pub/sub, cache subscribers delete affected keys. Versioned keys: `cache:user:v3:{id}` — bump version in config to invalidate all users atomically. Tag-based: associate keys with tags (`tag:user:123`), delete all keys for a tag on mutation. Stale-while-revalidate: serve stale data immediately, refresh in background — valid for data where slight staleness is acceptable (leaderboards, stats). Emit invalidation hook alongside every write operation.
26
-
27
- **Step 5 — Monitor hit/miss ratio**
28
- Instrument cache calls to emit metrics: hit count, miss count, eviction count, cache size. Redis provides `INFO stats` — parse `keyspace_hits` and `keyspace_misses`. Target hit ratio > 80% for hot-path caches; < 50% indicates wrong key granularity or TTL too short. Alert on sudden hit ratio drop (invalidation bug) or memory > 80% of `maxmemory` (eviction risk).
29
-
30
- #### Example
31
-
32
- ```typescript
33
- // Redis cache-aside middleware for Express/Fastify
34
- import { Redis } from 'ioredis';
35
- const redis = new Redis(REDIS_URL);
36
-
37
- const cacheMiddleware = (ttlSeconds: number, keyFn?: (req) => string) =>
38
- async (req, res, next) => {
39
- const key = keyFn ? keyFn(req) : `cache:${req.method}:${req.originalUrl}`;
40
- const cached = await redis.get(key);
41
- if (cached) {
42
- res.setHeader('X-Cache', 'HIT');
43
- return res.json(JSON.parse(cached));
44
- }
45
- const originalJson = res.json.bind(res);
46
- res.json = (data) => {
47
- // Only cache successful responses
48
- if (res.statusCode < 400) redis.setex(key, ttlSeconds, JSON.stringify(data));
49
- res.setHeader('X-Cache', 'MISS');
50
- return originalJson(data);
51
- };
52
- next();
53
- };
54
-
55
- // Usage: cache product list for 5 minutes
56
- app.get('/products', cacheMiddleware(300), async (req, res) => { /* handler */ });
57
-
58
- // Cache stampede prevention: mutex lock on cache miss
59
- const getWithLock = async <T>(key: string, fetchFn: () => Promise<T>, ttl: number): Promise<T> => {
60
- const cached = await redis.get(key);
61
- if (cached) return JSON.parse(cached);
62
-
63
- const lockKey = `lock:${key}`;
64
- const lock = await redis.set(lockKey, '1', 'EX', 10, 'NX'); // 10s lock
65
- if (!lock) {
66
- // Another process is fetching — wait briefly and retry
67
- await new Promise(r => setTimeout(r, 100));
68
- return getWithLock(key, fetchFn, ttl); // retry (max ~10 cycles within 10s lock)
69
- }
70
-
71
- try {
72
- const data = await fetchFn();
73
- await redis.setex(key, ttl, JSON.stringify(data));
74
- return data;
75
- } finally {
76
- await redis.del(lockKey);
77
- }
78
- };
79
-
80
- // Event-driven invalidation with Redis pub/sub
81
- const invalidateOnMutation = async (userId: string) => {
82
- await redis.del(`cache:user:${userId}`);
83
- await redis.publish('cache:invalidate', JSON.stringify({ type: 'user', id: userId }));
84
- };
85
-
86
- // Cache-Control headers for browser/CDN caching
87
- app.get('/products', (req, res) => {
88
- res.setHeader('Cache-Control', 'public, max-age=300, stale-while-revalidate=60');
89
- // ^ CDN caches 5min, serves stale for extra 60s while revalidating in background
90
- res.json(products);
91
- });
92
-
93
- app.get('/user/profile', authenticate, (req, res) => {
94
- res.setHeader('Cache-Control', 'private, max-age=60'); // user-specific, browser only
95
- res.json(profile);
96
- });
97
-
98
- // In-memory LRU cache for single-process hot data
99
- import LRU from 'lru-cache';
100
- const configCache = new LRU<string, unknown>({ max: 500, ttl: 60_000 }); // 500 entries, 1min TTL
101
-
102
- const getConfig = async (key: string) => {
103
- if (configCache.has(key)) return configCache.get(key);
104
- const value = await db.config.findUnique({ where: { key } });
105
- configCache.set(key, value);
106
- return value;
107
- };
108
- ```
1
+ ---
2
+ name: "caching-patterns"
3
+ pack: "@rune/backend"
4
+ description: "Caching strategies for backend applications — in-memory LRU, Redis distributed cache, CDN/edge cache, browser cache headers, invalidation, and stampede prevention."
5
+ model: sonnet
6
+ tools: [Read, Edit, Write, Grep, Glob, Bash]
7
+ ---
8
+
9
+ # caching-patterns
10
+
11
+ Caching strategies for backend applications — in-memory LRU, Redis distributed cache, CDN/edge cache, browser cache headers, invalidation, and stampede prevention.
12
+
13
+ #### Workflow
14
+
15
+ **Step 1 — Identify cacheable endpoints**
16
+ Scan routes for: (a) read-heavy endpoints called frequently with the same inputs (user profile, product catalog, config lookups), (b) expensive computations (aggregations, report generation), (c) external API calls that are rate-limited or slow. Flag endpoints that mutate state as NOT cacheable at the response level (cache the data layer instead). Output a cacheable/non-cacheable classification per endpoint.
17
+
18
+ **Step 2 — Select cache layer**
19
+ Choose layer based on access pattern: in-memory (node-cache, LRU-cache) for single-process data with sub-millisecond access and low cardinality; Redis for distributed cache shared across multiple server instances or processes; CDN (Cloudflare, Fastly) for public, user-agnostic responses (marketing pages, public API responses); browser cache (`Cache-Control` headers) for static assets and safe GET responses. Hybrid: in-memory L1 + Redis L2 for hot-path data that justifies two-layer lookup.
20
+
21
+ **Step 3 — Implement cache pattern**
22
+ Cache-aside (most common): application checks cache first, on miss fetches from DB, writes to cache. Write-through: write to cache and DB together on every write (cache always warm, higher write latency). Write-behind (write-back): write to cache immediately, flush to DB asynchronously (lowest write latency, risk of data loss on crash). Read-through: cache sits in front of DB, handles miss transparently (simpler app code, less control). For most web APIs: cache-aside for reads + TTL-based expiry is the correct default.
23
+
24
+ **Step 4 — Add invalidation strategy**
25
+ TTL-based: set appropriate TTL per data type (user session: match auth token TTL; product catalog: 5–15min; config: 1hr). Event-driven: on mutation, publish event to Redis pub/sub, cache subscribers delete affected keys. Versioned keys: `cache:user:v3:{id}` — bump version in config to invalidate all users atomically. Tag-based: associate keys with tags (`tag:user:123`), delete all keys for a tag on mutation. Stale-while-revalidate: serve stale data immediately, refresh in background — valid for data where slight staleness is acceptable (leaderboards, stats). Emit invalidation hook alongside every write operation.
26
+
27
+ **Step 5 — Monitor hit/miss ratio**
28
+ Instrument cache calls to emit metrics: hit count, miss count, eviction count, cache size. Redis provides `INFO stats` — parse `keyspace_hits` and `keyspace_misses`. Target hit ratio > 80% for hot-path caches; < 50% indicates wrong key granularity or TTL too short. Alert on sudden hit ratio drop (invalidation bug) or memory > 80% of `maxmemory` (eviction risk).
29
+
30
+ #### Example
31
+
32
+ ```typescript
33
+ // Redis cache-aside middleware for Express/Fastify
34
+ import { Redis } from 'ioredis';
35
+ const redis = new Redis(REDIS_URL);
36
+
37
+ const cacheMiddleware = (ttlSeconds: number, keyFn?: (req) => string) =>
38
+ async (req, res, next) => {
39
+ const key = keyFn ? keyFn(req) : `cache:${req.method}:${req.originalUrl}`;
40
+ const cached = await redis.get(key);
41
+ if (cached) {
42
+ res.setHeader('X-Cache', 'HIT');
43
+ return res.json(JSON.parse(cached));
44
+ }
45
+ const originalJson = res.json.bind(res);
46
+ res.json = (data) => {
47
+ // Only cache successful responses
48
+ if (res.statusCode < 400) redis.setex(key, ttlSeconds, JSON.stringify(data));
49
+ res.setHeader('X-Cache', 'MISS');
50
+ return originalJson(data);
51
+ };
52
+ next();
53
+ };
54
+
55
+ // Usage: cache product list for 5 minutes
56
+ app.get('/products', cacheMiddleware(300), async (req, res) => { /* handler */ });
57
+
58
+ // Cache stampede prevention: mutex lock on cache miss
59
+ const getWithLock = async <T>(key: string, fetchFn: () => Promise<T>, ttl: number): Promise<T> => {
60
+ const cached = await redis.get(key);
61
+ if (cached) return JSON.parse(cached);
62
+
63
+ const lockKey = `lock:${key}`;
64
+ const lock = await redis.set(lockKey, '1', 'EX', 10, 'NX'); // 10s lock
65
+ if (!lock) {
66
+ // Another process is fetching — wait briefly and retry
67
+ await new Promise(r => setTimeout(r, 100));
68
+ return getWithLock(key, fetchFn, ttl); // retry (max ~10 cycles within 10s lock)
69
+ }
70
+
71
+ try {
72
+ const data = await fetchFn();
73
+ await redis.setex(key, ttl, JSON.stringify(data));
74
+ return data;
75
+ } finally {
76
+ await redis.del(lockKey);
77
+ }
78
+ };
79
+
80
+ // Event-driven invalidation with Redis pub/sub
81
+ const invalidateOnMutation = async (userId: string) => {
82
+ await redis.del(`cache:user:${userId}`);
83
+ await redis.publish('cache:invalidate', JSON.stringify({ type: 'user', id: userId }));
84
+ };
85
+
86
+ // Cache-Control headers for browser/CDN caching
87
+ app.get('/products', (req, res) => {
88
+ res.setHeader('Cache-Control', 'public, max-age=300, stale-while-revalidate=60');
89
+ // ^ CDN caches 5min, serves stale for extra 60s while revalidating in background
90
+ res.json(products);
91
+ });
92
+
93
+ app.get('/user/profile', authenticate, (req, res) => {
94
+ res.setHeader('Cache-Control', 'private, max-age=60'); // user-specific, browser only
95
+ res.json(profile);
96
+ });
97
+
98
+ // In-memory LRU cache for single-process hot data
99
+ import LRU from 'lru-cache';
100
+ const configCache = new LRU<string, unknown>({ max: 500, ttl: 60_000 }); // 500 entries, 1min TTL
101
+
102
+ const getConfig = async (key: string) => {
103
+ if (configCache.has(key)) return configCache.get(key);
104
+ const value = await db.config.findUnique({ where: { key } });
105
+ configCache.set(key, value);
106
+ return value;
107
+ };
108
+ ```