@nicnocquee/dataqueue 1.33.0 → 1.35.0-beta.20260224075710

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/ai/build-docs-content.ts +96 -0
  2. package/ai/build-llms-full.ts +42 -0
  3. package/ai/docs-content.json +290 -0
  4. package/ai/rules/advanced.md +170 -0
  5. package/ai/rules/basic.md +159 -0
  6. package/ai/rules/react-dashboard.md +87 -0
  7. package/ai/skills/dataqueue-advanced/SKILL.md +370 -0
  8. package/ai/skills/dataqueue-core/SKILL.md +235 -0
  9. package/ai/skills/dataqueue-react/SKILL.md +201 -0
  10. package/dist/cli.cjs +577 -32
  11. package/dist/cli.cjs.map +1 -1
  12. package/dist/cli.d.cts +52 -2
  13. package/dist/cli.d.ts +52 -2
  14. package/dist/cli.js +575 -32
  15. package/dist/cli.js.map +1 -1
  16. package/dist/index.cjs +937 -108
  17. package/dist/index.cjs.map +1 -1
  18. package/dist/index.d.cts +358 -11
  19. package/dist/index.d.ts +358 -11
  20. package/dist/index.js +937 -108
  21. package/dist/index.js.map +1 -1
  22. package/dist/mcp-server.cjs +186 -0
  23. package/dist/mcp-server.cjs.map +1 -0
  24. package/dist/mcp-server.d.cts +32 -0
  25. package/dist/mcp-server.d.ts +32 -0
  26. package/dist/mcp-server.js +175 -0
  27. package/dist/mcp-server.js.map +1 -0
  28. package/migrations/1781200000005_add_retry_config_to_job_queue.sql +17 -0
  29. package/migrations/1781200000006_add_output_to_job_queue.sql +3 -0
  30. package/package.json +10 -4
  31. package/src/backend.ts +36 -3
  32. package/src/backends/postgres.ts +344 -42
  33. package/src/backends/redis-scripts.ts +173 -8
  34. package/src/backends/redis.test.ts +668 -0
  35. package/src/backends/redis.ts +244 -15
  36. package/src/cli.test.ts +65 -0
  37. package/src/cli.ts +56 -19
  38. package/src/db-util.ts +1 -1
  39. package/src/index.test.ts +811 -12
  40. package/src/index.ts +106 -14
  41. package/src/install-mcp-command.test.ts +216 -0
  42. package/src/install-mcp-command.ts +185 -0
  43. package/src/install-rules-command.test.ts +218 -0
  44. package/src/install-rules-command.ts +233 -0
  45. package/src/install-skills-command.test.ts +176 -0
  46. package/src/install-skills-command.ts +124 -0
  47. package/src/mcp-server.test.ts +162 -0
  48. package/src/mcp-server.ts +231 -0
  49. package/src/processor.ts +133 -49
  50. package/src/queue.test.ts +477 -0
  51. package/src/queue.ts +20 -3
  52. package/src/supervisor.test.ts +340 -0
  53. package/src/supervisor.ts +177 -0
  54. package/src/types.ts +318 -3
@@ -0,0 +1,201 @@
1
+ ---
2
+ name: dataqueue-react
3
+ description: React SDK and Dashboard patterns for DataQueue — useJob hook, DataqueueProvider, admin dashboard.
4
+ ---
5
+
6
+ # DataQueue React & Dashboard Patterns
7
+
8
+ ## React SDK — @nicnocquee/dataqueue-react
9
+
10
+ ### Installation
11
+
12
+ ```bash
13
+ npm install @nicnocquee/dataqueue-react
14
+ ```
15
+
16
+ Requires React 18+.
17
+
18
+ ### useJob Hook
19
+
20
+ Poll a job's status and progress from the browser.
21
+
22
+ ```tsx
23
+ 'use client';
24
+ import { useJob } from '@nicnocquee/dataqueue-react';
25
+
26
+ function JobTracker({ jobId }: { jobId: number }) {
27
+ const { status, progress, data, isLoading, error } = useJob(jobId, {
28
+ fetcher: (id) =>
29
+ fetch(`/api/jobs/${id}`)
30
+ .then((r) => r.json())
31
+ .then((d) => d.job),
32
+ pollingInterval: 1000,
33
+ onComplete: (job) => toast.success('Done!'),
34
+ onFailed: (job) => toast.error('Failed'),
35
+ onStatusChange: (newStatus, oldStatus) => {
36
+ console.log(`${oldStatus} → ${newStatus}`);
37
+ },
38
+ });
39
+
40
+ if (isLoading) return <p>Loading...</p>;
41
+ if (error) return <p>Error: {error.message}</p>;
42
+
43
+ return (
44
+ <div>
45
+ <p>Status: {status}</p>
46
+ <progress value={progress ?? 0} max={100} />
47
+ </div>
48
+ );
49
+ }
50
+ ```
51
+
52
+ Polling stops automatically on terminal statuses (`completed`, `failed`, `cancelled`).
53
+
54
+ ### DataqueueProvider
55
+
56
+ Avoid repeating `fetcher` and `pollingInterval` by wrapping your app in a provider.
57
+
58
+ ```tsx
59
+ 'use client';
60
+ import { DataqueueProvider } from '@nicnocquee/dataqueue-react';
61
+
62
+ const fetcher = (id: number) =>
63
+ fetch(`/api/jobs/${id}`)
64
+ .then((r) => r.json())
65
+ .then((d) => d.job);
66
+
67
+ export function Providers({ children }: { children: React.ReactNode }) {
68
+ return (
69
+ <DataqueueProvider fetcher={fetcher} pollingInterval={2000}>
70
+ {children}
71
+ </DataqueueProvider>
72
+ );
73
+ }
74
+ ```
75
+
76
+ Then use `useJob` without config:
77
+
78
+ ```tsx
79
+ const { status, progress } = useJob(jobId);
80
+ ```
81
+
82
+ ### API Route for Job Fetching (Next.js)
83
+
84
+ ```typescript
85
+ // app/api/jobs/[id]/route.ts
86
+ import { getJobQueue } from '@/lib/queue';
87
+ import { NextResponse } from 'next/server';
88
+
89
+ export async function GET(
90
+ _request: Request,
91
+ { params }: { params: Promise<{ id: string }> },
92
+ ) {
93
+ const { id } = await params;
94
+ const jobQueue = getJobQueue();
95
+ const job = await jobQueue.getJob(Number(id));
96
+ if (!job) {
97
+ return NextResponse.json({ error: 'Job not found' }, { status: 404 });
98
+ }
99
+ return NextResponse.json({ job });
100
+ }
101
+ ```
102
+
103
+ ### useJob Return Value
104
+
105
+ | Field | Type | Description |
106
+ | ----------- | ------------------- | ----------------------------------------------------- |
107
+ | `data` | `JobData \| null` | Latest job data from fetcher |
108
+ | `status` | `JobStatus \| null` | Current job status |
109
+ | `progress` | `number \| null` | Progress percentage (0–100) |
110
+ | `output` | `unknown \| null` | Handler output from `ctx.setOutput()` or return value |
111
+ | `isLoading` | `boolean` | True until first fetch resolves |
112
+ | `error` | `Error \| null` | Last fetch error |
113
+
114
+ ## Dashboard — @nicnocquee/dataqueue-dashboard
115
+
116
+ ### Installation
117
+
118
+ ```bash
119
+ npm install @nicnocquee/dataqueue-dashboard
120
+ ```
121
+
122
+ ### Setup (Next.js App Router)
123
+
124
+ Create a single catch-all route:
125
+
126
+ ```typescript
127
+ // app/admin/dataqueue/[[...path]]/route.ts
128
+ import { createDataqueueDashboard } from '@nicnocquee/dataqueue-dashboard/next';
129
+ import { getJobQueue, jobHandlers } from '@/lib/queue';
130
+
131
+ const { GET, POST } = createDataqueueDashboard({
132
+ jobQueue: getJobQueue(),
133
+ jobHandlers,
134
+ basePath: '/admin/dataqueue',
135
+ });
136
+
137
+ export { GET, POST };
138
+ ```
139
+
140
+ Visit `/admin/dataqueue` to open the dashboard.
141
+
142
+ The `basePath` must match the route file directory. For example, `app/jobs/dashboard/[[...path]]/route.ts` requires `basePath: '/jobs/dashboard'`.
143
+
144
+ ### Dashboard Features
145
+
146
+ - Jobs list with status filter tabs, pagination, auto-refresh
147
+ - Job detail view with payload, error history, step data, events timeline
148
+ - Inline actions: cancel pending/waiting jobs, retry failed/cancelled jobs
149
+ - Process Jobs button for one-shot processing (useful in dev)
150
+
151
+ ### Protecting the Dashboard
152
+
153
+ Wrap the handlers with your auth logic:
154
+
155
+ ```typescript
156
+ const dashboard = createDataqueueDashboard({
157
+ jobQueue: getJobQueue(),
158
+ jobHandlers,
159
+ basePath: '/admin/dataqueue',
160
+ });
161
+
162
+ export async function GET(req: Request, ctx: any) {
163
+ const session = await auth();
164
+ if (!session?.user?.isAdmin) {
165
+ return new Response('Unauthorized', { status: 401 });
166
+ }
167
+ return dashboard.GET(req, ctx);
168
+ }
169
+
170
+ export async function POST(req: Request, ctx: any) {
171
+ const session = await auth();
172
+ if (!session?.user?.isAdmin) {
173
+ return new Response('Unauthorized', { status: 401 });
174
+ }
175
+ return dashboard.POST(req, ctx);
176
+ }
177
+ ```
178
+
179
+ ### Progress Tracking from Handlers
180
+
181
+ Report progress via `ctx.setProgress(percent)` (0–100). The value persists to the database and is exposed via `getJob()` and the `useJob` hook's `progress` field.
182
+
183
+ ```typescript
184
+ const handler = async (payload, signal, ctx) => {
185
+ for (let i = 0; i < chunks.length; i++) {
186
+ await processChunk(chunks[i]);
187
+ await ctx.setProgress(Math.round(((i + 1) / chunks.length) * 100));
188
+ }
189
+ };
190
+ ```
191
+
192
+ ### Job Output from Handlers
193
+
194
+ Store results via `ctx.setOutput(data)` or by returning a value from the handler. Exposed via `getJob()` (`output` field) and the `useJob` hook's `output` property. If both are used, `ctx.setOutput()` takes precedence.
195
+
196
+ ```typescript
197
+ const handler = async (payload, signal, ctx) => {
198
+ const result = await doWork(payload);
199
+ return { url: result.downloadUrl }; // stored as output
200
+ };
201
+ ```