@nicnocquee/dataqueue 1.25.0 → 1.26.0-beta.20260223202259

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/ai/build-docs-content.ts +96 -0
  2. package/ai/build-llms-full.ts +42 -0
  3. package/ai/docs-content.json +284 -0
  4. package/ai/rules/advanced.md +150 -0
  5. package/ai/rules/basic.md +159 -0
  6. package/ai/rules/react-dashboard.md +83 -0
  7. package/ai/skills/dataqueue-advanced/SKILL.md +370 -0
  8. package/ai/skills/dataqueue-core/SKILL.md +234 -0
  9. package/ai/skills/dataqueue-react/SKILL.md +189 -0
  10. package/dist/cli.cjs +1149 -14
  11. package/dist/cli.cjs.map +1 -1
  12. package/dist/cli.d.cts +66 -1
  13. package/dist/cli.d.ts +66 -1
  14. package/dist/cli.js +1146 -13
  15. package/dist/cli.js.map +1 -1
  16. package/dist/index.cjs +3236 -1237
  17. package/dist/index.cjs.map +1 -1
  18. package/dist/index.d.cts +697 -23
  19. package/dist/index.d.ts +697 -23
  20. package/dist/index.js +3235 -1238
  21. package/dist/index.js.map +1 -1
  22. package/dist/mcp-server.cjs +186 -0
  23. package/dist/mcp-server.cjs.map +1 -0
  24. package/dist/mcp-server.d.cts +32 -0
  25. package/dist/mcp-server.d.ts +32 -0
  26. package/dist/mcp-server.js +175 -0
  27. package/dist/mcp-server.js.map +1 -0
  28. package/migrations/1781200000004_create_cron_schedules_table.sql +33 -0
  29. package/migrations/1781200000005_add_retry_config_to_job_queue.sql +17 -0
  30. package/package.json +24 -21
  31. package/src/backend.ts +170 -5
  32. package/src/backends/postgres.ts +992 -63
  33. package/src/backends/redis-scripts.ts +358 -26
  34. package/src/backends/redis.test.ts +1532 -0
  35. package/src/backends/redis.ts +993 -35
  36. package/src/cli.test.ts +82 -6
  37. package/src/cli.ts +73 -10
  38. package/src/cron.test.ts +126 -0
  39. package/src/cron.ts +40 -0
  40. package/src/db-util.ts +1 -1
  41. package/src/index.test.ts +1034 -11
  42. package/src/index.ts +267 -39
  43. package/src/init-command.test.ts +449 -0
  44. package/src/init-command.ts +709 -0
  45. package/src/install-mcp-command.test.ts +216 -0
  46. package/src/install-mcp-command.ts +185 -0
  47. package/src/install-rules-command.test.ts +218 -0
  48. package/src/install-rules-command.ts +233 -0
  49. package/src/install-skills-command.test.ts +176 -0
  50. package/src/install-skills-command.ts +124 -0
  51. package/src/mcp-server.test.ts +162 -0
  52. package/src/mcp-server.ts +231 -0
  53. package/src/processor.ts +104 -113
  54. package/src/queue.test.ts +465 -0
  55. package/src/queue.ts +34 -252
  56. package/src/supervisor.test.ts +340 -0
  57. package/src/supervisor.ts +177 -0
  58. package/src/types.ts +476 -12
  59. package/LICENSE +0 -21
package/src/cli.test.ts CHANGED
@@ -22,6 +22,11 @@ function makeDeps() {
22
22
  exit: vi.fn(),
23
23
  spawnSyncImpl: vi.fn(() => makeSpawnSyncReturns(0)),
24
24
  migrationsDir: '/migrations',
25
+ runInitImpl: vi.fn(),
26
+ runInstallSkillsImpl: vi.fn(),
27
+ runInstallRulesImpl: vi.fn(async () => {}),
28
+ runInstallMcpImpl: vi.fn(async () => {}),
29
+ startMcpServerImpl: vi.fn(async () => ({}) as any),
25
30
  } satisfies CliDeps;
26
31
  }
27
32
 
@@ -34,20 +39,30 @@ describe('runCli', () => {
34
39
 
35
40
  it('prints usage and exits with code 1 for no command', () => {
36
41
  runCli(['node', 'cli.js'], deps);
37
- expect(deps.log).toHaveBeenCalledWith(
38
- 'Usage: dataqueue-cli migrate [--envPath <path>] [-s <schema> | --schema <schema>]',
39
- );
42
+ expect(deps.log).toHaveBeenCalledWith('Usage:');
43
+ expect(deps.log).toHaveBeenCalledWith(' dataqueue-cli init');
40
44
  expect(deps.exit).toHaveBeenCalledWith(1);
41
45
  });
42
46
 
43
47
  it('prints usage and exits with code 1 for unknown command', () => {
44
48
  runCli(['node', 'cli.js', 'unknown'], deps);
45
- expect(deps.log).toHaveBeenCalledWith(
46
- 'Usage: dataqueue-cli migrate [--envPath <path>] [-s <schema> | --schema <schema>]',
47
- );
49
+ expect(deps.log).toHaveBeenCalledWith('Usage:');
50
+ expect(deps.log).toHaveBeenCalledWith(' dataqueue-cli init');
48
51
  expect(deps.exit).toHaveBeenCalledWith(1);
49
52
  });
50
53
 
54
+ it('routes init command to runInitImpl', () => {
55
+ runCli(['node', 'cli.js', 'init'], deps);
56
+ expect(deps.runInitImpl).toHaveBeenCalledWith(
57
+ expect.objectContaining({
58
+ log: deps.log,
59
+ error: deps.error,
60
+ exit: deps.exit,
61
+ }),
62
+ );
63
+ expect(deps.spawnSyncImpl).not.toHaveBeenCalled();
64
+ });
65
+
51
66
  it('calls spawnSyncImpl with correct args for migrate', () => {
52
67
  runCli(['node', 'cli.js', 'migrate'], deps);
53
68
  expect(deps.spawnSyncImpl).toHaveBeenCalledWith(
@@ -127,4 +142,65 @@ describe('runCli', () => {
127
142
  runCli(['node', 'cli.js', 'migrate'], deps);
128
143
  expect(deps.exit).toHaveBeenCalledWith(1);
129
144
  });
145
+
146
+ it('routes install-skills command to runInstallSkillsImpl', () => {
147
+ // Act
148
+ runCli(['node', 'cli.js', 'install-skills'], deps);
149
+
150
+ // Assert
151
+ expect(deps.runInstallSkillsImpl).toHaveBeenCalledWith(
152
+ expect.objectContaining({
153
+ log: deps.log,
154
+ error: deps.error,
155
+ exit: deps.exit,
156
+ }),
157
+ );
158
+ });
159
+
160
+ it('routes install-rules command to runInstallRulesImpl', () => {
161
+ // Act
162
+ runCli(['node', 'cli.js', 'install-rules'], deps);
163
+
164
+ // Assert
165
+ expect(deps.runInstallRulesImpl).toHaveBeenCalledWith(
166
+ expect.objectContaining({
167
+ log: deps.log,
168
+ error: deps.error,
169
+ exit: deps.exit,
170
+ }),
171
+ );
172
+ });
173
+
174
+ it('routes install-mcp command to runInstallMcpImpl', () => {
175
+ // Act
176
+ runCli(['node', 'cli.js', 'install-mcp'], deps);
177
+
178
+ // Assert
179
+ expect(deps.runInstallMcpImpl).toHaveBeenCalledWith(
180
+ expect.objectContaining({
181
+ log: deps.log,
182
+ error: deps.error,
183
+ exit: deps.exit,
184
+ }),
185
+ );
186
+ });
187
+
188
+ it('routes mcp command to startMcpServerImpl', () => {
189
+ // Act
190
+ runCli(['node', 'cli.js', 'mcp'], deps);
191
+
192
+ // Assert
193
+ expect(deps.startMcpServerImpl).toHaveBeenCalled();
194
+ });
195
+
196
+ it('shows new commands in usage output', () => {
197
+ // Act
198
+ runCli(['node', 'cli.js'], deps);
199
+
200
+ // Assert
201
+ expect(deps.log).toHaveBeenCalledWith(' dataqueue-cli install-skills');
202
+ expect(deps.log).toHaveBeenCalledWith(' dataqueue-cli install-rules');
203
+ expect(deps.log).toHaveBeenCalledWith(' dataqueue-cli install-mcp');
204
+ expect(deps.log).toHaveBeenCalledWith(' dataqueue-cli mcp');
205
+ });
130
206
  });
package/src/cli.ts CHANGED
@@ -2,6 +2,14 @@
2
2
  import { spawnSync, SpawnSyncReturns } from 'child_process';
3
3
  import path from 'path';
4
4
  import { fileURLToPath } from 'url';
5
+ import { InitDeps, runInit } from './init-command.js';
6
+ import {
7
+ runInstallSkills,
8
+ InstallSkillsDeps,
9
+ } from './install-skills-command.js';
10
+ import { runInstallRules, InstallRulesDeps } from './install-rules-command.js';
11
+ import { runInstallMcp, InstallMcpDeps } from './install-mcp-command.js';
12
+ import { startMcpServer } from './mcp-server.js';
5
13
 
6
14
  const __filename = fileURLToPath(import.meta.url);
7
15
  const __dirname = path.dirname(__filename);
@@ -12,25 +20,50 @@ export interface CliDeps {
12
20
  exit?: (code: number) => void;
13
21
  spawnSyncImpl?: (...args: any[]) => SpawnSyncReturns<any>;
14
22
  migrationsDir?: string;
23
+ initDeps?: InitDeps;
24
+ runInitImpl?: (deps?: InitDeps) => void;
25
+ installSkillsDeps?: InstallSkillsDeps;
26
+ runInstallSkillsImpl?: (deps?: InstallSkillsDeps) => void;
27
+ installRulesDeps?: InstallRulesDeps;
28
+ runInstallRulesImpl?: (deps?: InstallRulesDeps) => Promise<void>;
29
+ installMcpDeps?: InstallMcpDeps;
30
+ runInstallMcpImpl?: (deps?: InstallMcpDeps) => Promise<void>;
31
+ startMcpServerImpl?: typeof startMcpServer;
15
32
  }
16
33
 
17
34
  export function runCli(
18
35
  argv: string[],
19
36
  {
20
37
  log = console.log,
38
+ error = console.error,
21
39
  exit = (code: number) => process.exit(code),
22
40
  spawnSyncImpl = spawnSync,
23
41
  migrationsDir = path.join(__dirname, '../migrations'),
42
+ initDeps,
43
+ runInitImpl = runInit,
44
+ installSkillsDeps,
45
+ runInstallSkillsImpl = runInstallSkills,
46
+ installRulesDeps,
47
+ runInstallRulesImpl = runInstallRules,
48
+ installMcpDeps,
49
+ runInstallMcpImpl = runInstallMcp,
50
+ startMcpServerImpl = startMcpServer,
24
51
  }: CliDeps = {},
25
52
  ): void {
26
53
  const [, , command, ...restArgs] = argv;
27
54
 
28
55
  function printUsage() {
56
+ log('Usage:');
29
57
  log(
30
- 'Usage: dataqueue-cli migrate [--envPath <path>] [-s <schema> | --schema <schema>]',
58
+ ' dataqueue-cli migrate [--envPath <path>] [-s <schema> | --schema <schema>]',
31
59
  );
60
+ log(' dataqueue-cli init');
61
+ log(' dataqueue-cli install-skills');
62
+ log(' dataqueue-cli install-rules');
63
+ log(' dataqueue-cli install-mcp');
64
+ log(' dataqueue-cli mcp');
32
65
  log('');
33
- log('Options:');
66
+ log('Options for migrate:');
34
67
  log(
35
68
  ' --envPath <path> Path to a .env file to load environment variables (passed to node-pg-migrate)',
36
69
  );
@@ -38,16 +71,13 @@ export function runCli(
38
71
  ' -s, --schema <schema> Set the schema to use (passed to node-pg-migrate)',
39
72
  );
40
73
  log('');
41
- log('Notes:');
74
+ log('AI tooling commands:');
75
+ log(' install-skills Install DataQueue skill files for AI assistants');
76
+ log(' install-rules Install DataQueue agent rules for AI clients');
42
77
  log(
43
- ' - The PG_DATAQUEUE_DATABASE environment variable must be set to your Postgres connection string.',
44
- );
45
- log(
46
- ' - For managed Postgres (e.g., DigitalOcean) with SSL, set PGSSLMODE=require and PGSSLROOTCERT to your CA .crt file.',
47
- );
48
- log(
49
- ' Example: PGSSLMODE=require NODE_EXTRA_CA_CERTS=/absolute/path/to/ca.crt PG_DATAQUEUE_DATABASE=... npx dataqueue-cli migrate',
78
+ ' install-mcp Configure the DataQueue MCP server for AI clients',
50
79
  );
80
+ log(' mcp Start the DataQueue MCP server (stdio)');
51
81
  exit(1);
52
82
  }
53
83
 
@@ -89,6 +119,39 @@ export function runCli(
89
119
  { stdio: 'inherit' },
90
120
  );
91
121
  exit(result.status ?? 1);
122
+ } else if (command === 'init') {
123
+ runInitImpl({
124
+ log,
125
+ error,
126
+ exit,
127
+ ...initDeps,
128
+ });
129
+ } else if (command === 'install-skills') {
130
+ runInstallSkillsImpl({
131
+ log,
132
+ error,
133
+ exit,
134
+ ...installSkillsDeps,
135
+ });
136
+ } else if (command === 'install-rules') {
137
+ runInstallRulesImpl({
138
+ log,
139
+ error,
140
+ exit,
141
+ ...installRulesDeps,
142
+ });
143
+ } else if (command === 'install-mcp') {
144
+ runInstallMcpImpl({
145
+ log,
146
+ error,
147
+ exit,
148
+ ...installMcpDeps,
149
+ });
150
+ } else if (command === 'mcp') {
151
+ startMcpServerImpl().catch((err) => {
152
+ error('Failed to start MCP server:', err);
153
+ exit(1);
154
+ });
92
155
  } else {
93
156
  printUsage();
94
157
  }
@@ -0,0 +1,126 @@
1
+ import { describe, it, expect, afterEach, vi } from 'vitest';
2
+ import { getNextCronOccurrence, validateCronExpression } from './cron.js';
3
+
4
+ describe('getNextCronOccurrence', () => {
5
+ afterEach(() => {
6
+ vi.restoreAllMocks();
7
+ });
8
+
9
+ it('returns the next occurrence for a every-5-minutes expression', () => {
10
+ // Setup
11
+ const after = new Date('2026-01-15T10:02:00Z');
12
+
13
+ // Act
14
+ const next = getNextCronOccurrence('*/5 * * * *', 'UTC', after);
15
+
16
+ // Assert
17
+ expect(next).toEqual(new Date('2026-01-15T10:05:00Z'));
18
+ });
19
+
20
+ it('returns the next occurrence for a daily-at-midnight expression', () => {
21
+ // Setup
22
+ const after = new Date('2026-01-15T10:00:00Z');
23
+
24
+ // Act
25
+ const next = getNextCronOccurrence('0 0 * * *', 'UTC', after);
26
+
27
+ // Assert
28
+ expect(next).toEqual(new Date('2026-01-16T00:00:00Z'));
29
+ });
30
+
31
+ it('uses the current time when after is not provided', () => {
32
+ // Act
33
+ const next = getNextCronOccurrence('*/5 * * * *');
34
+
35
+ // Assert
36
+ expect(next).toBeInstanceOf(Date);
37
+ expect(next!.getTime()).toBeGreaterThan(Date.now() - 1000);
38
+ });
39
+
40
+ it('respects a non-UTC timezone', () => {
41
+ // Setup — 10:02 UTC is 19:02 in Asia/Tokyo (UTC+9)
42
+ const after = new Date('2026-01-15T10:02:00Z');
43
+
44
+ // Act — "0 20 * * *" = daily at 20:00 Tokyo time = 11:00 UTC
45
+ const next = getNextCronOccurrence('0 20 * * *', 'Asia/Tokyo', after);
46
+
47
+ // Assert
48
+ expect(next).toEqual(new Date('2026-01-15T11:00:00Z'));
49
+ });
50
+
51
+ it('returns null when expression cannot produce a future match', () => {
52
+ // Setup — Feb 30 never exists: "0 0 30 2 *"
53
+ const after = new Date('2026-01-01T00:00:00Z');
54
+
55
+ // Act
56
+ const next = getNextCronOccurrence('0 0 30 2 *', 'UTC', after);
57
+
58
+ // Assert
59
+ expect(next).toBeNull();
60
+ });
61
+
62
+ it('defaults to UTC timezone', () => {
63
+ // Setup
64
+ const after = new Date('2026-06-01T23:58:00Z');
65
+
66
+ // Act
67
+ const next = getNextCronOccurrence('0 0 * * *', undefined, after);
68
+
69
+ // Assert
70
+ expect(next).toEqual(new Date('2026-06-02T00:00:00Z'));
71
+ });
72
+ });
73
+
74
+ describe('validateCronExpression', () => {
75
+ afterEach(() => {
76
+ vi.restoreAllMocks();
77
+ });
78
+
79
+ it('returns true for a valid every-minute expression', () => {
80
+ // Act
81
+ const result = validateCronExpression('* * * * *');
82
+
83
+ // Assert
84
+ expect(result).toBe(true);
85
+ });
86
+
87
+ it('returns true for a valid complex expression', () => {
88
+ // Act
89
+ const result = validateCronExpression('0 9-17 * * 1-5');
90
+
91
+ // Assert
92
+ expect(result).toBe(true);
93
+ });
94
+
95
+ it('returns false for an invalid expression with too few fields', () => {
96
+ // Act
97
+ const result = validateCronExpression('* *');
98
+
99
+ // Assert
100
+ expect(result).toBe(false);
101
+ });
102
+
103
+ it('returns false for an empty string', () => {
104
+ // Act
105
+ const result = validateCronExpression('');
106
+
107
+ // Assert
108
+ expect(result).toBe(false);
109
+ });
110
+
111
+ it('returns false for a completely invalid string', () => {
112
+ // Act
113
+ const result = validateCronExpression('not a cron expression');
114
+
115
+ // Assert
116
+ expect(result).toBe(false);
117
+ });
118
+
119
+ it('returns true for an expression with step values', () => {
120
+ // Act
121
+ const result = validateCronExpression('*/15 * * * *');
122
+
123
+ // Assert
124
+ expect(result).toBe(true);
125
+ });
126
+ });
package/src/cron.ts ADDED
@@ -0,0 +1,40 @@
1
+ import { Cron } from 'croner';
2
+
3
+ /**
4
+ * Calculate the next occurrence of a cron expression after a given date.
5
+ *
6
+ * @param cronExpression - A standard cron expression (5 fields, e.g. "0 * * * *").
7
+ * @param timezone - IANA timezone string (default: "UTC").
8
+ * @param after - The reference date to compute the next run from (default: now).
9
+ * @param CronImpl - Cron class for dependency injection (default: croner's Cron).
10
+ * @returns The next occurrence as a Date, or null if the expression will never fire again.
11
+ */
12
+ export function getNextCronOccurrence(
13
+ cronExpression: string,
14
+ timezone: string = 'UTC',
15
+ after?: Date,
16
+ CronImpl: typeof Cron = Cron,
17
+ ): Date | null {
18
+ const cron = new CronImpl(cronExpression, { timezone });
19
+ const next = cron.nextRun(after ?? new Date());
20
+ return next ?? null;
21
+ }
22
+
23
+ /**
24
+ * Validate whether a string is a syntactically correct cron expression.
25
+ *
26
+ * @param cronExpression - The cron expression to validate.
27
+ * @param CronImpl - Cron class for dependency injection (default: croner's Cron).
28
+ * @returns True if the expression is valid, false otherwise.
29
+ */
30
+ export function validateCronExpression(
31
+ cronExpression: string,
32
+ CronImpl: typeof Cron = Cron,
33
+ ): boolean {
34
+ try {
35
+ new CronImpl(cronExpression);
36
+ return true;
37
+ } catch {
38
+ return false;
39
+ }
40
+ }
package/src/db-util.ts CHANGED
@@ -27,7 +27,7 @@ function loadPemOrFile(value?: string): string | undefined {
27
27
  * }
28
28
  */
29
29
  export const createPool = (
30
- config: PostgresJobQueueConfig['databaseConfig'],
30
+ config: NonNullable<PostgresJobQueueConfig['databaseConfig']>,
31
31
  ): Pool => {
32
32
  let searchPath: string | undefined;
33
33
  let ssl: any = undefined;