@geekmidas/cli 1.11.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +155 -0
- package/dist/config.d.cts +2 -2
- package/dist/config.d.mts +2 -2
- package/dist/{index-Dx9Kk5bR.d.mts → index-Bw548Bta.d.mts} +2 -2
- package/dist/{index-Dx9Kk5bR.d.mts.map → index-Bw548Bta.d.mts.map} +1 -1
- package/dist/{index-Dz2a7xQU.d.cts → index-DTpkeFb8.d.cts} +2 -2
- package/dist/{index-Dz2a7xQU.d.cts.map → index-DTpkeFb8.d.cts.map} +1 -1
- package/dist/index.cjs +283 -31
- package/dist/index.cjs.map +1 -1
- package/dist/index.mjs +283 -31
- package/dist/index.mjs.map +1 -1
- package/dist/{openapi-CG1LBk7s.cjs → openapi-BzNOC7_i.cjs} +14 -2
- package/dist/openapi-BzNOC7_i.cjs.map +1 -0
- package/dist/{openapi-CtRVU6RD.mjs → openapi-Cfwi0fqF.mjs} +14 -2
- package/dist/openapi-Cfwi0fqF.mjs.map +1 -0
- package/dist/openapi.cjs +1 -1
- package/dist/openapi.d.cts +1 -1
- package/dist/openapi.d.mts +1 -1
- package/dist/openapi.mjs +1 -1
- package/dist/{types-CqfhdmRi.d.mts → types-B3GQUg3k.d.mts} +3 -1
- package/dist/{types-CqfhdmRi.d.mts.map → types-B3GQUg3k.d.mts.map} +1 -1
- package/dist/{types-DdHfUbxk.d.cts → types-BWQDiooe.d.cts} +3 -1
- package/dist/{types-DdHfUbxk.d.cts.map → types-BWQDiooe.d.cts.map} +1 -1
- package/dist/workspace/index.d.cts +2 -2
- package/dist/workspace/index.d.mts +2 -2
- package/package.json +8 -7
- package/src/build/__tests__/index-new.spec.ts +3 -1
- package/src/build/__tests__/manifests.spec.ts +2 -2
- package/src/build/index.ts +61 -21
- package/src/build/manifests.ts +62 -5
- package/src/dev/index.ts +25 -11
- package/src/generators/EndpointGenerator.ts +14 -2
- package/src/generators/QueueGenerator.ts +245 -0
- package/src/generators/SubscriberGenerator.ts +5 -0
- package/src/generators/TopicGenerator.ts +45 -0
- package/src/generators/__tests__/QueueGenerator.spec.ts +141 -0
- package/src/generators/index.ts +2 -0
- package/src/init/generators/test.ts +2 -1
- package/src/init/versions.ts +9 -9
- package/src/types.ts +14 -51
- package/dist/openapi-CG1LBk7s.cjs.map +0 -1
- package/dist/openapi-CtRVU6RD.mjs.map +0 -1
package/src/build/index.ts
CHANGED
|
@@ -5,7 +5,9 @@ import { join, relative, resolve } from 'node:path';
|
|
|
5
5
|
import type { Cron } from '@geekmidas/constructs/crons';
|
|
6
6
|
import type { Endpoint } from '@geekmidas/constructs/endpoints';
|
|
7
7
|
import type { Function } from '@geekmidas/constructs/functions';
|
|
8
|
+
import type { Queue } from '@geekmidas/constructs/queue';
|
|
8
9
|
import type { Subscriber } from '@geekmidas/constructs/subscribers';
|
|
10
|
+
import type { Topic } from '@geekmidas/constructs/topic';
|
|
9
11
|
import {
|
|
10
12
|
loadAppConfig,
|
|
11
13
|
loadConfig,
|
|
@@ -24,7 +26,9 @@ import {
|
|
|
24
26
|
EndpointGenerator,
|
|
25
27
|
FunctionGenerator,
|
|
26
28
|
type GeneratedConstruct,
|
|
29
|
+
QueueGenerator,
|
|
27
30
|
SubscriberGenerator,
|
|
31
|
+
TopicGenerator,
|
|
28
32
|
} from '../generators';
|
|
29
33
|
import { generateOpenApi, openapiCommand } from '../openapi.js';
|
|
30
34
|
import {
|
|
@@ -166,29 +170,43 @@ export async function buildCommand(
|
|
|
166
170
|
const functionGenerator = new FunctionGenerator();
|
|
167
171
|
const cronGenerator = new CronGenerator();
|
|
168
172
|
const subscriberGenerator = new SubscriberGenerator();
|
|
173
|
+
const queueGenerator = new QueueGenerator();
|
|
174
|
+
const topicGenerator = new TopicGenerator();
|
|
169
175
|
|
|
170
176
|
// Load all constructs in parallel
|
|
171
|
-
const [
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
177
|
+
const [
|
|
178
|
+
allEndpoints,
|
|
179
|
+
allFunctions,
|
|
180
|
+
allCrons,
|
|
181
|
+
allSubscribers,
|
|
182
|
+
allQueues,
|
|
183
|
+
allTopics,
|
|
184
|
+
] = await Promise.all([
|
|
185
|
+
endpointGenerator.load(config.routes),
|
|
186
|
+
config.functions ? functionGenerator.load(config.functions) : [],
|
|
187
|
+
config.crons ? cronGenerator.load(config.crons) : [],
|
|
188
|
+
config.subscribers ? subscriberGenerator.load(config.subscribers) : [],
|
|
189
|
+
config.queues ? queueGenerator.load(config.queues) : [],
|
|
190
|
+
config.topics ? topicGenerator.load(config.topics) : [],
|
|
191
|
+
]);
|
|
178
192
|
|
|
179
193
|
logger.log(`Found ${allEndpoints.length} endpoints`);
|
|
180
194
|
logger.log(`Found ${allFunctions.length} functions`);
|
|
181
195
|
logger.log(`Found ${allCrons.length} crons`);
|
|
182
196
|
logger.log(`Found ${allSubscribers.length} subscribers`);
|
|
197
|
+
logger.log(`Found ${allQueues.length} queues`);
|
|
198
|
+
logger.log(`Found ${allTopics.length} topics`);
|
|
183
199
|
|
|
184
200
|
if (
|
|
185
201
|
allEndpoints.length === 0 &&
|
|
186
202
|
allFunctions.length === 0 &&
|
|
187
203
|
allCrons.length === 0 &&
|
|
188
|
-
allSubscribers.length === 0
|
|
204
|
+
allSubscribers.length === 0 &&
|
|
205
|
+
allQueues.length === 0 &&
|
|
206
|
+
allTopics.length === 0
|
|
189
207
|
) {
|
|
190
208
|
logger.log(
|
|
191
|
-
'No endpoints, functions, crons, or
|
|
209
|
+
'No endpoints, functions, crons, subscribers, queues, or topics found to process',
|
|
192
210
|
);
|
|
193
211
|
return {};
|
|
194
212
|
}
|
|
@@ -208,10 +226,14 @@ export async function buildCommand(
|
|
|
208
226
|
functionGenerator,
|
|
209
227
|
cronGenerator,
|
|
210
228
|
subscriberGenerator,
|
|
229
|
+
queueGenerator,
|
|
230
|
+
topicGenerator,
|
|
211
231
|
allEndpoints,
|
|
212
232
|
allFunctions,
|
|
213
233
|
allCrons,
|
|
214
234
|
allSubscribers,
|
|
235
|
+
allQueues,
|
|
236
|
+
allTopics,
|
|
215
237
|
resolved.enableOpenApi,
|
|
216
238
|
options.skipBundle ?? false,
|
|
217
239
|
options.stage,
|
|
@@ -236,10 +258,14 @@ async function buildForProvider(
|
|
|
236
258
|
functionGenerator: FunctionGenerator,
|
|
237
259
|
cronGenerator: CronGenerator,
|
|
238
260
|
subscriberGenerator: SubscriberGenerator,
|
|
261
|
+
queueGenerator: QueueGenerator,
|
|
262
|
+
topicGenerator: TopicGenerator,
|
|
239
263
|
endpoints: GeneratedConstruct<Endpoint<any, any, any, any, any, any>>[],
|
|
240
264
|
functions: GeneratedConstruct<Function<any, any, any, any>>[],
|
|
241
265
|
crons: GeneratedConstruct<Cron<any, any, any, any>>[],
|
|
242
266
|
subscribers: GeneratedConstruct<Subscriber<any, any, any, any, any, any>>[],
|
|
267
|
+
queues: GeneratedConstruct<Queue<any, any, any, any>>[],
|
|
268
|
+
topics: GeneratedConstruct<Topic<any, any>>[],
|
|
243
269
|
enableOpenApi: boolean,
|
|
244
270
|
skipBundle: boolean,
|
|
245
271
|
stage?: string,
|
|
@@ -254,26 +280,35 @@ async function buildForProvider(
|
|
|
254
280
|
// Build all constructs in parallel.
|
|
255
281
|
// context.markOptional is forwarded to each generator so that
|
|
256
282
|
// getEnvironment({ markOptional }) produces `VARNAME?` for optional vars.
|
|
257
|
-
const [
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
283
|
+
const [
|
|
284
|
+
routes,
|
|
285
|
+
functionInfos,
|
|
286
|
+
cronInfos,
|
|
287
|
+
subscriberInfos,
|
|
288
|
+
queueInfos,
|
|
289
|
+
topicInfos,
|
|
290
|
+
] = await Promise.all([
|
|
291
|
+
endpointGenerator.build(context, endpoints, outputDir, {
|
|
292
|
+
provider,
|
|
293
|
+
enableOpenApi,
|
|
294
|
+
}),
|
|
295
|
+
functionGenerator.build(context, functions, outputDir, { provider }),
|
|
296
|
+
cronGenerator.build(context, crons, outputDir, { provider }),
|
|
297
|
+
subscriberGenerator.build(context, subscribers, outputDir, { provider }),
|
|
298
|
+
queueGenerator.build(context, queues, outputDir, { provider }),
|
|
299
|
+
topicGenerator.build(context, topics, outputDir, { provider }),
|
|
300
|
+
]);
|
|
268
301
|
|
|
269
302
|
logger.log(
|
|
270
|
-
`Generated ${routes.length} routes, ${functionInfos.length} functions, ${cronInfos.length} crons, ${subscriberInfos.length} subscribers for ${provider}`,
|
|
303
|
+
`Generated ${routes.length} routes, ${functionInfos.length} functions, ${cronInfos.length} crons, ${subscriberInfos.length} subscribers, ${queueInfos.length} queues, ${topicInfos.length} topics for ${provider}`,
|
|
271
304
|
);
|
|
272
305
|
|
|
273
306
|
// Assemble manifest fields (flat or partitioned per construct type)
|
|
274
307
|
const manifestRoutes = assembleManifestField(routes, endpoints);
|
|
275
308
|
const manifestFunctions = assembleManifestField(functionInfos, functions);
|
|
276
309
|
const manifestCrons = assembleManifestField(cronInfos, crons);
|
|
310
|
+
const manifestQueues = assembleManifestField(queueInfos, queues);
|
|
311
|
+
const manifestTopics = assembleManifestField(topicInfos, topics);
|
|
277
312
|
const manifestSubscribers = assembleManifestField(
|
|
278
313
|
subscriberInfos,
|
|
279
314
|
subscribers,
|
|
@@ -303,6 +338,8 @@ async function buildForProvider(
|
|
|
303
338
|
appInfo,
|
|
304
339
|
serverRouteField,
|
|
305
340
|
manifestSubscribers,
|
|
341
|
+
manifestQueues,
|
|
342
|
+
manifestTopics,
|
|
306
343
|
);
|
|
307
344
|
|
|
308
345
|
// Bundle for production if enabled
|
|
@@ -317,6 +354,7 @@ async function buildForProvider(
|
|
|
317
354
|
...functions.map((f) => f.construct),
|
|
318
355
|
...crons.map((c) => c.construct),
|
|
319
356
|
...subscribers.map((s) => s.construct),
|
|
357
|
+
...queues.map((q) => q.construct),
|
|
320
358
|
];
|
|
321
359
|
|
|
322
360
|
// Get docker compose services for auto-populating env vars
|
|
@@ -351,6 +389,8 @@ async function buildForProvider(
|
|
|
351
389
|
manifestFunctions,
|
|
352
390
|
manifestCrons,
|
|
353
391
|
manifestSubscribers,
|
|
392
|
+
manifestQueues,
|
|
393
|
+
manifestTopics,
|
|
354
394
|
);
|
|
355
395
|
}
|
|
356
396
|
|
package/src/build/manifests.ts
CHANGED
|
@@ -3,8 +3,10 @@ import { join, relative } from 'node:path';
|
|
|
3
3
|
import type {
|
|
4
4
|
CronInfo,
|
|
5
5
|
FunctionInfo,
|
|
6
|
+
QueueInfo,
|
|
6
7
|
RouteInfo,
|
|
7
8
|
SubscriberInfo,
|
|
9
|
+
TopicInfo,
|
|
8
10
|
} from '../types';
|
|
9
11
|
|
|
10
12
|
const logger = console;
|
|
@@ -81,6 +83,8 @@ export async function generateAwsManifest(
|
|
|
81
83
|
functions: ManifestField<FunctionInfo>,
|
|
82
84
|
crons: ManifestField<CronInfo>,
|
|
83
85
|
subscribers: ManifestField<SubscriberInfo>,
|
|
86
|
+
queues: ManifestField<QueueInfo> = [],
|
|
87
|
+
topics: ManifestField<TopicInfo> = [],
|
|
84
88
|
): Promise<void> {
|
|
85
89
|
const manifestDir = join(outputDir, 'manifest');
|
|
86
90
|
await mkdir(manifestDir, { recursive: true });
|
|
@@ -92,12 +96,16 @@ export async function generateAwsManifest(
|
|
|
92
96
|
const functionsPartitioned = isPartitioned(functions);
|
|
93
97
|
const cronsPartitioned = isPartitioned(crons);
|
|
94
98
|
const subscribersPartitioned = isPartitioned(subscribers);
|
|
99
|
+
const queuesPartitioned = isPartitioned(queues);
|
|
100
|
+
const topicsPartitioned = isPartitioned(topics);
|
|
95
101
|
|
|
96
102
|
const content = `export const manifest = {
|
|
97
103
|
routes: ${serializeField(awsRoutes)},
|
|
98
104
|
functions: ${serializeField(functions)},
|
|
99
105
|
crons: ${serializeField(crons)},
|
|
100
106
|
subscribers: ${serializeField(subscribers)},
|
|
107
|
+
queues: ${serializeField(queues)},
|
|
108
|
+
topics: ${serializeField(topics)},
|
|
101
109
|
} as const;
|
|
102
110
|
|
|
103
111
|
// Derived types
|
|
@@ -105,6 +113,8 @@ ${generateDerivedType('routes', 'Route', routesPartitioned)}
|
|
|
105
113
|
${generateDerivedType('functions', 'Function', functionsPartitioned)}
|
|
106
114
|
${generateDerivedType('crons', 'Cron', cronsPartitioned)}
|
|
107
115
|
${generateDerivedType('subscribers', 'Subscriber', subscribersPartitioned)}
|
|
116
|
+
${generateDerivedType('queues', 'Queue', queuesPartitioned)}
|
|
117
|
+
${generateDerivedType('topics', 'Topic', topicsPartitioned)}
|
|
108
118
|
|
|
109
119
|
// Useful union types
|
|
110
120
|
export type Authorizer = Route['authorizer'];
|
|
@@ -116,7 +126,7 @@ export type RoutePath = Route['path'];
|
|
|
116
126
|
await writeFile(manifestPath, content);
|
|
117
127
|
|
|
118
128
|
logger.log(
|
|
119
|
-
`Generated AWS manifest with ${countItems(awsRoutes)} routes, ${countItems(functions)} functions, ${countItems(crons)} crons, ${countItems(subscribers)} subscribers`,
|
|
129
|
+
`Generated AWS manifest with ${countItems(awsRoutes)} routes, ${countItems(functions)} functions, ${countItems(crons)} crons, ${countItems(subscribers)} subscribers, ${countItems(queues)} queues, ${countItems(topics)} topics`,
|
|
120
130
|
);
|
|
121
131
|
logger.log(`Manifest: ${relative(process.cwd(), manifestPath)}`);
|
|
122
132
|
}
|
|
@@ -126,6 +136,8 @@ export async function generateServerManifest(
|
|
|
126
136
|
appInfo: ServerAppInfo,
|
|
127
137
|
routes: ManifestField<RouteInfo>,
|
|
128
138
|
subscribers: ManifestField<SubscriberInfo>,
|
|
139
|
+
queues: ManifestField<QueueInfo> = [],
|
|
140
|
+
topics: ManifestField<TopicInfo> = [],
|
|
129
141
|
): Promise<void> {
|
|
130
142
|
const manifestDir = join(outputDir, 'manifest');
|
|
131
143
|
await mkdir(manifestDir, { recursive: true });
|
|
@@ -136,18 +148,30 @@ export async function generateServerManifest(
|
|
|
136
148
|
// Server subscribers only need name and events
|
|
137
149
|
const serverSubscribers = mapSubscriberMetadata(subscribers);
|
|
138
150
|
|
|
151
|
+
// Server queues only need their name (the worker is wired by setupQueues)
|
|
152
|
+
const serverQueues = mapQueueMetadata(queues);
|
|
153
|
+
|
|
154
|
+
// Server topics only need name + events (pg-boss routes by event name locally)
|
|
155
|
+
const serverTopics = mapTopicMetadata(topics);
|
|
156
|
+
|
|
139
157
|
const routesPartitioned = isPartitioned(serverRoutes);
|
|
140
158
|
const subscribersPartitioned = isPartitioned(serverSubscribers);
|
|
159
|
+
const queuesPartitioned = isPartitioned(serverQueues);
|
|
160
|
+
const topicsPartitioned = isPartitioned(serverTopics);
|
|
141
161
|
|
|
142
162
|
const content = `export const manifest = {
|
|
143
163
|
app: ${JSON.stringify(appInfo, null, 2)},
|
|
144
164
|
routes: ${serializeField(serverRoutes)},
|
|
145
165
|
subscribers: ${serializeField(serverSubscribers)},
|
|
166
|
+
queues: ${serializeField(serverQueues)},
|
|
167
|
+
topics: ${serializeField(serverTopics)},
|
|
146
168
|
} as const;
|
|
147
169
|
|
|
148
170
|
// Derived types
|
|
149
171
|
${generateDerivedType('routes', 'Route', routesPartitioned)}
|
|
150
172
|
${generateDerivedType('subscribers', 'Subscriber', subscribersPartitioned)}
|
|
173
|
+
${generateDerivedType('queues', 'Queue', queuesPartitioned)}
|
|
174
|
+
${generateDerivedType('topics', 'Topic', topicsPartitioned)}
|
|
151
175
|
|
|
152
176
|
// Useful union types
|
|
153
177
|
export type Authorizer = Route['authorizer'];
|
|
@@ -159,7 +183,7 @@ export type RoutePath = Route['path'];
|
|
|
159
183
|
await writeFile(manifestPath, content);
|
|
160
184
|
|
|
161
185
|
logger.log(
|
|
162
|
-
`Generated server manifest with ${countItems(serverRoutes)} routes, ${countItems(serverSubscribers)} subscribers`,
|
|
186
|
+
`Generated server manifest with ${countItems(serverRoutes)} routes, ${countItems(serverSubscribers)} subscribers, ${countItems(serverQueues)} queues, ${countItems(serverTopics)} topics`,
|
|
163
187
|
);
|
|
164
188
|
logger.log(`Manifest: ${relative(process.cwd(), manifestPath)}`);
|
|
165
189
|
}
|
|
@@ -210,7 +234,7 @@ function mapRouteMetadata(
|
|
|
210
234
|
*/
|
|
211
235
|
function mapSubscriberMetadata(
|
|
212
236
|
subscribers: ManifestField<SubscriberInfo>,
|
|
213
|
-
): ManifestField<{ name: string; subscribedEvents: string[] }> {
|
|
237
|
+
): ManifestField<{ name: string; subscribedEvents: readonly string[] }> {
|
|
214
238
|
const mapFn = (s: SubscriberInfo) => ({
|
|
215
239
|
name: s.name,
|
|
216
240
|
subscribedEvents: s.subscribedEvents,
|
|
@@ -219,10 +243,43 @@ function mapSubscriberMetadata(
|
|
|
219
243
|
if (Array.isArray(subscribers)) {
|
|
220
244
|
return subscribers.map(mapFn);
|
|
221
245
|
}
|
|
222
|
-
const result: Record<
|
|
223
|
-
|
|
246
|
+
const result: Record<
|
|
247
|
+
string,
|
|
248
|
+
{ name: string; subscribedEvents: readonly string[] }[]
|
|
249
|
+
> = {};
|
|
224
250
|
for (const [partition, partitionSubs] of Object.entries(subscribers)) {
|
|
225
251
|
result[partition] = partitionSubs.map(mapFn);
|
|
226
252
|
}
|
|
227
253
|
return result;
|
|
228
254
|
}
|
|
255
|
+
|
|
256
|
+
function mapQueueMetadata(
|
|
257
|
+
queues: ManifestField<QueueInfo>,
|
|
258
|
+
): ManifestField<{ name: string }> {
|
|
259
|
+
const mapFn = (q: QueueInfo) => ({ name: q.name });
|
|
260
|
+
|
|
261
|
+
if (Array.isArray(queues)) {
|
|
262
|
+
return queues.map(mapFn);
|
|
263
|
+
}
|
|
264
|
+
const result: Record<string, { name: string }[]> = {};
|
|
265
|
+
for (const [partition, partitionQueues] of Object.entries(queues)) {
|
|
266
|
+
result[partition] = partitionQueues.map(mapFn);
|
|
267
|
+
}
|
|
268
|
+
return result;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function mapTopicMetadata(
|
|
272
|
+
topics: ManifestField<TopicInfo>,
|
|
273
|
+
): ManifestField<{ name: string; events: readonly string[] }> {
|
|
274
|
+
const mapFn = (t: TopicInfo) => ({ name: t.name, events: t.events });
|
|
275
|
+
|
|
276
|
+
if (Array.isArray(topics)) {
|
|
277
|
+
return topics.map(mapFn);
|
|
278
|
+
}
|
|
279
|
+
const result: Record<string, { name: string; events: readonly string[] }[]> =
|
|
280
|
+
{};
|
|
281
|
+
for (const [partition, partitionTopics] of Object.entries(topics)) {
|
|
282
|
+
result[partition] = partitionTopics.map(mapFn);
|
|
283
|
+
}
|
|
284
|
+
return result;
|
|
285
|
+
}
|
package/src/dev/index.ts
CHANGED
|
@@ -33,7 +33,9 @@ import {
|
|
|
33
33
|
CronGenerator,
|
|
34
34
|
EndpointGenerator,
|
|
35
35
|
FunctionGenerator,
|
|
36
|
+
QueueGenerator,
|
|
36
37
|
SubscriberGenerator,
|
|
38
|
+
TopicGenerator,
|
|
37
39
|
} from '../generators';
|
|
38
40
|
import {
|
|
39
41
|
generateOpenApi,
|
|
@@ -1148,19 +1150,29 @@ async function buildServer(
|
|
|
1148
1150
|
const functionGenerator = new FunctionGenerator();
|
|
1149
1151
|
const cronGenerator = new CronGenerator();
|
|
1150
1152
|
const subscriberGenerator = new SubscriberGenerator();
|
|
1153
|
+
const queueGenerator = new QueueGenerator();
|
|
1154
|
+
const topicGenerator = new TopicGenerator();
|
|
1151
1155
|
|
|
1152
1156
|
// Load all constructs (resolve paths relative to appRoot)
|
|
1153
|
-
const [
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1157
|
+
const [
|
|
1158
|
+
allEndpoints,
|
|
1159
|
+
allFunctions,
|
|
1160
|
+
allCrons,
|
|
1161
|
+
allSubscribers,
|
|
1162
|
+
allQueues,
|
|
1163
|
+
allTopics,
|
|
1164
|
+
] = await Promise.all([
|
|
1165
|
+
endpointGenerator.load(config.routes, appRoot, bustCache),
|
|
1166
|
+
config.functions
|
|
1167
|
+
? functionGenerator.load(config.functions, appRoot, bustCache)
|
|
1168
|
+
: [],
|
|
1169
|
+
config.crons ? cronGenerator.load(config.crons, appRoot, bustCache) : [],
|
|
1170
|
+
config.subscribers
|
|
1171
|
+
? subscriberGenerator.load(config.subscribers, appRoot, bustCache)
|
|
1172
|
+
: [],
|
|
1173
|
+
config.queues ? queueGenerator.load(config.queues, appRoot, bustCache) : [],
|
|
1174
|
+
config.topics ? topicGenerator.load(config.topics, appRoot, bustCache) : [],
|
|
1175
|
+
]);
|
|
1164
1176
|
|
|
1165
1177
|
// Ensure .gkm directory exists in app root
|
|
1166
1178
|
const outputDir = join(appRoot, '.gkm', provider);
|
|
@@ -1175,6 +1187,8 @@ async function buildServer(
|
|
|
1175
1187
|
functionGenerator.build(context, allFunctions, outputDir, { provider }),
|
|
1176
1188
|
cronGenerator.build(context, allCrons, outputDir, { provider }),
|
|
1177
1189
|
subscriberGenerator.build(context, allSubscribers, outputDir, { provider }),
|
|
1190
|
+
queueGenerator.build(context, allQueues, outputDir, { provider }),
|
|
1191
|
+
topicGenerator.build(context, allTopics, outputDir, { provider }),
|
|
1178
1192
|
]);
|
|
1179
1193
|
}
|
|
1180
1194
|
|
|
@@ -618,6 +618,7 @@ import { Hono } from 'hono';
|
|
|
618
618
|
import type { Hono as HonoType } from 'hono';
|
|
619
619
|
import { setupEndpoints } from './endpoints.js';
|
|
620
620
|
import { setupSubscribers } from './subscribers.js';
|
|
621
|
+
import { setupQueues } from './queues.js';
|
|
621
622
|
import ${context.envParserImportPattern} from '${relativeEnvParserPath}';
|
|
622
623
|
import ${context.loggerImportPattern} from '${relativeLoggerPath}';
|
|
623
624
|
${telescopeImports}
|
|
@@ -691,6 +692,11 @@ ${afterSetupCall}
|
|
|
691
692
|
logger.error({ error }, 'Failed to start subscribers');
|
|
692
693
|
});
|
|
693
694
|
|
|
695
|
+
// Start queue workers in background (non-blocking, local development only)
|
|
696
|
+
await setupQueues(envParser, logger).catch((error) => {
|
|
697
|
+
logger.error({ error }, 'Failed to start queue workers');
|
|
698
|
+
});
|
|
699
|
+
|
|
694
700
|
logger.info({ port }, 'Starting server');
|
|
695
701
|
|
|
696
702
|
// Start HTTP server using provided serve function
|
|
@@ -800,18 +806,24 @@ export const handler = ${exportName};
|
|
|
800
806
|
`;
|
|
801
807
|
}
|
|
802
808
|
|
|
803
|
-
// Subscriber setup code
|
|
809
|
+
// Subscriber + queue setup code (background workers share the same flag)
|
|
804
810
|
const subscriberSetup = includeSubscribers
|
|
805
811
|
? `
|
|
806
812
|
// Start subscribers in background
|
|
807
813
|
await setupSubscribers(envParser, logger).catch((error) => {
|
|
808
814
|
logger.error({ error }, 'Failed to start subscribers');
|
|
809
815
|
});
|
|
816
|
+
|
|
817
|
+
// Start queue workers in background
|
|
818
|
+
await setupQueues(envParser, logger).catch((error) => {
|
|
819
|
+
logger.error({ error }, 'Failed to start queue workers');
|
|
820
|
+
});
|
|
810
821
|
`
|
|
811
822
|
: '';
|
|
812
823
|
|
|
813
824
|
const subscriberImport = includeSubscribers
|
|
814
|
-
? `import { setupSubscribers } from './subscribers.js'
|
|
825
|
+
? `import { setupSubscribers } from './subscribers.js';
|
|
826
|
+
import { setupQueues } from './queues.js';`
|
|
815
827
|
: '';
|
|
816
828
|
|
|
817
829
|
// Graceful shutdown code
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { dirname, join, relative } from 'node:path';
|
|
3
|
+
import { Queue } from '@geekmidas/constructs/queue';
|
|
4
|
+
import type { BuildContext } from '../build/types';
|
|
5
|
+
import type { QueueInfo } from '../types';
|
|
6
|
+
import {
|
|
7
|
+
ConstructGenerator,
|
|
8
|
+
type GeneratedConstruct,
|
|
9
|
+
type GeneratorOptions,
|
|
10
|
+
} from './Generator';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Generates the runtime for `q` queue workers.
|
|
14
|
+
*
|
|
15
|
+
* - **server** (`gkm dev`): a single `queues.ts` exposing `setupQueues()` that
|
|
16
|
+
* runs an in-process pg-boss poller alongside the Hono server — no SQS/Lambda.
|
|
17
|
+
* Each queue subscribes by its **name** on the shared event connection.
|
|
18
|
+
* - **aws-lambda**: one handler file per queue wrapping `AWSLambdaQueue`, backed
|
|
19
|
+
* by an SQS event-source mapping.
|
|
20
|
+
*/
|
|
21
|
+
export class QueueGenerator extends ConstructGenerator<
|
|
22
|
+
Queue<any, any, any, any>,
|
|
23
|
+
QueueInfo[]
|
|
24
|
+
> {
|
|
25
|
+
isConstruct(value: any): value is Queue<any, any, any, any> {
|
|
26
|
+
return Queue.isQueue(value);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async build(
|
|
30
|
+
context: BuildContext,
|
|
31
|
+
constructs: GeneratedConstruct<Queue<any, any, any, any>>[],
|
|
32
|
+
outputDir: string,
|
|
33
|
+
options?: GeneratorOptions,
|
|
34
|
+
): Promise<QueueInfo[]> {
|
|
35
|
+
const provider = options?.provider || 'aws-lambda';
|
|
36
|
+
const logger = console;
|
|
37
|
+
const queueInfos: QueueInfo[] = [];
|
|
38
|
+
|
|
39
|
+
if (provider === 'server') {
|
|
40
|
+
// Generate queues.ts for in-process polling (even if empty, so the
|
|
41
|
+
// server entry can always import setupQueues).
|
|
42
|
+
await this.generateServerQueuesFile(outputDir, constructs);
|
|
43
|
+
logger.log(
|
|
44
|
+
`Generated server queues file with ${constructs.length} queues (polling mode)`,
|
|
45
|
+
);
|
|
46
|
+
return queueInfos;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
if (constructs.length === 0 || provider !== 'aws-lambda') {
|
|
50
|
+
return queueInfos;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const queuesDir = join(outputDir, 'queues');
|
|
54
|
+
await mkdir(queuesDir, { recursive: true });
|
|
55
|
+
|
|
56
|
+
for (const { key, construct, path } of constructs) {
|
|
57
|
+
const handlerFile = await this.generateQueueHandler(
|
|
58
|
+
queuesDir,
|
|
59
|
+
path.relative,
|
|
60
|
+
key,
|
|
61
|
+
context,
|
|
62
|
+
);
|
|
63
|
+
|
|
64
|
+
queueInfos.push({
|
|
65
|
+
name: construct.name,
|
|
66
|
+
handler: relative(process.cwd(), handlerFile).replace(
|
|
67
|
+
/\.ts$/,
|
|
68
|
+
'.handler',
|
|
69
|
+
),
|
|
70
|
+
batchSize: construct.batchSize,
|
|
71
|
+
fifo: construct.fifo,
|
|
72
|
+
timeout: construct.timeout,
|
|
73
|
+
environment: await construct.getEnvironment({
|
|
74
|
+
markOptional: context.markOptional,
|
|
75
|
+
}),
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
logger.log(`Generated queue handler: ${key}`);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
return queueInfos;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
private async generateQueueHandler(
|
|
85
|
+
outputDir: string,
|
|
86
|
+
sourceFile: string,
|
|
87
|
+
exportName: string,
|
|
88
|
+
context: BuildContext,
|
|
89
|
+
): Promise<string> {
|
|
90
|
+
const handlerPath = join(outputDir, `${exportName}.ts`);
|
|
91
|
+
const importPath = relative(dirname(handlerPath), sourceFile).replace(
|
|
92
|
+
/\.ts$/,
|
|
93
|
+
'.js',
|
|
94
|
+
);
|
|
95
|
+
const relativeEnvParserPath = relative(
|
|
96
|
+
dirname(handlerPath),
|
|
97
|
+
context.envParserPath,
|
|
98
|
+
);
|
|
99
|
+
|
|
100
|
+
const content = `import { AWSLambdaQueue } from '@geekmidas/constructs/aws';
|
|
101
|
+
import { ${exportName} } from '${importPath}';
|
|
102
|
+
import ${context.envParserImportPattern} from '${relativeEnvParserPath}';
|
|
103
|
+
|
|
104
|
+
const adapter = new AWSLambdaQueue(envParser, ${exportName});
|
|
105
|
+
|
|
106
|
+
export const handler = adapter.handler;
|
|
107
|
+
`;
|
|
108
|
+
|
|
109
|
+
await writeFile(handlerPath, content);
|
|
110
|
+
return handlerPath;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
private async generateServerQueuesFile(
|
|
114
|
+
outputDir: string,
|
|
115
|
+
queues: GeneratedConstruct<Queue<any, any, any, any>>[],
|
|
116
|
+
): Promise<string> {
|
|
117
|
+
await mkdir(outputDir, { recursive: true });
|
|
118
|
+
|
|
119
|
+
const queuesPath = join(outputDir, 'queues.ts');
|
|
120
|
+
|
|
121
|
+
// Group imports by file
|
|
122
|
+
const importsByFile = new Map<string, string[]>();
|
|
123
|
+
for (const { path, key } of queues) {
|
|
124
|
+
const importPath = relative(dirname(queuesPath), path.relative).replace(
|
|
125
|
+
/\.ts$/,
|
|
126
|
+
'.js',
|
|
127
|
+
);
|
|
128
|
+
if (!importsByFile.has(importPath)) {
|
|
129
|
+
importsByFile.set(importPath, []);
|
|
130
|
+
}
|
|
131
|
+
importsByFile.get(importPath)?.push(key);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const imports = Array.from(importsByFile.entries())
|
|
135
|
+
.map(
|
|
136
|
+
([importPath, exports]) =>
|
|
137
|
+
`import { ${exports.join(', ')} } from '${importPath}';`,
|
|
138
|
+
)
|
|
139
|
+
.join('\n');
|
|
140
|
+
|
|
141
|
+
const allExportNames = queues.map(({ key }) => key);
|
|
142
|
+
|
|
143
|
+
const content = `/**
|
|
144
|
+
* Generated queue workers setup
|
|
145
|
+
*
|
|
146
|
+
* ⚠️ WARNING: This is for LOCAL DEVELOPMENT ONLY
|
|
147
|
+
* This runs an in-process pg-boss poller alongside the HTTP server. Each queue
|
|
148
|
+
* subscribes by its name on the shared event connection
|
|
149
|
+
* (EVENT_SUBSCRIBER_CONNECTION_STRING), so producers using
|
|
150
|
+
* <NAME>_PUBLISHER_CONNECTION_STRING (pgboss:// locally) reach it.
|
|
151
|
+
*
|
|
152
|
+
* For production, queues run as AWS Lambda with SQS event-source mappings.
|
|
153
|
+
*/
|
|
154
|
+
import type { EnvironmentParser } from '@geekmidas/envkit';
|
|
155
|
+
import type { Logger } from '@geekmidas/logger';
|
|
156
|
+
import { EventConnectionFactory, Subscriber } from '@geekmidas/events';
|
|
157
|
+
import type { EventConnection, EventSubscriber } from '@geekmidas/events';
|
|
158
|
+
import { ServiceDiscovery } from '@geekmidas/services';
|
|
159
|
+
${imports}
|
|
160
|
+
|
|
161
|
+
const queues = [
|
|
162
|
+
${allExportNames.join(',\n ')}
|
|
163
|
+
];
|
|
164
|
+
|
|
165
|
+
const activeSubscribers: EventSubscriber<any>[] = [];
|
|
166
|
+
|
|
167
|
+
export async function setupQueues(
|
|
168
|
+
envParser: EnvironmentParser<any>,
|
|
169
|
+
logger: Logger,
|
|
170
|
+
): Promise<void> {
|
|
171
|
+
if (queues.length === 0) {
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
logger.info('Setting up queue workers in polling mode (local development)');
|
|
176
|
+
|
|
177
|
+
const config = envParser.create((get) => ({
|
|
178
|
+
connectionString: get('EVENT_SUBSCRIBER_CONNECTION_STRING').string(),
|
|
179
|
+
})).parse();
|
|
180
|
+
|
|
181
|
+
const serviceDiscovery = ServiceDiscovery.getInstance(envParser);
|
|
182
|
+
|
|
183
|
+
let connection: EventConnection;
|
|
184
|
+
try {
|
|
185
|
+
connection = await EventConnectionFactory.fromConnectionString(config.connectionString);
|
|
186
|
+
const connectionType = new URL(config.connectionString).protocol.replace(':', '');
|
|
187
|
+
logger.info({ connectionType }, 'Created shared event connection for queues');
|
|
188
|
+
} catch (error) {
|
|
189
|
+
logger.error({ error }, 'Failed to create event connection for queues');
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
for (const queue of queues) {
|
|
194
|
+
try {
|
|
195
|
+
const eventSubscriber = await Subscriber.fromConnection(connection);
|
|
196
|
+
|
|
197
|
+
const services = queue.services.length > 0
|
|
198
|
+
? await serviceDiscovery.register(queue.services)
|
|
199
|
+
: {};
|
|
200
|
+
|
|
201
|
+
// A queue subscribes to a single "type" — its own name.
|
|
202
|
+
await eventSubscriber.subscribe([queue.name], async (message) => {
|
|
203
|
+
try {
|
|
204
|
+
const validation = await queue.messageSchema['~standard'].validate(message.payload);
|
|
205
|
+
if (validation.issues) {
|
|
206
|
+
logger.error({ issues: validation.issues, queue: queue.name }, 'Queue message failed validation');
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
await queue.handler({
|
|
211
|
+
messages: [validation.value],
|
|
212
|
+
services: services,
|
|
213
|
+
logger: queue.logger,
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
logger.debug({ queue: queue.name }, 'Successfully processed queue message');
|
|
217
|
+
} catch (error) {
|
|
218
|
+
logger.error({ error, queue: queue.name }, 'Failed to process queue message');
|
|
219
|
+
// Message will become visible again for retry.
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
activeSubscribers.push(eventSubscriber);
|
|
224
|
+
logger.info({ queue: queue.name }, 'Queue worker started polling');
|
|
225
|
+
} catch (error) {
|
|
226
|
+
logger.error({ error, queue: queue.name }, 'Failed to setup queue worker');
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const shutdown = () => {
|
|
231
|
+
logger.info('Stopping all queue workers');
|
|
232
|
+
for (const _ of activeSubscribers) {
|
|
233
|
+
connection.stop();
|
|
234
|
+
}
|
|
235
|
+
};
|
|
236
|
+
|
|
237
|
+
process.on('SIGTERM', shutdown);
|
|
238
|
+
process.on('SIGINT', shutdown);
|
|
239
|
+
}
|
|
240
|
+
`;
|
|
241
|
+
|
|
242
|
+
await writeFile(queuesPath, content);
|
|
243
|
+
return queuesPath;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
@@ -68,6 +68,11 @@ export class SubscriberGenerator extends ConstructGenerator<
|
|
|
68
68
|
'.handler',
|
|
69
69
|
),
|
|
70
70
|
subscribedEvents: construct.subscribedEvents || [],
|
|
71
|
+
// A subscriber bound to a topic (via `s.topic(topic)`) records the
|
|
72
|
+
// binding so infra wires the SNS subscription.
|
|
73
|
+
...(construct.topicName
|
|
74
|
+
? { transport: 'topic' as const, topic: construct.topicName }
|
|
75
|
+
: {}),
|
|
71
76
|
timeout: construct.timeout,
|
|
72
77
|
memorySize: construct.memorySize,
|
|
73
78
|
environment: await construct.getEnvironment({
|