@geekmidas/cli 1.10.41 → 1.12.0

package/src/build/manifests.ts

@@ -3,8 +3,10 @@ import { join, relative } from 'node:path';
 import type {
 	CronInfo,
 	FunctionInfo,
+	QueueInfo,
 	RouteInfo,
 	SubscriberInfo,
+	TopicInfo,
 } from '../types';
 
 const logger = console;
@@ -81,6 +83,8 @@ export async function generateAwsManifest(
 	functions: ManifestField<FunctionInfo>,
 	crons: ManifestField<CronInfo>,
 	subscribers: ManifestField<SubscriberInfo>,
+	queues: ManifestField<QueueInfo> = [],
+	topics: ManifestField<TopicInfo> = [],
 ): Promise<void> {
 	const manifestDir = join(outputDir, 'manifest');
 	await mkdir(manifestDir, { recursive: true });
@@ -92,12 +96,16 @@ export async function generateAwsManifest(
 	const functionsPartitioned = isPartitioned(functions);
 	const cronsPartitioned = isPartitioned(crons);
 	const subscribersPartitioned = isPartitioned(subscribers);
+	const queuesPartitioned = isPartitioned(queues);
+	const topicsPartitioned = isPartitioned(topics);
 
 	const content = `export const manifest = {
   routes: ${serializeField(awsRoutes)},
   functions: ${serializeField(functions)},
   crons: ${serializeField(crons)},
   subscribers: ${serializeField(subscribers)},
+  queues: ${serializeField(queues)},
+  topics: ${serializeField(topics)},
 } as const;
 
 // Derived types
@@ -105,6 +113,8 @@ ${generateDerivedType('routes', 'Route', routesPartitioned)}
 ${generateDerivedType('functions', 'Function', functionsPartitioned)}
 ${generateDerivedType('crons', 'Cron', cronsPartitioned)}
 ${generateDerivedType('subscribers', 'Subscriber', subscribersPartitioned)}
+${generateDerivedType('queues', 'Queue', queuesPartitioned)}
+${generateDerivedType('topics', 'Topic', topicsPartitioned)}
 
 // Useful union types
 export type Authorizer = Route['authorizer'];
@@ -116,7 +126,7 @@ export type RoutePath = Route['path'];
 	await writeFile(manifestPath, content);
 
 	logger.log(
-		`Generated AWS manifest with ${countItems(awsRoutes)} routes, ${countItems(functions)} functions, ${countItems(crons)} crons, ${countItems(subscribers)} subscribers`,
+		`Generated AWS manifest with ${countItems(awsRoutes)} routes, ${countItems(functions)} functions, ${countItems(crons)} crons, ${countItems(subscribers)} subscribers, ${countItems(queues)} queues, ${countItems(topics)} topics`,
 	);
 	logger.log(`Manifest: ${relative(process.cwd(), manifestPath)}`);
 }
@@ -126,6 +136,8 @@ export async function generateServerManifest(
 	appInfo: ServerAppInfo,
 	routes: ManifestField<RouteInfo>,
 	subscribers: ManifestField<SubscriberInfo>,
+	queues: ManifestField<QueueInfo> = [],
+	topics: ManifestField<TopicInfo> = [],
 ): Promise<void> {
 	const manifestDir = join(outputDir, 'manifest');
 	await mkdir(manifestDir, { recursive: true });
@@ -136,18 +148,30 @@ export async function generateServerManifest(
 	// Server subscribers only need name and events
 	const serverSubscribers = mapSubscriberMetadata(subscribers);
 
+	// Server queues only need their name (the worker is wired by setupQueues)
+	const serverQueues = mapQueueMetadata(queues);
+
+	// Server topics only need name + events (pg-boss routes by event name locally)
+	const serverTopics = mapTopicMetadata(topics);
+
 	const routesPartitioned = isPartitioned(serverRoutes);
 	const subscribersPartitioned = isPartitioned(serverSubscribers);
+	const queuesPartitioned = isPartitioned(serverQueues);
+	const topicsPartitioned = isPartitioned(serverTopics);
 
 	const content = `export const manifest = {
   app: ${JSON.stringify(appInfo, null, 2)},
   routes: ${serializeField(serverRoutes)},
   subscribers: ${serializeField(serverSubscribers)},
+  queues: ${serializeField(serverQueues)},
+  topics: ${serializeField(serverTopics)},
 } as const;
 
 // Derived types
 ${generateDerivedType('routes', 'Route', routesPartitioned)}
 ${generateDerivedType('subscribers', 'Subscriber', subscribersPartitioned)}
+${generateDerivedType('queues', 'Queue', queuesPartitioned)}
+${generateDerivedType('topics', 'Topic', topicsPartitioned)}
 
 // Useful union types
 export type Authorizer = Route['authorizer'];
@@ -159,7 +183,7 @@ export type RoutePath = Route['path'];
 	await writeFile(manifestPath, content);
 
 	logger.log(
-		`Generated server manifest with ${countItems(serverRoutes)} routes, ${countItems(serverSubscribers)} subscribers`,
+		`Generated server manifest with ${countItems(serverRoutes)} routes, ${countItems(serverSubscribers)} subscribers, ${countItems(serverQueues)} queues, ${countItems(serverTopics)} topics`,
 	);
 	logger.log(`Manifest: ${relative(process.cwd(), manifestPath)}`);
 }
@@ -210,7 +234,7 @@ function mapRouteMetadata(
  */
 function mapSubscriberMetadata(
 	subscribers: ManifestField<SubscriberInfo>,
-): ManifestField<{ name: string; subscribedEvents: string[] }> {
+): ManifestField<{ name: string; subscribedEvents: readonly string[] }> {
 	const mapFn = (s: SubscriberInfo) => ({
 		name: s.name,
 		subscribedEvents: s.subscribedEvents,
@@ -219,10 +243,43 @@ function mapSubscriberMetadata(
 	if (Array.isArray(subscribers)) {
 		return subscribers.map(mapFn);
 	}
-	const result: Record<string, { name: string; subscribedEvents: string[] }[]> =
-		{};
+	const result: Record<
+		string,
+		{ name: string; subscribedEvents: readonly string[] }[]
+	> = {};
 	for (const [partition, partitionSubs] of Object.entries(subscribers)) {
 		result[partition] = partitionSubs.map(mapFn);
 	}
 	return result;
 }
+
+function mapQueueMetadata(
+	queues: ManifestField<QueueInfo>,
+): ManifestField<{ name: string }> {
+	const mapFn = (q: QueueInfo) => ({ name: q.name });
+
+	if (Array.isArray(queues)) {
+		return queues.map(mapFn);
+	}
+	const result: Record<string, { name: string }[]> = {};
+	for (const [partition, partitionQueues] of Object.entries(queues)) {
+		result[partition] = partitionQueues.map(mapFn);
+	}
+	return result;
+}
+
+function mapTopicMetadata(
+	topics: ManifestField<TopicInfo>,
+): ManifestField<{ name: string; events: readonly string[] }> {
+	const mapFn = (t: TopicInfo) => ({ name: t.name, events: t.events });
+
+	if (Array.isArray(topics)) {
+		return topics.map(mapFn);
+	}
+	const result: Record<string, { name: string; events: readonly string[] }[]> =
+		{};
+	for (const [partition, partitionTopics] of Object.entries(topics)) {
+		result[partition] = partitionTopics.map(mapFn);
+	}
+	return result;
+}

package/src/dev/index.ts

@@ -33,7 +33,9 @@ import {
 	CronGenerator,
 	EndpointGenerator,
 	FunctionGenerator,
+	QueueGenerator,
 	SubscriberGenerator,
+	TopicGenerator,
 } from '../generators';
 import {
 	generateOpenApi,
@@ -1148,19 +1150,29 @@ async function buildServer(
 	const functionGenerator = new FunctionGenerator();
 	const cronGenerator = new CronGenerator();
 	const subscriberGenerator = new SubscriberGenerator();
+	const queueGenerator = new QueueGenerator();
+	const topicGenerator = new TopicGenerator();
 
 	// Load all constructs (resolve paths relative to appRoot)
-	const [allEndpoints, allFunctions, allCrons, allSubscribers] =
-		await Promise.all([
-			endpointGenerator.load(config.routes, appRoot, bustCache),
-			config.functions
-				? functionGenerator.load(config.functions, appRoot, bustCache)
-				: [],
-			config.crons ? cronGenerator.load(config.crons, appRoot, bustCache) : [],
-			config.subscribers
-				? subscriberGenerator.load(config.subscribers, appRoot, bustCache)
-				: [],
-		]);
+	const [
+		allEndpoints,
+		allFunctions,
+		allCrons,
+		allSubscribers,
+		allQueues,
+		allTopics,
+	] = await Promise.all([
+		endpointGenerator.load(config.routes, appRoot, bustCache),
+		config.functions
+			? functionGenerator.load(config.functions, appRoot, bustCache)
+			: [],
+		config.crons ? cronGenerator.load(config.crons, appRoot, bustCache) : [],
+		config.subscribers
+			? subscriberGenerator.load(config.subscribers, appRoot, bustCache)
+			: [],
+		config.queues ? queueGenerator.load(config.queues, appRoot, bustCache) : [],
+		config.topics ? topicGenerator.load(config.topics, appRoot, bustCache) : [],
+	]);
 
 	// Ensure .gkm directory exists in app root
 	const outputDir = join(appRoot, '.gkm', provider);
@@ -1175,6 +1187,8 @@ async function buildServer(
 		functionGenerator.build(context, allFunctions, outputDir, { provider }),
 		cronGenerator.build(context, allCrons, outputDir, { provider }),
 		subscriberGenerator.build(context, allSubscribers, outputDir, { provider }),
+		queueGenerator.build(context, allQueues, outputDir, { provider }),
+		topicGenerator.build(context, allTopics, outputDir, { provider }),
 	]);
 }
 

package/src/generators/EndpointGenerator.ts

@@ -618,6 +618,7 @@ import { Hono } from 'hono';
 import type { Hono as HonoType } from 'hono';
 import { setupEndpoints } from './endpoints.js';
 import { setupSubscribers } from './subscribers.js';
+import { setupQueues } from './queues.js';
 import ${context.envParserImportPattern} from '${relativeEnvParserPath}';
 import ${context.loggerImportPattern} from '${relativeLoggerPath}';
 ${telescopeImports}
@@ -691,6 +692,11 @@ ${afterSetupCall}
         logger.error({ error }, 'Failed to start subscribers');
       });
 
+      // Start queue workers in background (non-blocking, local development only)
+      await setupQueues(envParser, logger).catch((error) => {
+        logger.error({ error }, 'Failed to start queue workers');
+      });
+
       logger.info({ port }, 'Starting server');
 
       // Start HTTP server using provided serve function
@@ -800,18 +806,24 @@ export const handler = ${exportName};
 `;
 		}
 
-		// Subscriber setup code
+		// Subscriber + queue setup code (background workers share the same flag)
 		const subscriberSetup = includeSubscribers
 			? `
       // Start subscribers in background
       await setupSubscribers(envParser, logger).catch((error) => {
         logger.error({ error }, 'Failed to start subscribers');
       });
+
+      // Start queue workers in background
+      await setupQueues(envParser, logger).catch((error) => {
+        logger.error({ error }, 'Failed to start queue workers');
+      });
 `
 			: '';
 
 		const subscriberImport = includeSubscribers
-			? `import { setupSubscribers } from './subscribers.js';`
+			? `import { setupSubscribers } from './subscribers.js';
+import { setupQueues } from './queues.js';`
 			: '';
 
 		// Graceful shutdown code

package/src/generators/QueueGenerator.ts

@@ -0,0 +1,245 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname, join, relative } from 'node:path';
+import { Queue } from '@geekmidas/constructs/queue';
+import type { BuildContext } from '../build/types';
+import type { QueueInfo } from '../types';
+import {
+	ConstructGenerator,
+	type GeneratedConstruct,
+	type GeneratorOptions,
+} from './Generator';
+
+/**
+ * Generates the runtime for `q` queue workers.
+ *
+ * - **server** (`gkm dev`): a single `queues.ts` exposing `setupQueues()` that
+ *   runs an in-process pg-boss poller alongside the Hono server — no SQS/Lambda.
+ *   Each queue subscribes by its **name** on the shared event connection.
+ * - **aws-lambda**: one handler file per queue wrapping `AWSLambdaQueue`, backed
+ *   by an SQS event-source mapping.
+ */
+export class QueueGenerator extends ConstructGenerator<
+	Queue<any, any, any, any>,
+	QueueInfo[]
+> {
+	isConstruct(value: any): value is Queue<any, any, any, any> {
+		return Queue.isQueue(value);
+	}
+
+	async build(
+		context: BuildContext,
+		constructs: GeneratedConstruct<Queue<any, any, any, any>>[],
+		outputDir: string,
+		options?: GeneratorOptions,
+	): Promise<QueueInfo[]> {
+		const provider = options?.provider || 'aws-lambda';
+		const logger = console;
+		const queueInfos: QueueInfo[] = [];
+
+		if (provider === 'server') {
+			// Generate queues.ts for in-process polling (even if empty, so the
+			// server entry can always import setupQueues).
+			await this.generateServerQueuesFile(outputDir, constructs);
+			logger.log(
+				`Generated server queues file with ${constructs.length} queues (polling mode)`,
+			);
+			return queueInfos;
+		}
+
+		if (constructs.length === 0 || provider !== 'aws-lambda') {
+			return queueInfos;
+		}
+
+		const queuesDir = join(outputDir, 'queues');
+		await mkdir(queuesDir, { recursive: true });
+
+		for (const { key, construct, path } of constructs) {
+			const handlerFile = await this.generateQueueHandler(
+				queuesDir,
+				path.relative,
+				key,
+				context,
+			);
+
+			queueInfos.push({
+				name: construct.name,
+				handler: relative(process.cwd(), handlerFile).replace(
+					/\.ts$/,
+					'.handler',
+				),
+				batchSize: construct.batchSize,
+				fifo: construct.fifo,
+				timeout: construct.timeout,
+				environment: await construct.getEnvironment({
+					markOptional: context.markOptional,
+				}),
+			});
+
+			logger.log(`Generated queue handler: ${key}`);
+		}
+
+		return queueInfos;
+	}
+
+	private async generateQueueHandler(
+		outputDir: string,
+		sourceFile: string,
+		exportName: string,
+		context: BuildContext,
+	): Promise<string> {
+		const handlerPath = join(outputDir, `${exportName}.ts`);
+		const importPath = relative(dirname(handlerPath), sourceFile).replace(
+			/\.ts$/,
+			'.js',
+		);
+		const relativeEnvParserPath = relative(
+			dirname(handlerPath),
+			context.envParserPath,
+		);
+
+		const content = `import { AWSLambdaQueue } from '@geekmidas/constructs/aws';
+import { ${exportName} } from '${importPath}';
+import ${context.envParserImportPattern} from '${relativeEnvParserPath}';
+
+const adapter = new AWSLambdaQueue(envParser, ${exportName});
+
+export const handler = adapter.handler;
+`;
+
+		await writeFile(handlerPath, content);
+		return handlerPath;
+	}
+
+	private async generateServerQueuesFile(
+		outputDir: string,
+		queues: GeneratedConstruct<Queue<any, any, any, any>>[],
+	): Promise<string> {
+		await mkdir(outputDir, { recursive: true });
+
+		const queuesPath = join(outputDir, 'queues.ts');
+
+		// Group imports by file
+		const importsByFile = new Map<string, string[]>();
+		for (const { path, key } of queues) {
+			const importPath = relative(dirname(queuesPath), path.relative).replace(
+				/\.ts$/,
+				'.js',
+			);
+			if (!importsByFile.has(importPath)) {
+				importsByFile.set(importPath, []);
+			}
+			importsByFile.get(importPath)?.push(key);
+		}
+
+		const imports = Array.from(importsByFile.entries())
+			.map(
+				([importPath, exports]) =>
+					`import { ${exports.join(', ')} } from '${importPath}';`,
+			)
+			.join('\n');
+
+		const allExportNames = queues.map(({ key }) => key);
+
+		const content = `/**
+ * Generated queue workers setup
+ *
+ * ⚠️  WARNING: This is for LOCAL DEVELOPMENT ONLY
+ * This runs an in-process pg-boss poller alongside the HTTP server. Each queue
+ * subscribes by its name on the shared event connection
+ * (EVENT_SUBSCRIBER_CONNECTION_STRING), so producers using
+ * <NAME>_PUBLISHER_CONNECTION_STRING (pgboss:// locally) reach it.
+ *
+ * For production, queues run as AWS Lambda with SQS event-source mappings.
+ */
+import type { EnvironmentParser } from '@geekmidas/envkit';
+import type { Logger } from '@geekmidas/logger';
+import { EventConnectionFactory, Subscriber } from '@geekmidas/events';
+import type { EventConnection, EventSubscriber } from '@geekmidas/events';
+import { ServiceDiscovery } from '@geekmidas/services';
+${imports}
+
+const queues = [
+  ${allExportNames.join(',\n  ')}
+];
+
+const activeSubscribers: EventSubscriber<any>[] = [];
+
+export async function setupQueues(
+  envParser: EnvironmentParser<any>,
+  logger: Logger,
+): Promise<void> {
+  if (queues.length === 0) {
+    return;
+  }
+
+  logger.info('Setting up queue workers in polling mode (local development)');
+
+  const config = envParser.create((get) => ({
+    connectionString: get('EVENT_SUBSCRIBER_CONNECTION_STRING').string(),
+  })).parse();
+
+  const serviceDiscovery = ServiceDiscovery.getInstance(envParser);
+
+  let connection: EventConnection;
+  try {
+    connection = await EventConnectionFactory.fromConnectionString(config.connectionString);
+    const connectionType = new URL(config.connectionString).protocol.replace(':', '');
+    logger.info({ connectionType }, 'Created shared event connection for queues');
+  } catch (error) {
+    logger.error({ error }, 'Failed to create event connection for queues');
+    return;
+  }
+
+  for (const queue of queues) {
+    try {
+      const eventSubscriber = await Subscriber.fromConnection(connection);
+
+      const services = queue.services.length > 0
+        ? await serviceDiscovery.register(queue.services)
+        : {};
+
+      // A queue subscribes to a single "type" — its own name.
+      await eventSubscriber.subscribe([queue.name], async (message) => {
+        try {
+          const validation = await queue.messageSchema['~standard'].validate(message.payload);
+          if (validation.issues) {
+            logger.error({ issues: validation.issues, queue: queue.name }, 'Queue message failed validation');
+            return;
+          }
+
+          await queue.handler({
+            messages: [validation.value],
+            services: services,
+            logger: queue.logger,
+          });
+
+          logger.debug({ queue: queue.name }, 'Successfully processed queue message');
+        } catch (error) {
+          logger.error({ error, queue: queue.name }, 'Failed to process queue message');
+          // Message will become visible again for retry.
+        }
+      });
+
+      activeSubscribers.push(eventSubscriber);
+      logger.info({ queue: queue.name }, 'Queue worker started polling');
+    } catch (error) {
+      logger.error({ error, queue: queue.name }, 'Failed to setup queue worker');
+    }
+  }
+
+  const shutdown = () => {
+    logger.info('Stopping all queue workers');
+    for (const _ of activeSubscribers) {
+      connection.stop();
+    }
+  };
+
+  process.on('SIGTERM', shutdown);
+  process.on('SIGINT', shutdown);
+}
+`;
+
+		await writeFile(queuesPath, content);
+		return queuesPath;
+	}
+}

package/src/generators/SubscriberGenerator.ts

@@ -68,6 +68,11 @@ export class SubscriberGenerator extends ConstructGenerator<
 					'.handler',
 				),
 				subscribedEvents: construct.subscribedEvents || [],
+				// A subscriber bound to a topic (via `s.topic(topic)`) records the
+				// binding so infra wires the SNS subscription.
+				...(construct.topicName
+					? { transport: 'topic' as const, topic: construct.topicName }
+					: {}),
 				timeout: construct.timeout,
 				memorySize: construct.memorySize,
 				environment: await construct.getEnvironment({

package/src/generators/TopicGenerator.ts

@@ -0,0 +1,45 @@
+import { Topic } from '@geekmidas/constructs/topic';
+import type { BuildContext } from '../build/types';
+import type { TopicInfo } from '../types';
+import {
+	ConstructGenerator,
+	type GeneratedConstruct,
+	type GeneratorOptions,
+} from './Generator';
+
+/**
+ * Discovers `t` topics into `manifest.topics`. A topic is a *resource*, not a
+ * function — it has no handler to generate. Producers reach it via its derived
+ * publisher (`topic.publisher`, whose connection string is sniffed onto the
+ * producing construct) and subscribers bind via `s.topic(topic)`; this generator
+ * only records the topic + its event contract so infra can provision the SNS
+ * topic. Identical output for every provider (locally pg-boss routes by event
+ * name, so there's nothing to run).
+ */
+export class TopicGenerator extends ConstructGenerator<
+	Topic<any, any>,
+	TopicInfo[]
+> {
+	isConstruct(value: any): value is Topic<any, any> {
+		return Topic.isTopic(value);
+	}
+
+	async build(
+		_context: BuildContext,
+		constructs: GeneratedConstruct<Topic<any, any>>[],
+		_outputDir: string,
+		_options?: GeneratorOptions,
+	): Promise<TopicInfo[]> {
+		const logger = console;
+		const topicInfos = constructs.map(({ construct }) => ({
+			name: construct.name,
+			events: construct.eventTypes,
+		}));
+
+		for (const topic of topicInfos) {
+			logger.log(`Discovered topic: ${topic.name}`);
+		}
+
+		return topicInfos;
+	}
+}