@microfox/ai-worker-cli 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +428 -15
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +428 -15
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
|
-
import { Command as
|
|
4
|
+
import { Command as Command3 } from "commander";
|
|
5
5
|
|
|
6
6
|
// src/commands/push.ts
|
|
7
7
|
import { Command } from "commander";
|
|
@@ -132,6 +132,50 @@ async function collectEnvUsageForWorkers(workerEntryFiles, projectRoot) {
|
|
|
132
132
|
buildtimeKeys.delete("node");
|
|
133
133
|
return { runtimeKeys, buildtimeKeys };
|
|
134
134
|
}
|
|
135
|
+
async function collectCalleeWorkerIds(workers, projectRoot) {
|
|
136
|
+
void projectRoot;
|
|
137
|
+
const calleeIdsByWorker = /* @__PURE__ */ new Map();
|
|
138
|
+
const workerIds = new Set(workers.map((w) => w.id));
|
|
139
|
+
for (const worker of workers) {
|
|
140
|
+
const calleeIds = /* @__PURE__ */ new Set();
|
|
141
|
+
const visited = /* @__PURE__ */ new Set();
|
|
142
|
+
const queue = [worker.filePath];
|
|
143
|
+
while (queue.length > 0) {
|
|
144
|
+
const file = queue.pop();
|
|
145
|
+
const normalized = path.resolve(file);
|
|
146
|
+
if (visited.has(normalized)) continue;
|
|
147
|
+
visited.add(normalized);
|
|
148
|
+
if (!fs.existsSync(normalized) || !fs.statSync(normalized).isFile()) continue;
|
|
149
|
+
const src = fs.readFileSync(normalized, "utf-8");
|
|
150
|
+
const re = /(?:ctx\.)?dispatchWorker\s*\(\s*['"]([^'"]+)['"]/g;
|
|
151
|
+
for (const match of src.matchAll(re)) {
|
|
152
|
+
if (match[1]) calleeIds.add(match[1]);
|
|
153
|
+
}
|
|
154
|
+
const specifiers = extractImportSpecifiers(src);
|
|
155
|
+
for (const spec of specifiers) {
|
|
156
|
+
if (!spec || !spec.startsWith(".")) continue;
|
|
157
|
+
const resolved = tryResolveLocalImport(normalized, spec);
|
|
158
|
+
if (resolved) queue.push(resolved);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (calleeIds.size > 0) {
|
|
162
|
+
for (const calleeId of calleeIds) {
|
|
163
|
+
if (!workerIds.has(calleeId)) {
|
|
164
|
+
console.warn(
|
|
165
|
+
chalk.yellow(
|
|
166
|
+
`\u26A0\uFE0F Worker "${worker.id}" calls "${calleeId}" which is not in scanned workers (typo or other service?). Queue URL will not be auto-injected.`
|
|
167
|
+
)
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
calleeIdsByWorker.set(worker.id, calleeIds);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
return calleeIdsByWorker;
|
|
175
|
+
}
|
|
176
|
+
function sanitizeWorkerIdForEnv(workerId) {
|
|
177
|
+
return workerId.replace(/-/g, "_").toUpperCase();
|
|
178
|
+
}
|
|
135
179
|
function readJsonFile(filePath) {
|
|
136
180
|
try {
|
|
137
181
|
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
|
@@ -184,6 +228,20 @@ async function collectRuntimeDependenciesForWorkers(workerEntryFiles, projectRoo
|
|
|
184
228
|
deps.delete("@microfox/ai-worker");
|
|
185
229
|
return deps;
|
|
186
230
|
}
|
|
231
|
+
function getJobStoreType() {
|
|
232
|
+
const raw = process.env.WORKER_DATABASE_TYPE?.toLowerCase();
|
|
233
|
+
if (raw === "mongodb" || raw === "upstash-redis") return raw;
|
|
234
|
+
return "upstash-redis";
|
|
235
|
+
}
|
|
236
|
+
function filterDepsForJobStore(runtimeDeps, jobStoreType) {
|
|
237
|
+
const filtered = new Set(runtimeDeps);
|
|
238
|
+
filtered.delete("mongodb");
|
|
239
|
+
filtered.delete("@upstash/redis");
|
|
240
|
+
if (jobStoreType === "mongodb") filtered.add("mongodb");
|
|
241
|
+
else filtered.add("@upstash/redis");
|
|
242
|
+
if (runtimeDeps.has("mongodb")) filtered.add("mongodb");
|
|
243
|
+
return filtered;
|
|
244
|
+
}
|
|
187
245
|
function buildDependenciesMap(projectRoot, deps) {
|
|
188
246
|
const projectPkg = readJsonFile(path.join(projectRoot, "package.json")) || {};
|
|
189
247
|
const projectDeps = projectPkg.dependencies || {};
|
|
@@ -253,8 +311,116 @@ async function scanWorkers(aiPath = "app/ai") {
|
|
|
253
311
|
}
|
|
254
312
|
return workers;
|
|
255
313
|
}
|
|
256
|
-
async function
|
|
314
|
+
async function scanQueues(aiPath = "app/ai") {
|
|
315
|
+
const base = aiPath.replace(/\\/g, "/");
|
|
316
|
+
const pattern = `${base}/queues/**/*.queue.ts`;
|
|
317
|
+
const files = await glob(pattern);
|
|
318
|
+
const queues = [];
|
|
319
|
+
for (const filePath of files) {
|
|
320
|
+
try {
|
|
321
|
+
const content = fs.readFileSync(filePath, "utf-8");
|
|
322
|
+
const idMatch = content.match(/defineWorkerQueue\s*\(\s*\{[\s\S]*?id:\s*['"]([^'"]+)['"]/);
|
|
323
|
+
if (!idMatch) {
|
|
324
|
+
console.warn(chalk.yellow(`\u26A0\uFE0F Skipping ${filePath}: No queue id found in defineWorkerQueue`));
|
|
325
|
+
continue;
|
|
326
|
+
}
|
|
327
|
+
const queueId = idMatch[1];
|
|
328
|
+
const steps = [];
|
|
329
|
+
const stepsMatch = content.match(/steps:\s*\[([\s\S]*?)\]/);
|
|
330
|
+
if (stepsMatch) {
|
|
331
|
+
const stepsStr = stepsMatch[1];
|
|
332
|
+
const stepRegex = /\{\s*workerId:\s*['"]([^'"]+)['"](?:,\s*delaySeconds:\s*(\d+))?(?:,\s*mapInputFromPrev:\s*['"]([^'"]+)['"])?\s*\}/g;
|
|
333
|
+
let m;
|
|
334
|
+
while ((m = stepRegex.exec(stepsStr)) !== null) {
|
|
335
|
+
steps.push({
|
|
336
|
+
workerId: m[1],
|
|
337
|
+
delaySeconds: m[2] ? parseInt(m[2], 10) : void 0,
|
|
338
|
+
mapInputFromPrev: m[3]
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
let schedule;
|
|
343
|
+
const scheduleStrMatch = content.match(/schedule:\s*['"]([^'"]+)['"]/);
|
|
344
|
+
const scheduleObjMatch = content.match(/schedule:\s*(\{[^}]+(?:\{[^}]*\}[^}]*)*\})/);
|
|
345
|
+
if (scheduleStrMatch) {
|
|
346
|
+
schedule = scheduleStrMatch[1];
|
|
347
|
+
} else if (scheduleObjMatch) {
|
|
348
|
+
try {
|
|
349
|
+
schedule = new Function("return " + scheduleObjMatch[1])();
|
|
350
|
+
} catch {
|
|
351
|
+
schedule = void 0;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
queues.push({ id: queueId, filePath, steps, schedule });
|
|
355
|
+
} catch (error) {
|
|
356
|
+
console.error(chalk.red(`\u274C Error processing ${filePath}:`), error);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
return queues;
|
|
360
|
+
}
|
|
361
|
+
function generateQueueRegistry(queues, outputDir, projectRoot) {
|
|
362
|
+
const generatedDir = path.join(outputDir, "generated");
|
|
363
|
+
if (!fs.existsSync(generatedDir)) {
|
|
364
|
+
fs.mkdirSync(generatedDir, { recursive: true });
|
|
365
|
+
}
|
|
366
|
+
const registryContent = `/**
|
|
367
|
+
* Auto-generated queue registry. DO NOT EDIT.
|
|
368
|
+
* Generated by @microfox/ai-worker-cli from .queue.ts files.
|
|
369
|
+
*/
|
|
370
|
+
|
|
371
|
+
const QUEUES = ${JSON.stringify(queues.map((q) => ({ id: q.id, steps: q.steps, schedule: q.schedule })), null, 2)};
|
|
372
|
+
|
|
373
|
+
export function getQueueById(queueId) {
|
|
374
|
+
return QUEUES.find((q) => q.id === queueId);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
export function getNextStep(queueId, stepIndex) {
|
|
378
|
+
const queue = getQueueById(queueId);
|
|
379
|
+
if (!queue || !queue.steps || stepIndex < 0 || stepIndex >= queue.steps.length - 1) {
|
|
380
|
+
return undefined;
|
|
381
|
+
}
|
|
382
|
+
const step = queue.steps[stepIndex + 1];
|
|
383
|
+
return step ? { workerId: step.workerId, delaySeconds: step.delaySeconds, mapInputFromPrev: step.mapInputFromPrev } : undefined;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
export function invokeMapInput(_queueId, _stepIndex, prevOutput, _initialInput) {
|
|
387
|
+
return prevOutput;
|
|
388
|
+
}
|
|
389
|
+
`;
|
|
390
|
+
const registryPath = path.join(generatedDir, "workerQueues.registry.js");
|
|
391
|
+
fs.writeFileSync(registryPath, registryContent);
|
|
392
|
+
console.log(chalk.green(`\u2713 Generated queue registry: ${registryPath}`));
|
|
393
|
+
}
|
|
394
|
+
function getWorkersInQueues(queues) {
|
|
395
|
+
const set = /* @__PURE__ */ new Set();
|
|
396
|
+
for (const q of queues) {
|
|
397
|
+
for (const step of q.steps) {
|
|
398
|
+
set.add(step.workerId);
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
return set;
|
|
402
|
+
}
|
|
403
|
+
function mergeQueueCallees(calleeIds, queues, workers) {
|
|
404
|
+
const merged = new Map(calleeIds);
|
|
405
|
+
const workerIds = new Set(workers.map((w) => w.id));
|
|
406
|
+
for (const queue of queues) {
|
|
407
|
+
for (let i = 0; i < queue.steps.length - 1; i++) {
|
|
408
|
+
const fromWorkerId = queue.steps[i].workerId;
|
|
409
|
+
const toWorkerId = queue.steps[i + 1].workerId;
|
|
410
|
+
if (!workerIds.has(toWorkerId)) continue;
|
|
411
|
+
let callees = merged.get(fromWorkerId);
|
|
412
|
+
if (!callees) {
|
|
413
|
+
callees = /* @__PURE__ */ new Set();
|
|
414
|
+
merged.set(fromWorkerId, callees);
|
|
415
|
+
}
|
|
416
|
+
callees.add(toWorkerId);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
return merged;
|
|
420
|
+
}
|
|
421
|
+
async function generateHandlers(workers, outputDir, queues = []) {
|
|
257
422
|
const handlersDir = path.join(outputDir, "handlers");
|
|
423
|
+
const workersInQueues = getWorkersInQueues(queues);
|
|
258
424
|
if (fs.existsSync(handlersDir)) {
|
|
259
425
|
fs.rmSync(handlersDir, { recursive: true, force: true });
|
|
260
426
|
}
|
|
@@ -279,18 +445,76 @@ async function generateHandlers(workers, outputDir) {
|
|
|
279
445
|
const exportName = exportMatch ? exportMatch[2] : "worker";
|
|
280
446
|
const tempEntryFile = handlerFile.replace(".js", ".temp.ts");
|
|
281
447
|
const workerRef = defaultExport ? "workerModule.default" : `workerModule.${exportName}`;
|
|
282
|
-
const
|
|
448
|
+
const inQueue = workersInQueues.has(worker.id);
|
|
449
|
+
const registryRelPath = path.relative(path.dirname(path.resolve(handlerFile)), path.join(outputDir, "generated", "workerQueues.registry")).split(path.sep).join("/");
|
|
450
|
+
const registryImportPath = registryRelPath.startsWith(".") ? registryRelPath : "./" + registryRelPath;
|
|
451
|
+
const handlerCreation = inQueue ? `
|
|
452
|
+
import { createLambdaHandler, wrapHandlerForQueue } from '@microfox/ai-worker/handler';
|
|
453
|
+
import * as queueRegistry from '${registryImportPath}';
|
|
454
|
+
import * as workerModule from '${relativeImportPath}';
|
|
455
|
+
|
|
456
|
+
const WORKER_LOG_PREFIX = '[WorkerEntrypoint]';
|
|
457
|
+
|
|
458
|
+
const workerAgent = ${workerRef};
|
|
459
|
+
if (!workerAgent || typeof workerAgent.handler !== 'function') {
|
|
460
|
+
throw new Error('Worker module must export a createWorker result (default or named) with .handler');
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
const queueRuntime = {
|
|
464
|
+
getNextStep: queueRegistry.getNextStep,
|
|
465
|
+
invokeMapInput: queueRegistry.invokeMapInput,
|
|
466
|
+
};
|
|
467
|
+
const wrappedHandler = wrapHandlerForQueue(workerAgent.handler, queueRuntime);
|
|
468
|
+
|
|
469
|
+
const baseHandler = createLambdaHandler(wrappedHandler, workerAgent.outputSchema);
|
|
470
|
+
|
|
471
|
+
export const handler = async (event: any, context: any) => {
|
|
472
|
+
const records = Array.isArray((event as any)?.Records) ? (event as any).Records.length : 0;
|
|
473
|
+
try {
|
|
474
|
+
console.log(WORKER_LOG_PREFIX, {
|
|
475
|
+
workerId: workerAgent.id,
|
|
476
|
+
inQueue: true,
|
|
477
|
+
records,
|
|
478
|
+
requestId: (context as any)?.awsRequestId,
|
|
479
|
+
});
|
|
480
|
+
} catch {
|
|
481
|
+
// Best-effort logging only
|
|
482
|
+
}
|
|
483
|
+
return baseHandler(event, context);
|
|
484
|
+
};
|
|
485
|
+
|
|
486
|
+
export const exportedWorkerConfig = workerModule.workerConfig || workerAgent?.workerConfig;
|
|
487
|
+
` : `
|
|
283
488
|
import { createLambdaHandler } from '@microfox/ai-worker/handler';
|
|
284
489
|
import * as workerModule from '${relativeImportPath}';
|
|
285
490
|
|
|
491
|
+
const WORKER_LOG_PREFIX = '[WorkerEntrypoint]';
|
|
492
|
+
|
|
286
493
|
const workerAgent = ${workerRef};
|
|
287
494
|
if (!workerAgent || typeof workerAgent.handler !== 'function') {
|
|
288
495
|
throw new Error('Worker module must export a createWorker result (default or named) with .handler');
|
|
289
496
|
}
|
|
290
497
|
|
|
291
|
-
|
|
498
|
+
const baseHandler = createLambdaHandler(workerAgent.handler, workerAgent.outputSchema);
|
|
499
|
+
|
|
500
|
+
export const handler = async (event: any, context: any) => {
|
|
501
|
+
const records = Array.isArray((event as any)?.Records) ? (event as any).Records.length : 0;
|
|
502
|
+
try {
|
|
503
|
+
console.log(WORKER_LOG_PREFIX, {
|
|
504
|
+
workerId: workerAgent.id,
|
|
505
|
+
inQueue: false,
|
|
506
|
+
records,
|
|
507
|
+
requestId: (context as any)?.awsRequestId,
|
|
508
|
+
});
|
|
509
|
+
} catch {
|
|
510
|
+
// Best-effort logging only
|
|
511
|
+
}
|
|
512
|
+
return baseHandler(event, context);
|
|
513
|
+
};
|
|
514
|
+
|
|
292
515
|
export const exportedWorkerConfig = workerModule.workerConfig || workerAgent?.workerConfig;
|
|
293
516
|
`;
|
|
517
|
+
const tempEntryContent = handlerCreation;
|
|
294
518
|
fs.writeFileSync(tempEntryFile, tempEntryContent);
|
|
295
519
|
try {
|
|
296
520
|
const fixLazyCachePlugin = {
|
|
@@ -702,7 +926,76 @@ export const handler = async (event: APIGatewayProxyEvent): Promise<APIGatewayPr
|
|
|
702
926
|
fs.unlinkSync(tempEntryFile);
|
|
703
927
|
console.log(chalk.green(`\u2713 Generated /workers/trigger handler`));
|
|
704
928
|
}
|
|
705
|
-
function
|
|
929
|
+
function generateQueueStarterHandler(outputDir, queue, serviceName) {
|
|
930
|
+
const safeId = queue.id.replace(/[^a-zA-Z0-9]/g, "");
|
|
931
|
+
const handlerFile = path.join(outputDir, "handlers", `queue-starter-${safeId}.js`);
|
|
932
|
+
const tempEntryFile = handlerFile.replace(".js", ".temp.ts");
|
|
933
|
+
const handlerDir = path.dirname(handlerFile);
|
|
934
|
+
if (!fs.existsSync(handlerDir)) {
|
|
935
|
+
fs.mkdirSync(handlerDir, { recursive: true });
|
|
936
|
+
}
|
|
937
|
+
const firstWorkerId = queue.steps[0]?.workerId;
|
|
938
|
+
if (!firstWorkerId) return;
|
|
939
|
+
const handlerContent = `/**
|
|
940
|
+
* Auto-generated queue-starter for queue "${queue.id}"
|
|
941
|
+
* DO NOT EDIT - This file is generated by @microfox/ai-worker-cli
|
|
942
|
+
*/
|
|
943
|
+
|
|
944
|
+
import { ScheduledHandler } from 'aws-lambda';
|
|
945
|
+
import { SQSClient, GetQueueUrlCommand, SendMessageCommand } from '@aws-sdk/client-sqs';
|
|
946
|
+
|
|
947
|
+
const QUEUE_ID = ${JSON.stringify(queue.id)};
|
|
948
|
+
const FIRST_WORKER_ID = ${JSON.stringify(firstWorkerId)};
|
|
949
|
+
const SERVICE_NAME = ${JSON.stringify(serviceName)};
|
|
950
|
+
|
|
951
|
+
export const handler: ScheduledHandler = async () => {
|
|
952
|
+
const stage = process.env.ENVIRONMENT || process.env.STAGE || 'prod';
|
|
953
|
+
const region = process.env.AWS_REGION || 'us-east-1';
|
|
954
|
+
const queueName = \`\${SERVICE_NAME}-\${FIRST_WORKER_ID}-\${stage}\`;
|
|
955
|
+
|
|
956
|
+
const sqs = new SQSClient({ region });
|
|
957
|
+
const { QueueUrl } = await sqs.send(new GetQueueUrlCommand({ QueueName: queueName }));
|
|
958
|
+
if (!QueueUrl) {
|
|
959
|
+
throw new Error('Queue URL not found: ' + queueName);
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
const jobId = 'job-' + Date.now() + '-' + Math.random().toString(36).slice(2, 11);
|
|
963
|
+
const initialInput = {};
|
|
964
|
+
const messageBody = {
|
|
965
|
+
workerId: FIRST_WORKER_ID,
|
|
966
|
+
jobId,
|
|
967
|
+
input: {
|
|
968
|
+
...initialInput,
|
|
969
|
+
__workerQueue: { id: QUEUE_ID, stepIndex: 0, initialInput },
|
|
970
|
+
},
|
|
971
|
+
context: {},
|
|
972
|
+
metadata: { __workerQueue: { id: QUEUE_ID, stepIndex: 0, initialInput } },
|
|
973
|
+
timestamp: new Date().toISOString(),
|
|
974
|
+
};
|
|
975
|
+
|
|
976
|
+
await sqs.send(new SendMessageCommand({
|
|
977
|
+
QueueUrl,
|
|
978
|
+
MessageBody: JSON.stringify(messageBody),
|
|
979
|
+
}));
|
|
980
|
+
|
|
981
|
+
console.log('[queue-starter] Dispatched first worker for queue:', { queueId: QUEUE_ID, jobId, workerId: FIRST_WORKER_ID });
|
|
982
|
+
};
|
|
983
|
+
`;
|
|
984
|
+
fs.writeFileSync(tempEntryFile, handlerContent);
|
|
985
|
+
esbuild.buildSync({
|
|
986
|
+
entryPoints: [tempEntryFile],
|
|
987
|
+
bundle: true,
|
|
988
|
+
platform: "node",
|
|
989
|
+
target: "node20",
|
|
990
|
+
outfile: handlerFile,
|
|
991
|
+
external: ["aws-sdk", "canvas", "@microfox/puppeteer-sls", "@sparticuz/chromium"],
|
|
992
|
+
packages: "bundle",
|
|
993
|
+
logLevel: "error"
|
|
994
|
+
});
|
|
995
|
+
fs.unlinkSync(tempEntryFile);
|
|
996
|
+
console.log(chalk.green(`\u2713 Generated queue-starter for ${queue.id}`));
|
|
997
|
+
}
|
|
998
|
+
function generateWorkersConfigHandler(outputDir, workers, serviceName, queues = []) {
|
|
706
999
|
const handlerFile = path.join(outputDir, "handlers", "workers-config.js");
|
|
707
1000
|
const tempEntryFile = handlerFile.replace(".js", ".temp.ts");
|
|
708
1001
|
const handlerDir = path.dirname(handlerFile);
|
|
@@ -718,8 +1011,9 @@ function generateWorkersConfigHandler(outputDir, workers, serviceName) {
|
|
|
718
1011
|
import { APIGatewayProxyEvent, APIGatewayProxyResult } from 'aws-lambda';
|
|
719
1012
|
import { SQSClient, GetQueueUrlCommand } from '@aws-sdk/client-sqs';
|
|
720
1013
|
|
|
721
|
-
// Worker IDs embedded at build time
|
|
1014
|
+
// Worker IDs and queue definitions embedded at build time.
|
|
722
1015
|
const WORKER_IDS: string[] = ${JSON.stringify(workers.map((w) => w.id), null, 2)};
|
|
1016
|
+
const QUEUES = ${JSON.stringify(queues.map((q) => ({ id: q.id, steps: q.steps, schedule: q.schedule })), null, 2)};
|
|
723
1017
|
const SERVICE_NAME = ${JSON.stringify(serviceName)};
|
|
724
1018
|
|
|
725
1019
|
export const handler = async (
|
|
@@ -790,6 +1084,7 @@ export const handler = async (
|
|
|
790
1084
|
stage,
|
|
791
1085
|
region,
|
|
792
1086
|
workers,
|
|
1087
|
+
queues: QUEUES,
|
|
793
1088
|
...(debug ? { attemptedQueueNames, errors } : {}),
|
|
794
1089
|
}),
|
|
795
1090
|
};
|
|
@@ -899,7 +1194,7 @@ function processScheduleEvents(scheduleConfig) {
|
|
|
899
1194
|
}
|
|
900
1195
|
return events;
|
|
901
1196
|
}
|
|
902
|
-
function generateServerlessConfig(workers, stage, region, envVars, serviceName) {
|
|
1197
|
+
function generateServerlessConfig(workers, stage, region, envVars, serviceName, calleeIds = /* @__PURE__ */ new Map(), queues = []) {
|
|
903
1198
|
const resources = {
|
|
904
1199
|
Resources: {},
|
|
905
1200
|
Outputs: {}
|
|
@@ -988,6 +1283,21 @@ function generateServerlessConfig(workers, stage, region, envVars, serviceName)
|
|
|
988
1283
|
if (worker.workerConfig?.layers?.length) {
|
|
989
1284
|
functions[functionName].layers = worker.workerConfig.layers;
|
|
990
1285
|
}
|
|
1286
|
+
const callees = calleeIds.get(worker.id);
|
|
1287
|
+
if (callees && callees.size > 0) {
|
|
1288
|
+
const env = {};
|
|
1289
|
+
for (const calleeId of callees) {
|
|
1290
|
+
const calleeWorker = workers.find((w) => w.id === calleeId);
|
|
1291
|
+
if (calleeWorker) {
|
|
1292
|
+
const queueLogicalId = `WorkerQueue${calleeWorker.id.replace(/[^a-zA-Z0-9]/g, "")}${stage}`;
|
|
1293
|
+
const envKey = `WORKER_QUEUE_URL_${sanitizeWorkerIdForEnv(calleeId)}`;
|
|
1294
|
+
env[envKey] = { Ref: queueLogicalId };
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
if (Object.keys(env).length > 0) {
|
|
1298
|
+
functions[functionName].environment = env;
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
991
1301
|
}
|
|
992
1302
|
functions["getDocs"] = {
|
|
993
1303
|
handler: "handlers/docs.handler",
|
|
@@ -1025,8 +1335,21 @@ function generateServerlessConfig(workers, stage, region, envVars, serviceName)
|
|
|
1025
1335
|
}
|
|
1026
1336
|
]
|
|
1027
1337
|
};
|
|
1338
|
+
for (const queue of queues) {
|
|
1339
|
+
if (queue.schedule) {
|
|
1340
|
+
const safeId = queue.id.replace(/[^a-zA-Z0-9]/g, "");
|
|
1341
|
+
const fnName = `queueStarter${safeId}`;
|
|
1342
|
+
const scheduleEvents = processScheduleEvents(queue.schedule);
|
|
1343
|
+
functions[fnName] = {
|
|
1344
|
+
handler: `handlers/queue-starter-${safeId}.handler`,
|
|
1345
|
+
timeout: 60,
|
|
1346
|
+
memorySize: 128,
|
|
1347
|
+
events: scheduleEvents
|
|
1348
|
+
};
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1028
1351
|
const safeEnvVars = {};
|
|
1029
|
-
const allowedPrefixes = ["OPENAI_", "ANTHROPIC_", "DATABASE_", "MONGODB_", "REDIS_", "WORKERS_", "REMOTION_"];
|
|
1352
|
+
const allowedPrefixes = ["OPENAI_", "ANTHROPIC_", "DATABASE_", "MONGODB_", "REDIS_", "UPSTASH_", "WORKER_", "WORKERS_", "WORKFLOW_", "REMOTION_", "QUEUE_JOB_", "DEBUG_WORKER_QUEUES"];
|
|
1030
1353
|
for (const [key, value] of Object.entries(envVars)) {
|
|
1031
1354
|
if (allowedPrefixes.some((prefix) => key.startsWith(prefix))) {
|
|
1032
1355
|
safeEnvVars[key] = value;
|
|
@@ -1179,7 +1502,9 @@ async function build2(args) {
|
|
|
1179
1502
|
workers.map((w) => w.filePath),
|
|
1180
1503
|
process.cwd()
|
|
1181
1504
|
);
|
|
1182
|
-
const
|
|
1505
|
+
const jobStoreType = getJobStoreType();
|
|
1506
|
+
const filteredDeps = filterDepsForJobStore(runtimeDeps, jobStoreType);
|
|
1507
|
+
const dependencies = buildDependenciesMap(process.cwd(), filteredDeps);
|
|
1183
1508
|
const packageJson = {
|
|
1184
1509
|
name: "ai-router-workers",
|
|
1185
1510
|
version: "1.0.0",
|
|
@@ -1238,8 +1563,13 @@ async function build2(args) {
|
|
|
1238
1563
|
console.warn(chalk.yellow("\u26A0\uFE0F Failed to parse microfox.json, using default service name"));
|
|
1239
1564
|
}
|
|
1240
1565
|
}
|
|
1566
|
+
const queues = await scanQueues(aiPath);
|
|
1567
|
+
if (queues.length > 0) {
|
|
1568
|
+
console.log(chalk.blue(`\u2139\uFE0F Found ${queues.length} queue(s): ${queues.map((q) => q.id).join(", ")}`));
|
|
1569
|
+
generateQueueRegistry(queues, serverlessDir, process.cwd());
|
|
1570
|
+
}
|
|
1241
1571
|
ora("Generating handlers...").start().succeed("Generated handlers");
|
|
1242
|
-
await generateHandlers(workers, serverlessDir);
|
|
1572
|
+
await generateHandlers(workers, serverlessDir, queues);
|
|
1243
1573
|
const extractSpinner = ora("Extracting worker configs from bundled handlers...").start();
|
|
1244
1574
|
for (const worker of workers) {
|
|
1245
1575
|
try {
|
|
@@ -1286,17 +1616,24 @@ async function build2(args) {
|
|
|
1286
1616
|
}
|
|
1287
1617
|
}
|
|
1288
1618
|
extractSpinner.succeed("Extracted configs");
|
|
1289
|
-
generateWorkersConfigHandler(serverlessDir, workers, serviceName);
|
|
1619
|
+
generateWorkersConfigHandler(serverlessDir, workers, serviceName, queues);
|
|
1290
1620
|
generateDocsHandler(serverlessDir, serviceName, stage, region);
|
|
1291
1621
|
generateTriggerHandler(serverlessDir, serviceName);
|
|
1292
|
-
|
|
1622
|
+
for (const queue of queues) {
|
|
1623
|
+
if (queue.schedule) {
|
|
1624
|
+
generateQueueStarterHandler(serverlessDir, queue, serviceName);
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
let calleeIds = await collectCalleeWorkerIds(workers, process.cwd());
|
|
1628
|
+
calleeIds = mergeQueueCallees(calleeIds, queues, workers);
|
|
1629
|
+
const config = generateServerlessConfig(workers, stage, region, envVars, serviceName, calleeIds, queues);
|
|
1293
1630
|
const envStage = fs.existsSync(microfoxJsonPath) ? "prod" : stage;
|
|
1294
1631
|
const safeEnvVars = {
|
|
1295
1632
|
ENVIRONMENT: envStage,
|
|
1296
1633
|
STAGE: envStage,
|
|
1297
1634
|
NODE_ENV: envStage
|
|
1298
1635
|
};
|
|
1299
|
-
const allowedPrefixes = ["OPENAI_", "ANTHROPIC_", "DATABASE_", "MONGODB_", "REDIS_", "WORKERS_", "REMOTION_"];
|
|
1636
|
+
const allowedPrefixes = ["OPENAI_", "ANTHROPIC_", "DATABASE_", "MONGODB_", "REDIS_", "UPSTASH_", "WORKER_", "WORKERS_", "WORKFLOW_", "REMOTION_", "QUEUE_JOB_", "DEBUG_WORKER_QUEUES"];
|
|
1300
1637
|
for (const [key, value] of Object.entries(envVars)) {
|
|
1301
1638
|
if (key.startsWith("AWS_")) continue;
|
|
1302
1639
|
if (allowedPrefixes.some((prefix) => key.startsWith(prefix)) || referencedEnvKeys.has(key)) {
|
|
@@ -1370,10 +1707,86 @@ var pushCommand = new Command().name("push").description("Build and deploy backg
|
|
|
1370
1707
|
await deploy(options);
|
|
1371
1708
|
});
|
|
1372
1709
|
|
|
1710
|
+
// src/commands/new.ts
|
|
1711
|
+
import { Command as Command2 } from "commander";
|
|
1712
|
+
import * as fs2 from "fs";
|
|
1713
|
+
import * as path2 from "path";
|
|
1714
|
+
import chalk2 from "chalk";
|
|
1715
|
+
import ora2 from "ora";
|
|
1716
|
+
var newCommand = new Command2().name("new").description("Scaffold a new background worker file").argument("<id>", "Worker ID (used as the worker id and filename)").option("--dir <path>", "Directory for the worker file", "app/ai/workers").option("--schedule <expression>", 'Optional schedule expression (e.g. "cron(0 3 * * ? *)" or "rate(1 hour)")').option("--timeout <seconds>", "Lambda timeout in seconds", "300").option("--memory <mb>", "Lambda memory size in MB", "512").action((id, options) => {
|
|
1717
|
+
const spinner = ora2("Scaffolding worker...").start();
|
|
1718
|
+
try {
|
|
1719
|
+
const projectRoot = process.cwd();
|
|
1720
|
+
const dir = path2.resolve(projectRoot, options.dir || "app/ai/workers");
|
|
1721
|
+
if (!fs2.existsSync(dir)) {
|
|
1722
|
+
fs2.mkdirSync(dir, { recursive: true });
|
|
1723
|
+
}
|
|
1724
|
+
const fileSafeId = id.trim().replace(/[^a-zA-Z0-9_-]+/g, "-");
|
|
1725
|
+
const filePath = path2.join(dir, `${fileSafeId}.worker.ts`);
|
|
1726
|
+
if (fs2.existsSync(filePath)) {
|
|
1727
|
+
spinner.fail(`File already exists: ${path2.relative(projectRoot, filePath)}`);
|
|
1728
|
+
process.exitCode = 1;
|
|
1729
|
+
return;
|
|
1730
|
+
}
|
|
1731
|
+
const timeout = Number(options.timeout || "300") || 300;
|
|
1732
|
+
const memorySize = Number(options.memory || "512") || 512;
|
|
1733
|
+
const scheduleLine = options.schedule ? ` schedule: '${options.schedule}',
|
|
1734
|
+
` : "";
|
|
1735
|
+
const contents = `import { createWorker, type WorkerConfig } from '@microfox/ai-worker';
|
|
1736
|
+
import { z } from 'zod';
|
|
1737
|
+
import type { WorkerHandlerParams } from '@microfox/ai-worker/handler';
|
|
1738
|
+
|
|
1739
|
+
const InputSchema = z.object({
|
|
1740
|
+
// TODO: define input fields
|
|
1741
|
+
});
|
|
1742
|
+
|
|
1743
|
+
const OutputSchema = z.object({
|
|
1744
|
+
// TODO: define output fields
|
|
1745
|
+
});
|
|
1746
|
+
|
|
1747
|
+
type Input = z.infer<typeof InputSchema>;
|
|
1748
|
+
type Output = z.infer<typeof OutputSchema>;
|
|
1749
|
+
|
|
1750
|
+
export const workerConfig: WorkerConfig = {
|
|
1751
|
+
timeout: ${timeout},
|
|
1752
|
+
memorySize: ${memorySize},
|
|
1753
|
+
${scheduleLine}};
|
|
1754
|
+
|
|
1755
|
+
export default createWorker<typeof InputSchema, Output>({
|
|
1756
|
+
id: '${id}',
|
|
1757
|
+
inputSchema: InputSchema,
|
|
1758
|
+
outputSchema: OutputSchema,
|
|
1759
|
+
async handler({ input, ctx }: WorkerHandlerParams<Input, Output>) {
|
|
1760
|
+
const { jobId, workerId, jobStore, dispatchWorker } = ctx;
|
|
1761
|
+
console.log('[${id}] start', { jobId, workerId });
|
|
1762
|
+
|
|
1763
|
+
await jobStore?.update({ status: 'running' });
|
|
1764
|
+
|
|
1765
|
+
// TODO: implement your business logic here
|
|
1766
|
+
const result: Output = {} as any;
|
|
1767
|
+
|
|
1768
|
+
await jobStore?.update({ status: 'completed', output: result });
|
|
1769
|
+
return result;
|
|
1770
|
+
},
|
|
1771
|
+
});
|
|
1772
|
+
`;
|
|
1773
|
+
fs2.writeFileSync(filePath, contents, "utf-8");
|
|
1774
|
+
spinner.succeed(
|
|
1775
|
+
`Created worker: ${chalk2.cyan(path2.relative(projectRoot, filePath))}
|
|
1776
|
+
Next: run ${chalk2.yellow("npx @microfox/ai-worker-cli@latest push")} to build & deploy your workers.`
|
|
1777
|
+
);
|
|
1778
|
+
} catch (error) {
|
|
1779
|
+
spinner.fail("Failed to scaffold worker");
|
|
1780
|
+
console.error(chalk2.red(error?.stack || error?.message || String(error)));
|
|
1781
|
+
process.exitCode = 1;
|
|
1782
|
+
}
|
|
1783
|
+
});
|
|
1784
|
+
|
|
1373
1785
|
// src/index.ts
|
|
1374
|
-
var program = new
|
|
1375
|
-
program.name("ai-worker").description("CLI tooling for deploying ai-router background workers").version("
|
|
1786
|
+
var program = new Command3();
|
|
1787
|
+
program.name("ai-worker").description("CLI tooling for deploying ai-router background workers").version("1.0.0");
|
|
1376
1788
|
program.addCommand(pushCommand);
|
|
1789
|
+
program.addCommand(newCommand);
|
|
1377
1790
|
program.parse(process.argv);
|
|
1378
1791
|
var aiWorkerCli = program;
|
|
1379
1792
|
export {
|