langaro-api 1.0.4 → 1.0.6

package/lib/cli/init.js

@@ -1,9 +1,24 @@
 const fs = require('fs');
 const path = require('path');
 const readline = require('readline');
+const crypto = require('crypto');
 
 const OWN_VERSION = require('../../package.json').version;
 
+function generateUUID() {
+  return crypto.randomUUID();
+}
+
+function generateRandomString(length = 24) {
+  const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+  let result = '';
+  const bytes = crypto.randomBytes(length);
+  for (let i = 0; i < length; i++) {
+    result += chars[bytes[i] % chars.length];
+  }
+  return result;
+}
+
 function ask(rl, question, defaultVal) {
   const suffix = defaultVal ? ` (${defaultVal})` : '';
   return new Promise((resolve) => {
@@ -199,11 +214,11 @@ newrelic_agent.log
 @types/generated/
 `;
 
-function envContent(port, dbName, redisPort) {
+function envContent(port, dbName, redisPort, { jwtSecret = '', masterPass = '' } = {}) {
   return `NODE_ENV="development"
 TZ="UTC"
 PORT="${port}"
-APP_URL="http://localhost:${port}"
+APP_URL="http://localhost:3000"
 API_URL="http://localhost:${port}"
 
 # Database
@@ -216,13 +231,13 @@ DB_PORT="3306"
 # Redis
 REDIS_HOST="127.0.0.1"
 REDIS_PORT="${redisPort}"
-REDIS_PASSWORD=""
+# REDIS_PASSWORD=""
 # REDIS_USERNAME=""
 # REDIS_TLS=""
 
 # Security
-JWT_SECRET=""
-MASTER_PASS=""
+JWT_SECRET="${jwtSecret}"
+MASTER_PASS="${masterPass}"
 
 # Queue Management
 BULLBOARD_PATH="/admin/queues"
@@ -620,19 +635,16 @@ module.exports = function (services) {
 };
 `;
 
-const UTILS_INDEX = `const { ApiError } = require('trigger-api-error');
-const { StatusCodes } = require('http-status-codes');
-const { validateRequiredFields } = require('validate-required-fields');
-const { validateFieldsFormat } = require('validate-fields-format');
-const { fieldShouldNotBeSent } = require('forbid-fields-sent');
+const UTILS_INDEX = `export * from 'validate-required-fields';
+export * from 'trigger-api-error';
+export * from 'http-status-codes';
+export * from 'forbid-fields-sent';
+export * from 'validate-fields-format';
+export * from './sleep';
+`;
 
-module.exports = {
-  ApiError,
-  StatusCodes,
-  validateRequiredFields,
-  validateFieldsFormat,
-  fieldShouldNotBeSent,
-};
+const UTILS_SLEEP = `// eslint-disable-next-line no-promise-executor-return
+exports.sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 `;
 
 const GLOBAL_SETUP_TESTS = `require('dotenv').config({ path: '.env.test' });
@@ -648,6 +660,319 @@ const matchers = require('jest-extended');
 expect.extend(matchers);
 `;
 
+const QUEUES_JS = `require('dotenv').config({
+  path: process.env.NODE_ENV === 'test' ? '.env.test' : '.env',
+});
+
+const Sentry = require('@/config/instrument');
+const redisConfig = require('@/database/redis-config');
+const IORedis = require('ioredis');
+const { Queue, Worker } = require('bullmq');
+const { BullMQAdapter } = require('@bull-board/api/bullMQAdapter');
+const { sleep } = require('@/utils');
+
+const isTestEnv = process.env.NODE_ENV === 'test';
+const inactivityCheckIntervalMs = 1 * 60 * 1000;
+const inactivityTimeLimitMs = 10 * 60 * 1000;
+
+let isShuttingDown = false;
+let connection;
+const jobConfigs = new Map();
+const activeQueues = new Map();
+let ioRef;
+let bullBoardUpdateQueuesRef;
+let cleanupIntervalId = null;
+
+function createRedisConnection() {
+  const connectionInstance = new IORedis(redisConfig.connection);
+  connectionInstance.on('error', (error) => {
+    console.error('Redis Connection Error:', error);
+    Sentry.captureException(error, { tags: { service: 'redis' } });
+  });
+  return connectionInstance;
+}
+
+function getDefaultJobOptions() {
+  return {
+    attempts: 100,
+    backoff: { type: 'fixed', delay: 1000 * 60 },
+    removeOnComplete: { age: 60 * 60 * 24 * 30, count: 1000 },
+    removeOnFail: { age: 60 * 60 * 24 * 90, count: 5000 },
+  };
+}
+
+function getDefaultWorkerOptions() {
+  return {
+    autorun: false,
+    limiter: { max: 2, duration: 1000 },
+    maxStalledCount: 5,
+  };
+}
+
+function updateBullBoard() {
+  if (bullBoardUpdateQueuesRef && typeof bullBoardUpdateQueuesRef === 'function') {
+    bullBoardUpdateQueuesRef(Array.from(activeQueues.values()).map((qd) => new BullMQAdapter(qd.queue)));
+  }
+}
+
+async function getOrCreateQueueAndWorker(queueName, groupId, baseConfig) {
+  const queueNameWithGroupId = \`\${queueName}\${groupId ? \`-\${groupId}\` : ''}\`;
+
+  if (activeQueues.has(queueNameWithGroupId)) {
+    const existing = activeQueues.get(queueNameWithGroupId);
+    existing.lastActivity = Date.now();
+    return existing;
+  }
+
+  console.log(\`Creating queue and worker for: \${queueNameWithGroupId}\`);
+
+  const finalJobOptions = { ...getDefaultJobOptions(), ...(baseConfig.jobOptions || {}) };
+  const queue = new Queue(queueNameWithGroupId, { connection, defaultJobOptions: finalJobOptions });
+
+  const finalWorkerOptions = { connection, ...getDefaultWorkerOptions(), ...(baseConfig.workerOptions || {}) };
+  const worker = new Worker(
+    queueNameWithGroupId,
+    async (jobObj) => {
+      const queueData = activeQueues.get(queueNameWithGroupId);
+      if (queueData) { queueData.lastActivity = Date.now(); }
+      // eslint-disable-next-line no-use-before-define
+      await baseConfig.handle(jobObj.data, jobObj, { add }, ioRef);
+    },
+    finalWorkerOptions,
+  );
+
+  worker.on('failed', (job, error) => {
+    Sentry.captureException(error, {
+      tags: { queue: queueNameWithGroupId },
+      contexts: { data: { 'Job data': job?.data, queue: queueNameWithGroupId } },
+    });
+  });
+
+  worker.on('error', (error) => {
+    Sentry.captureException(error, { tags: { queue: queueNameWithGroupId, event: 'worker-error' } });
+  });
+
+  const queueData = {
+    queue, worker, lastActivity: Date.now(), name: queueNameWithGroupId, queueName, groupId,
+  };
+
+  if (!isShuttingDown) {
+    activeQueues.set(queueNameWithGroupId, queueData);
+    worker.run();
+    console.log(\`Worker running for: \${queueNameWithGroupId}\`);
+  }
+
+  updateBullBoard();
+  return queueData;
+}
+
+// eslint-disable-next-line no-unused-vars
+async function add(queueName, data, options = {}) {
+  const baseConfig = jobConfigs.get(queueName);
+  if (!baseConfig) {
+    throw new Error(\`Queue configuration not found for: \${queueName}\`);
+  }
+
+  try {
+    if (baseConfig.group && !options.groupId) {
+      throw new Error(\`Missing groupId for grouped queue: \${queueName}\`);
+    }
+
+    const queueData = await getOrCreateQueueAndWorker(queueName, options.groupId, baseConfig);
+    await queueData.queue.add(queueData.name, data, options);
+  } catch (error) {
+    console.error(\`Failed to add job to queue \${queueName}:\`, error);
+    Sentry.captureException(error, { tags: { queue: queueName, operation: 'addJob' } });
+    throw error;
+  }
+}
+
+async function closeInactiveQueues() {
+  const now = Date.now();
+  const queuesToRemove = [];
+
+  const checks = Array.from(activeQueues.entries()).map(async ([queueName, queueData]) => {
+    if (now - queueData.lastActivity <= inactivityTimeLimitMs) return;
+    try {
+      const [activeCount, waitingCount, delayedCount] = await Promise.all([
+        queueData.queue.getActiveCount(),
+        queueData.queue.getWaitingCount(),
+        queueData.queue.getDelayedCount(),
+      ]);
+      if (activeCount === 0 && waitingCount === 0 && delayedCount === 0) {
+        queuesToRemove.push(queueName);
+      }
+    } catch (error) {
+      Sentry.captureException(error, { tags: { queue: queueName, operation: 'checkBeforeClose' } });
+    }
+  });
+
+  await Promise.allSettled(checks);
+  if (queuesToRemove.length === 0) return;
+
+  await Promise.allSettled(
+    queuesToRemove.map(async (queueName) => {
+      const queueData = activeQueues.get(queueName);
+      if (!queueData) return;
+      try {
+        await queueData.worker.close();
+        await queueData.queue.close();
+        activeQueues.delete(queueName);
+      } catch (error) {
+        Sentry.captureException(error, { tags: { queue: queueName, operation: 'closeInactive' } });
+      }
+    }),
+  );
+
+  updateBullBoard();
+}
+
+function startCleanupInterval() {
+  if (cleanupIntervalId) clearInterval(cleanupIntervalId);
+  if (!isTestEnv) {
+    cleanupIntervalId = setInterval(closeInactiveQueues, inactivityCheckIntervalMs);
+  }
+}
+
+async function startWorkersForExistingQueues() {
+  if (!connection) return;
+
+  const stream = connection.scanStream({ match: 'bull:*:meta', count: 1000 });
+  const potentialQueueNames = new Set();
+
+  stream.on('data', (keys) => {
+    (keys || []).forEach((key) => {
+      const match = key.match(/^bull:(.+):meta$/);
+      if (match && match[1]) potentialQueueNames.add(match[1]);
+    });
+  });
+
+  try {
+    await new Promise((resolve, reject) => { stream.on('end', resolve); stream.on('error', reject); });
+  } catch (error) {
+    Sentry.captureException(error, { tags: { operation: 'startWorkersForExistingQueues' } });
+    return;
+  }
+
+  if (potentialQueueNames.size === 0) return;
+
+  const sortedConfigKeys = Array.from(jobConfigs.keys()).sort((a, b) => b.length - a.length);
+
+  await Promise.all(
+    Array.from(potentialQueueNames).map(async (fullQueueName) => {
+      let baseName = null;
+      let groupId = null;
+      let config = null;
+
+      sortedConfigKeys.some((potentialBase) => {
+        const baseConfig = jobConfigs.get(potentialBase);
+        if (fullQueueName === potentialBase && !baseConfig.group) {
+          baseName = potentialBase;
+          config = baseConfig;
+          return true;
+        }
+        if (baseConfig.group && fullQueueName.startsWith(\`\${potentialBase}-\`)) {
+          baseName = potentialBase;
+          groupId = fullQueueName.slice(potentialBase.length + 1);
+          config = baseConfig;
+          return true;
+        }
+        return false;
+      });
+
+      if (!config || activeQueues.has(fullQueueName)) return;
+
+      try {
+        const activeCount = await connection.llen(\`bull:\${fullQueueName}:active\`);
+        const waitCount = await connection.llen(\`bull:\${fullQueueName}:wait\`);
+        const delayedCount = await connection.zcard(\`bull:\${fullQueueName}:delayed\`);
+
+        if (waitCount > 0 || delayedCount > 0 || activeCount > 0) {
+          await getOrCreateQueueAndWorker(baseName, groupId, config);
+        }
+      } catch (error) {
+        Sentry.captureException(error, { tags: { queue: fullQueueName, operation: 'startupScanInit' } });
+      }
+    }),
+  );
+}
+
+async function initializeQueues(services, socketIo, bullBoardReplaceQueues) {
+  if (isTestEnv) { return { add: jest.fn() }; }
+
+  console.log('Queue system initializing...');
+
+  connection = createRedisConnection();
+  ioRef = socketIo;
+  bullBoardUpdateQueuesRef = bullBoardReplaceQueues;
+
+  const { loadJobs } = require('langaro-api');
+  const jobs = loadJobs(services);
+  Object.entries(jobs).forEach(([name, job]) => {
+    jobConfigs.set(name, {
+      handle: job.handle,
+      jobOptions: job.jobOptions || {},
+      workerOptions: job.workerOptions || {},
+      group: job.group,
+    });
+  });
+
+  await startWorkersForExistingQueues();
+  startCleanupInterval();
+
+  console.log('Queue system initialized.');
+  return { add };
+}
+
+async function shutdownQueues() {
+  if (isShuttingDown) return;
+  isShuttingDown = true;
+
+  console.log('Gracefully shutting down queues...');
+
+  if (cleanupIntervalId) {
+    clearInterval(cleanupIntervalId);
+    cleanupIntervalId = null;
+  }
+
+  const activeQueuesValues = Array.from(activeQueues.values());
+
+  await Promise.allSettled(activeQueuesValues.map(async (qd) => {
+    try { await qd.worker.pause(); } catch (e) { console.error(\`Error pausing \${qd.name}:\`, e); }
+  }));
+
+  let retries = 0;
+  const maxRetries = 400;
+  while (retries++ < maxRetries) {
+    const stillActive = await Promise.all(activeQueuesValues.map((qd) => qd.queue.getActiveCount()));
+    if (stillActive.reduce((a, b) => a + b, 0) === 0) break;
+    await sleep(3000);
+  }
+
+  await Promise.allSettled(activeQueuesValues.map(async (qd) => {
+    try {
+      await qd.worker.close();
+      await qd.queue.close();
+    } catch (e) {
+      Sentry.captureException(e, { tags: { queue: qd.name, operation: 'shutdown' } });
+    }
+  }));
+
+  activeQueues.clear();
+  jobConfigs.clear();
+
+  if (connection) {
+    try { await connection.quit(); } catch (e) {
+      Sentry.captureException(e, { tags: { service: 'redis', operation: 'quit' } });
+    }
+  }
+
+  console.log('Queue shutdown complete.');
+}
+
+module.exports = { initializeQueues, shutdownQueues };
+`;
+
 const MODELS_INDEX = `const { loadModels } = require('langaro-api');
 
 module.exports = (knexInstance) => loadModels(knexInstance, __dirname);
@@ -726,12 +1051,16 @@ async function run() {
   writeFile(root, 'jest.config.js', JEST_CONFIG);
   writeFile(root, '.gitignore', GITIGNORE);
   writeFile(root, '.env.example', envContent(port, dbName, redisPort));
-  writeFile(root, '.env', envContent(port, dbName, redisPort));
+  writeFile(root, '.env', envContent(port, dbName, redisPort, {
+    jwtSecret: generateUUID(),
+    masterPass: generateRandomString(24),
+  }));
 
   // src/config/
   writeFile(root, 'src/config/server.js', serverJs(port));
   writeFile(root, 'src/config/app.js', APP_JS);
   writeFile(root, 'src/config/instrument.js', INSTRUMENT_JS);
+  writeFile(root, 'src/config/queues.js', QUEUES_JS);
   writeFile(root, 'src/config/tests/global-setup-tests.js', GLOBAL_SETUP_TESTS);
   writeFile(root, 'src/config/tests/setup-tests.js', SETUP_TESTS);
 
@@ -753,6 +1082,7 @@ async function run() {
 
   // src/utils
   writeFile(root, 'src/utils/index.js', UTILS_INDEX);
+  writeFile(root, 'src/utils/sleep.js', UTILS_SLEEP);
 
   // Empty directories
   ['src/integrations', 'src/static', 'src/temp', 'src/database/tests'].forEach((dir) => {
@@ -762,7 +1092,7 @@ async function run() {
   console.log(`\n\x1b[32mProject scaffolded!\x1b[0m\n`);
   console.log('Next steps:');
   console.log(`  1. \x1b[36mnpm install\x1b[0m`);
-  console.log(`  2. Fill in your secrets in \x1b[36m.env\x1b[0m (JWT_SECRET, DB_PASSWORD, etc.)`);
+  console.log(`  2. Fill in your \x1b[36m.env\x1b[0m (DB_PASSWORD, etc.)`);
   console.log(`  3. Create the \x1b[36m${dbName}\x1b[0m database in MySQL`);
   console.log(`  4. \x1b[36mnpm run dev\x1b[0m`);
   console.log(`  5. \x1b[36mnpx langaro-api new\x1b[0m to create your first resource\n`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langaro-api",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "description": "Auto-generate TypeScript types, JSDoc annotations, and boilerplate loaders for knex-extended-crud projects",
   "main": "lib/index.js",
   "bin": {
@@ -21,7 +21,7 @@
     "boilerplate"
   ],
   "peerDependencies": {
-    "knex-extended-crud": ">=2.0.0",
+    "knex-extended-crud": ">=2.0.31",
     "cron": ">=3.0.0"
   },
   "peerDependenciesMeta": {