@microlight/core 0.2.0

package/README.md

@@ -0,0 +1,77 @@
+![Logo](/public/logo500.png)
+
+- [Docs](/docs/docs.md)
+- [Change Log](/docs/changelog.md)
+
+## About
+Microlight is a simple task(cron and adhoc) runner. The idea is this, there are tasks that needs to be done. For developers its easy to write a small script to execute the task - e.g. restart AWS redis. But to ask marketing team/sales team to restart redis by running a script is not practical. At the same time, as a small team, the devops guy getting called by the sales team to restart redis is also not practical. 
+
+This is a solution for small teams. This enables tech team to offload operational tasks to the actual team who needs the task to be executed. This also allows the stack to be light weight and tech teams can write simple scripts to do this. 
+
+Example usecases, 
+- restart redis/some server when overloaded. 
+- get fresh data . e.g. customer support person wants to know if customer has made a payment. Lets assume your system is not realtime and the CS person wants to trigger a sync. 
+- cron jobs
+
+Microlight is designed with the following principles in mind
+- Microlight is designed for light work loads such as 
+  - personal use, 
+  - small company - less than 10 people 
+  - simple data pipeline usecases
+  - simple automation/devops usecases
+- Running tasks should feel simple and easy
+  - Developers gets to write scripts to automate a task. 
+  - Business team members gets to trigger the job at their convinience using a UI. 
+  - Webhooks for other subsystems to trigger jobs
+- Setting up and running microlight should be very easy and should have small footprint
+  - Single instance by design - no cluster config overhead
+  - uses sqlite only - host everything in one single server/container
+
+
+Microlight also provides a UI for users to run cron jobs
+
+Ways to trigger a job: 
+- Trigger jobs from UI
+- Trigger jobs via webhooks
+- Trigger jos from cronjob 
+
+
+### Microlight can do
+- Execution logic has to be written with code (will not no support no code ever)
+- Tasks can be executed by non techincal people from a GUI
+- Tasks can be scheduled 
+- Show console logs and execution output
+
+thats it
+
+### Microlight cannot and will not do
+- No code support
+- Multiple server or master/worker mode
+
+
+
+### Simplicity
+
+Microlight is designed as simple solution. This is not designed for massive scale. Intentionally Microlight does not support master/worker node. The Microlight server and the execution of task happens on the same machine. If you are looking for a multi node setup, then we recomment using rundeck. For most small startups, rundeck is an overkill. Scaling microlight can only be done by vertical scaling (increasing the size of the server). Microlight does not support horizontal scaling. Microlight is **NOT** designed to scale horizontally. This is intentional. 
+
+Microlight also uses sqlite. So you can run all microlight dependancies on one machine/container. 
+
+
+## Alternatives to microlight
+
+- trigger.dev
+- rundeck.com
+- activepieces.com
+
+Microlight is the stripped down, simple version of these meant to be run on a single instance. 
+
+Compared to these solutions, 
+- microlight does **NOT** support horizontal scaling
+- microlight does **NOT** have a hosted solution
+- microlight does **NOT** support drag and drop UI tasks. 
+- microlight requires you to write scripts using a text editor and commit the code
+- microlight is suitable for one man tech team
+
+You should choose one of the alternatives if any of the above points are deal breakers. 
+
+

package/bin/microlight-core.js

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+
+import { Command } from "commander";
+import path from "path";
+
+import { packageUp } from "package-up";
+
+import { prepareTasks } from "../scripts/prepareTasks.js";
+import { prepareServer } from "../scripts/prepareServer.js";
+
+
+
+const program = new Command();
+
+let basePath = await packageUp();
+basePath = basePath.split("/");
+basePath.pop();
+const tasksDir = path.join("/", ...basePath, "src", "tasks");
+
+program
+  .name("microlight-core")
+  .description("Microlight core utilities")
+  .version("1.0.0");
+
+// Define the `prepare` command
+const prepareCommand = program
+  .command("prepare")
+  .description("Prepare the environment");
+// .action(() => {
+//   console.log("Preparing environment...");
+// });
+
+prepareCommand
+  .command("all")
+  .description("Perform all prepare commands")
+  .action(async() => {
+    await prepareTasks();
+    console.log('\n');
+    const { prepareFolders } = await import("../scripts/prepareFolders.js");
+    await prepareFolders();
+    
+    console.log('\n');
+    await prepareServer();
+  });
+
+prepareCommand
+  .command("tasks")
+  .description("Index tasks and create an importer")
+  .action(async () => {
+    await prepareTasks();
+  });
+
+prepareCommand
+  .command("folders")
+  .description("Index folders")
+  .action(async () => {
+    const { prepareFolders } = await import("../scripts/prepareFolders.js");
+    await prepareFolders();
+  });
+
+
+prepareCommand
+  .command("server")
+  .description("Setup the server")
+  .action(async () => {
+    prepareServer();
+  });
+
+// Parse CLI arguments
+program.parse(process.argv);

package/dist/scripts/generate-folder-index.js

@@ -0,0 +1,120 @@
+#!/usr/bin/env node
+import { glob } from 'glob';
+import path from 'path';
+import fs from 'fs';
+// import taskMap from '@/tasks/tasks';
+
+// Ensure we're looking in the right directory relative to the script
+const tasksDir = path.join(process.cwd(), 'src', 'tasks');
+const outputFile = path.resolve(process.cwd(), '.microlight', 'folderMap.js');
+const taskMapFile = path.resolve(process.cwd(), '.microlight', 'taskMap.js');
+const taskMap = (await import(taskMapFile))?.default;
+let taskMapByFileName = {};
+Object.keys(taskMap).forEach(function (slug) {
+  const task = taskMap[slug];
+  taskMapByFileName[task.__folder + '/' + task.__file_name] = task;
+});
+
+// console.log(taskMapByFileName);
+
+// Create tasks directory if it doesn't exist
+if (!fs.existsSync(tasksDir)) {
+  fs.mkdirSync(tasksDir, {
+    recursive: true
+  });
+}
+
+// Find all folder files
+const folderPatterns = ['__ml.js', 'ml.js', 'ml.folder.js', '**/microlight.folder.js'];
+const folderFiles = glob.sync(folderPatterns, {
+  cwd: tasksDir,
+  absolute: false
+});
+
+// console.log(folderFiles);
+
+// Import all task files dynamically
+const folders = await Promise.all(folderFiles.map(async file => {
+  const filePath = path.resolve(tasksDir, file);
+  let folderName = file.split('/');
+  folderName.pop();
+  folderName = folderName.join('/');
+  console.log(folderName);
+  let folder = (await import(filePath))?.default;
+  // console.log(folder.default);
+  // const taskName = path.basename(filePath, '.task.js');
+  return [folderName, folder];
+  // return [task?.default?.slug, {...task?.default,...{file_name:taskName}}];
+}));
+// console.log(folders);
+
+// Convert array of entries to an object
+const folderMap = Object.fromEntries(folders);
+// console.log(folderMap);
+
+Object.keys(folderMap).map(folderName => {
+  folderMap[folderName].contents = [];
+  const contentList = fs.readdirSync(path.resolve(tasksDir, folderName));
+  // console.log(contentList);
+  for (const filename of contentList) {
+    const file = path.resolve(tasksDir, folderName, filename);
+    // const file = project_folder+`${dir}/${filename}`
+    // console.log(file);
+    if (fs.statSync(file).isDirectory()) {
+      // console.log('\n\n\n====');
+      // console.log(file);
+      const subFolderName = file.split('/src/tasks/')[1];
+      // console.log(subFolderName);
+      if (folderMap[subFolderName]) {
+        folderMap[folderName].contents.push({
+          type: 'folder',
+          slug: subFolderName,
+          ...folderMap[subFolderName]
+        });
+      }
+    } else {
+      if (filename.indexOf('.task.js') > -1) {
+        // console.log('\n\n\n====');
+        // console.log(filename);
+        const taskName = path.basename(filename, '.task.js');
+        // console.log(taskMapByFileName[folderName+'/'+filename]);
+        let item = {
+          type: 'task',
+          slug: taskName,
+          ...taskMapByFileName[folderName + '/' + filename]
+        };
+        folderMap[folderName].contents.push(item);
+      }
+    }
+    // console.log(filename);
+  }
+});
+// folderMap.map(folder=>{
+
+//     
+//     folder.content = [
+//       {
+//         type:'task',
+//         slug:'takes_time'
+//       },
+//       {
+//         type:'folder',
+//         slug:'1.intro'
+//       }
+//     ]
+// })
+
+// Write the generated code to a file
+const outputCode = 'export default ' + JSON.stringify(folderMap, null, 2);
+
+// Create the output directory if it doesn't exist
+const outputDir = path.dirname(outputFile);
+if (!fs.existsSync(outputDir)) {
+  fs.mkdirSync(outputDir, {
+    recursive: true
+  });
+}
+
+// Write the file
+fs.writeFileSync(outputFile, outputCode, 'utf-8');
+console.log(`Generated folder index at: ${outputFile}`);

package/dist/scripts/generate-task-imports.js

@@ -0,0 +1,64 @@
+#!/usr/bin/env node
+import { glob } from 'glob';
+import path from 'path';
+// import { join, relative, dirname } from 'path';
+import fs from 'fs';
+
+// Ensure we're looking in the right directory relative to the script
+const tasksDir = path.join(process.cwd(), 'src', 'tasks');
+const outputFile = path.resolve(process.cwd(), '.microlight', 'importTaskModule.js');
+
+// Create tasks directory if it doesn't exist
+if (!fs.existsSync(tasksDir)) {
+  fs.mkdirSync(tasksDir, {
+    recursive: true
+  });
+}
+
+// Find all task files
+const taskFiles = glob.sync('**/*.task.js', {
+  cwd: tasksDir,
+  absolute: false
+});
+
+// Generate the import switch statement
+const generateImportSwitch = async tasks => {
+  let cases = await Promise.all(tasks.map(async file => {
+    const filePath = path.resolve(tasksDir, file);
+    const task = await import(filePath);
+    // console.log(task);
+    const taskName = path.basename(file, '.task.js');
+    const taskSlug = task?.default?.slug;
+    // const taskName = path.basename(filePath, '.task.js');
+    // return [taskName, task.default];
+    // return [task?.default?.slug, {...task?.default,...{file_name:taskName}}];
+
+    return `    case "${taskSlug || taskName}":\n      return await import("../src/tasks/${file}");`;
+  }));
+  cases = cases.join('\n');
+  // console.log(cases);
+
+  return `
+export const importTaskModule = async (task_slug) => {
+  switch (task_slug) {
+${cases}
+    default:
+      return { default: { name: "Library", description: "All executable tasks" } };
+  }
+};`.trim();
+};
+
+// Write the generated code to a file
+const outputCode = await generateImportSwitch(taskFiles);
+
+// Create the output directory if it doesn't exist
+const outputDir = path.dirname(outputFile);
+if (!fs.existsSync(outputDir)) {
+  fs.mkdirSync(outputDir, {
+    recursive: true
+  });
+}
+
+// Write the file
+fs.writeFileSync(outputFile, outputCode, 'utf-8');
+console.log(`Generated import switch at: ${outputFile}`);

package/dist/scripts/generate-task-index.js

@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+import { glob } from 'glob';
+import path from 'path';
+// import { join, relative, dirname } from 'path';
+import fs from 'fs';
+
+// Ensure we're looking in the right directory relative to the script
+const tasksDir = path.join(process.cwd(), 'src', 'tasks');
+const outputFile = path.resolve(process.cwd(), '.microlight', 'taskMap.js');
+
+// Create tasks directory if it doesn't exist
+if (!fs.existsSync(tasksDir)) {
+  fs.mkdirSync(tasksDir, {
+    recursive: true
+  });
+}
+
+// Find all task files
+const taskFiles = glob.sync('**/*.task.js', {
+  cwd: tasksDir,
+  absolute: false
+});
+console.log(taskFiles);
+
+// Import all task files dynamically
+const tasks = await Promise.all(taskFiles.map(async file => {
+  const filePath = path.resolve(tasksDir, file);
+  const task = await import(filePath);
+  const taskName = path.basename(filePath);
+  let folderName = file.split('/');
+  folderName.pop();
+  folderName = folderName.join('/');
+  // return [taskName, task.default];
+  return [task?.default?.slug, {
+    ...task?.default,
+    ...{
+      __file_name: taskName,
+      __folder: folderName
+    }
+  }];
+}));
+// console.log(tasks);
+
+// Convert array of entries to an object
+const taskMap = Object.fromEntries(tasks);
+console.log(taskMap);
+
+// Write the generated code to a file
+const outputCode = 'export default ' + JSON.stringify(taskMap, null, 2);
+
+// Create the output directory if it doesn't exist
+const outputDir = path.dirname(outputFile);
+if (!fs.existsSync(outputDir)) {
+  fs.mkdirSync(outputDir, {
+    recursive: true
+  });
+}
+
+// Write the file
+fs.writeFileSync(outputFile, outputCode, 'utf-8');
+console.log(`Generated tasks index at: ${outputFile}`);

package/dist/scripts/prepareFolders.js

@@ -0,0 +1,119 @@
+import { glob } from 'glob';
+import path from 'path';
+// import { join, relative, dirname } from 'path';
+import fs from 'fs';
+const tasksDir = path.join(process.cwd(), 'src', 'tasks');
+const outputFile = path.resolve(process.cwd(), '.microlight', 'folderMap.js');
+const taskMapFile = path.resolve(process.cwd(), '.microlight', 'taskMap.js');
+const taskMap = (await import(taskMapFile))?.default;
+let taskMapByFileName = {};
+Object.keys(taskMap).forEach(function (slug) {
+  const task = taskMap[slug];
+  taskMapByFileName[task.__folder + '/' + task.__file_name] = task;
+});
+
+// Create tasks directory if it doesn't exist
+if (!fs.existsSync(tasksDir)) {
+  fs.mkdirSync(tasksDir, {
+    recursive: true
+  });
+}
+const getFolderFiles = function () {
+  // Find all folder files
+  const folderPatterns = ['__ml.js', 'ml.js', 'ml.folder.js', '**/microlight.folder.js'];
+  const folderFiles = glob.sync(folderPatterns, {
+    cwd: tasksDir,
+    absolute: false
+  });
+  return folderFiles;
+};
+export async function prepareFolders() {
+  console.log('Preparing folders now');
+  const folderFiles = getFolderFiles();
+
+  // Import all task files dynamically
+  const folders = await Promise.all(folderFiles.map(async file => {
+    const filePath = path.resolve(tasksDir, file);
+    let folderName = file.split('/');
+    folderName.pop();
+    folderName = folderName.join('/');
+    // console.log(folderName);
+    let folder = (await import(filePath))?.default;
+    // console.log(folder.default);
+    // const taskName = path.basename(filePath, '.task.js');
+    return [folderName, folder];
+    // return [task?.default?.slug, {...task?.default,...{file_name:taskName}}];
+  }));
+  // console.log(folders);
+
+  // Convert array of entries to an object
+  const folderMap = Object.fromEntries(folders);
+  // console.log(folderMap);
+
+  Object.keys(folderMap).map(folderName => {
+    folderMap[folderName].contents = [];
+    const contentList = fs.readdirSync(path.resolve(tasksDir, folderName));
+    // console.log(contentList);
+    for (const filename of contentList) {
+      const file = path.resolve(tasksDir, folderName, filename);
+      // const file = project_folder+`${dir}/${filename}`
+      // console.log(file);
+      if (fs.statSync(file).isDirectory()) {
+        // console.log('\n\n\n====');
+        // console.log(file);
+        const subFolderName = file.split('/src/tasks/')[1];
+        // console.log(subFolderName);
+        if (folderMap[subFolderName]) {
+          folderMap[folderName].contents.push({
+            type: 'folder',
+            slug: subFolderName,
+            ...folderMap[subFolderName]
+          });
+        }
+      } else {
+        if (filename.indexOf('.task.js') > -1) {
+          // console.log('\n\n\n====');
+          // console.log(filename);
+          const taskName = path.basename(filename, '.task.js');
+          // console.log(taskMapByFileName[folderName+'/'+filename]);
+          let item = {
+            type: 'task',
+            slug: taskName,
+            ...taskMapByFileName[folderName + '/' + filename]
+          };
+          folderMap[folderName].contents.push(item);
+        }
+      }
+      // console.log(filename);
+    }
+  });
+  // folderMap.map(folder=>{
+
+  //     
+  //     folder.content = [
+  //       {
+  //         type:'task',
+  //         slug:'takes_time'
+  //       },
+  //       {
+  //         type:'folder',
+  //         slug:'1.intro'
+  //       }
+  //     ]
+  // })
+
+  // Write the generated code to a file
+  const outputCode = 'export default ' + JSON.stringify(folderMap, null, 2);
+
+  // Create the output directory if it doesn't exist
+  const outputDir = path.dirname(outputFile);
+  if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, {
+      recursive: true
+    });
+  }
+
+  // Write the file
+  fs.writeFileSync(outputFile, outputCode, 'utf-8');
+  console.log(`Generated folder index at: ${outputFile}`);
+}

package/dist/scripts/prepareServer.js

@@ -0,0 +1,34 @@
+import path from "path";
+import fs from "fs";
+import { copySync } from "fs-extra";
+import { createRequire } from "module";
+const require = createRequire(import.meta.url); // Required for resolving CommonJS modules
+
+export async function prepareServer() {
+  console.log('preparing the server');
+  const coreModulePath = require.resolve("@microlight/core/package.json"); // Find package.json
+  const coreRoot = path.dirname(coreModulePath); // Get package root
+  console.log(coreModulePath);
+  console.log(coreRoot);
+  const serverSrcDir = path.join(coreRoot, "dist", "server"); // Resolve /dist/server
+  console.log(serverSrcDir);
+
+  // Destination path
+  const processDir = process.cwd();
+  const serverDestDir = path.join(processDir, ".microlight", "server");
+  console.log("serverSrcDir:", serverSrcDir);
+  console.log("serverDestDir:", serverDestDir);
+
+  // Ensure destination exists
+  if (!fs.existsSync(serverDestDir)) {
+    fs.mkdirSync(serverDestDir, {
+      recursive: true
+    });
+  }
+
+  // Copy files
+  copySync(serverSrcDir, serverDestDir, {
+    overwrite: true
+  });
+  console.log("Server files copied successfully!");
+}

package/dist/scripts/prepareTasks.js

@@ -0,0 +1,114 @@
+import { glob } from 'glob';
+import path from 'path';
+// import { join, relative, dirname } from 'path';
+import fs from 'fs';
+const tasksDir = path.join(process.cwd(), 'src', 'tasks');
+
+// Create tasks directory if it doesn't exist
+if (!fs.existsSync(tasksDir)) {
+  fs.mkdirSync(tasksDir, {
+    recursive: true
+  });
+}
+const getTaskFiles = function () {
+  // Find all task files
+  const taskFiles = glob.sync('**/*.task.js', {
+    cwd: tasksDir,
+    absolute: false
+  });
+
+  // console.log(taskFiles);
+  return taskFiles;
+};
+
+// Generate the import switch statement
+const generateImportSwitch = async tasks => {
+  let cases = await Promise.all(tasks.map(async file => {
+    const filePath = path.resolve(tasksDir, file);
+    const task = await import(filePath);
+    // console.log(task);
+    const taskName = path.basename(file, '.task.js');
+    const taskSlug = task?.default?.slug;
+    // const taskName = path.basename(filePath, '.task.js');
+    // return [taskName, task.default];
+    // return [task?.default?.slug, {...task?.default,...{file_name:taskName}}];
+
+    return `    case "${taskSlug || taskName}":\n      return await import("../src/tasks/${file}");`;
+  }));
+  cases = cases.join('\n');
+  // console.log(cases);
+
+  return `
+export const importTaskModule = async (task_slug) => {
+  switch (task_slug) {
+${cases}
+    default:
+      return { default: { name: "Library", description: "All executable tasks" } };
+  }
+};`.trim();
+};
+export async function prepareTasks() {
+  console.log('Preparing tasks now');
+  await prepareTasksIndex();
+  await prepareTasksImports();
+}
+export async function prepareTasksIndex() {
+  const taskFiles = getTaskFiles();
+
+  // Import all task files dynamically
+  const tasks = await Promise.all(taskFiles.map(async file => {
+    const filePath = path.resolve(tasksDir, file);
+    const task = await import(filePath);
+    const taskName = path.basename(filePath);
+    let folderName = file.split('/');
+    folderName.pop();
+    folderName = folderName.join('/');
+    // return [taskName, task.default];
+    return [task?.default?.slug, {
+      ...task?.default,
+      ...{
+        __file_name: taskName,
+        __folder: folderName
+      }
+    }];
+  }));
+  // console.log(tasks);
+
+  // Convert array of entries to an object
+  const taskMap = Object.fromEntries(tasks);
+
+  // console.log(taskMap);
+
+  // Write the generated code to a file
+  const outputCode = 'export default ' + JSON.stringify(taskMap, null, 2);
+  const outputFile = path.resolve(process.cwd(), '.microlight', 'taskMap.js');
+  // Create the output directory if it doesn't exist
+  const outputDir = path.dirname(outputFile);
+  if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, {
+      recursive: true
+    });
+  }
+
+  // Write the file
+  fs.writeFileSync(outputFile, outputCode, 'utf-8');
+  console.log(`Generated tasks index at: ${outputFile}`);
+}
+export async function prepareTasksImports() {
+  const taskFiles = getTaskFiles();
+
+  // Write the generated code to a file
+  const outputCode = await generateImportSwitch(taskFiles);
+  const outputFile = path.resolve(process.cwd(), '.microlight', 'importTaskModule.js');
+  // Create the output directory if it doesn't exist
+  const outputDir = path.dirname(outputFile);
+  if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, {
+      recursive: true
+    });
+  }
+
+  // Write the file
+  fs.writeFileSync(outputFile, outputCode, 'utf-8');
+  console.log(`Generated importTaskModule at: ${outputFile}`);
+}