@microlight/core 0.2.0

package/dist/server/database/microlight/index.js

@@ -0,0 +1,52 @@
+import { Sequelize } from 'sequelize';
+import Runs from "./tables/Runs.model.js";
+import Logs from "./tables/Logs.model.js";
+import sqlite3 from 'sqlite3';
+import pg from 'pg';
+
+/*======================Initialize Sequelize======================*/
+let sequelize;
+if (process.env.ML_DB_PG) {
+  sequelize = new Sequelize(process.env.ML_DB_PG, {
+    logging: false,
+    dialect: 'postgres',
+    dialectOptions: {
+      ssl: {
+        require: true,
+        // This will force the SSL requirement
+        rejectUnauthorized: false // This is to avoid errors due to self-signed certificates
+      }
+    },
+    dialectModule: pg
+  });
+} else {
+  sequelize = new Sequelize({
+    dialect: 'sqlite',
+    storage: '.microlight/microlight.db',
+    logging: false,
+    dialectModule: sqlite3
+  });
+}
+
+/*======================Initialize models======================*/
+const models = {
+  Runs: Runs(sequelize),
+  Logs: Logs(sequelize)
+  // Schedule: Schedule(sequelize),
+};
+
+//Create db object
+const microlightDB = {
+  ...models,
+  sequelize,
+  Sequelize,
+  // Add sync method as a property using arrow function
+  sync: (...args) => sequelize.sync(...args)
+};
+
+// Only sync database in development mode
+// if (process.env.NODE_ENV === 'development') {
+await microlightDB.sync();
+// }
+
+export default microlightDB;

package/dist/server/database/microlight/tables/Logs.model.js

@@ -0,0 +1,34 @@
+import { DataTypes } from 'sequelize';
+export default sequelize => {
+  return sequelize.define('Logs', {
+    id: {
+      type: DataTypes.INTEGER,
+      primaryKey: true,
+      autoIncrement: true
+      // defaultValue: sequelize.literal("nextval('stock_consumption_id_seq'::regclass)")
+    },
+    created_at: {
+      type: DataTypes.DATE,
+      allowNull: false,
+      defaultValue: DataTypes.NOW
+    },
+    type: {
+      // can be log,json,markdown
+      type: DataTypes.TEXT,
+      allowNull: true
+    },
+    content: {
+      // can be pending, running, complete, failed, timeout
+      type: DataTypes.TEXT,
+      allowNull: true
+    },
+    run: {
+      type: DataTypes.INTEGER,
+      allowNull: false
+    }
+  }, {
+    tableName: 'logs',
+    timestamps: false
+    // schema: 'public'
+  });
+};

package/dist/server/database/microlight/tables/Runs.model.js

@@ -0,0 +1,61 @@
+import { DataTypes } from 'sequelize';
+export default sequelize => {
+  return sequelize.define('Runs', {
+    id: {
+      type: DataTypes.INTEGER,
+      primaryKey: true,
+      autoIncrement: true
+      // defaultValue: sequelize.literal("nextval('stock_consumption_id_seq'::regclass)")
+    },
+    task: {
+      type: DataTypes.TEXT,
+      allowNull: true
+    },
+    status: {
+      // can be pending, running, complete, failed, timeout
+      type: DataTypes.TEXT,
+      allowNull: true
+    },
+    started_at: {
+      type: DataTypes.DATE,
+      allowNull: true
+    },
+    completed_at: {
+      type: DataTypes.DATE,
+      allowNull: true
+    },
+    inputs: {
+      type: DataTypes.JSON,
+      allowNull: true,
+      get() {
+        const inputs = this.getDataValue('inputs');
+        return inputs ? typeof inputs === 'string' ? JSON.parse(inputs) : inputs : null;
+      }
+    },
+    triggered_by: {
+      // user,api,webhook,schedule
+      type: DataTypes.TEXT,
+      allowNull: true
+    },
+    created_at: {
+      type: DataTypes.DATE,
+      allowNull: false,
+      defaultValue: DataTypes.NOW
+    },
+    updated_at: {
+      type: DataTypes.DATE,
+      allowNull: false,
+      defaultValue: DataTypes.NOW
+    },
+    duration: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      defaultValue: 0
+      // defaultValue: sequelize.literal("nextval('stock_consumption_id_seq'::regclass)")
+    }
+  }, {
+    tableName: 'runs',
+    timestamps: false
+    // schema: 'public'
+  });
+};

package/dist/server/instrumentation.js

@@ -0,0 +1,16 @@
+export const register = async () => {
+  if (process.env.NEXT_RUNTIME === 'nodejs') {
+    // // Dynamically import loadSchedules only when we're in Node.js runtime
+    // const { default: loadSchedules } = await import('./lib/loadSchedules');
+    // await loadSchedules();
+    // // const interval = setInterval(async () => {
+    // //   await executeRuns();
+    // //   console.log('Instrumentation check running...');
+    // // }, 5000);
+
+    // // Clean up interval on process exit
+    process.on('SIGTERM', () => {
+      clearInterval(interval);
+    });
+  }
+};

package/dist/server/lib/executeRun.js

@@ -0,0 +1,80 @@
+"use server";
+
+import getTaskDetails from "./getTaskDetails";
+import microlightDB from "../database/microlight";
+import generateDisplayfunctions from "./generateDisplayFunctions";
+import async from 'async';
+export default async function executeRun(run) {
+  const workflow = {
+    startRun: async function () {
+      let started_at = new Date();
+      await microlightDB.Runs.update({
+        status: 'running',
+        started_at: started_at,
+        updated_at: started_at
+      }, {
+        where: {
+          id: run.id
+        },
+        returning: true
+      });
+      return started_at;
+    },
+    executeTask: ['startRun', async function (results) {
+      let params = {
+        slug: run.task
+      };
+      let taskDef = await getTaskDetails({
+        params
+      });
+      const ml = generateDisplayfunctions(run.id);
+      try {
+        await taskDef.fn(ml, run.inputs);
+        await ml.log('=== ML: run completed ===');
+        return {
+          status: 'complete'
+        };
+      } catch (e) {
+        await ml.error(e);
+        await ml.log('=== ML: run completed ===');
+        return {
+          status: 'failed'
+        };
+      }
+    }],
+    updateRun: ['executeTask', async function (results) {
+      let update = {
+        status: results.executeTask?.status,
+        completed_at: new Date(),
+        updated_at: new Date()
+      };
+      update.duration = update.completed_at - results.startRun;
+      return await microlightDB.Runs.update(update, {
+        where: {
+          id: run.id
+        },
+        returning: true
+      });
+    }]
+    // addToQueue:['createRun',function(results,cb){
+    //   // let options ={
+    //   //   task_slug:run.task,
+    //   // }
+    //   // queue.daily_mis_report.push(options,cb)
+    // }],
+    // triggerRun:['createRun',async function(results){
+    //   // await tasksQueue.add(run.task, { foo: 'bar' });
+    //   // await importQueue.add(run.task, { foo: 'bar' });
+    //   // const handle = await tasks.trigger(
+    //   //   "hello-world",
+    //   //   "James"
+    //   // );
+    //   // return { handle };
+    // }]
+  };
+  const results = await async.auto(workflow);
+  console.log(results);
+  return {
+    success: true
+  };
+}

package/dist/server/lib/generateDisplayFunctions.js

@@ -0,0 +1,89 @@
+import microlightDB from "../database/microlight";
+import markdownit from 'markdown-it';
+const md = markdownit();
+const cleanMD = function (md) {
+  if (!md) md = '<empty>';
+  var lines = md.split('\n');
+  var trimmed_lines = [];
+  lines.forEach(function (line) {
+    trimmed_lines.push(line.trim());
+  });
+  return trimmed_lines.join('\n');
+};
+export default function generateDisplayfunctions(run_id) {
+  let ml = {
+    log: async function (text) {
+      await microlightDB.Logs.create({
+        created_at: new Date(),
+        run: run_id,
+        type: 'log',
+        content: text
+      });
+    },
+    json: async function (data) {
+      await microlightDB.Logs.create({
+        created_at: new Date(),
+        run: run_id,
+        type: 'json',
+        content: JSON.stringify(data)
+      });
+    },
+    markdown: async function (text) {
+      await microlightDB.Logs.create({
+        created_at: new Date(),
+        run: run_id,
+        type: 'markdown',
+        content: md.render(cleanMD(text))
+      });
+    },
+    error: async function (error) {
+      const data = {
+        message: error.message,
+        stack: error.stack
+      };
+      await microlightDB.Logs.create({
+        created_at: new Date(),
+        run: run_id,
+        type: 'error',
+        content: JSON.stringify(data)
+      });
+    },
+    wait: async function (time) {
+      this.log(`waiting for ${time / 1000} secs`);
+      return await new Promise(resolve => setTimeout(resolve, time));
+    },
+    danger: async function (text) {
+      await microlightDB.Logs.create({
+        created_at: new Date(),
+        run: run_id,
+        type: 'danger',
+        content: text
+      });
+    },
+    warn: async function (text) {
+      await microlightDB.Logs.create({
+        created_at: new Date(),
+        run: run_id,
+        type: 'warn',
+        content: text
+      });
+    },
+    info: async function (text) {
+      await microlightDB.Logs.create({
+        created_at: new Date(),
+        run: run_id,
+        type: 'info',
+        content: text
+      });
+    },
+    success: async function (text) {
+      await microlightDB.Logs.create({
+        created_at: new Date(),
+        run: run_id,
+        type: 'success',
+        content: text
+      });
+    }
+  };
+  return ml;
+}

package/dist/server/lib/getAllTasks.js

@@ -0,0 +1,32 @@
+import fs from 'fs';
+import path from 'path';
+function findAllTaskFiles(basePath) {
+  const tasks = [];
+  try {
+    const files = fs.readdirSync(basePath);
+    for (const file of files) {
+      const filePath = path.join(basePath, file);
+      const stat = fs.statSync(filePath);
+      if (stat.isDirectory()) {
+        // Recursively search subdirectories
+        tasks.push(...findAllTaskFiles(filePath));
+      } else if (file.endsWith('.task.js')) {
+        tasks.push(filePath);
+      }
+    }
+  } catch (err) {
+    console.error('Error reading directory:', err);
+  }
+  return tasks;
+}
+export default async function getAllTasks() {
+  const tasksDir = path.join(process.cwd(), 'src', 'tasks');
+  const taskPaths = findAllTaskFiles(tasksDir);
+  const tasks = await Promise.all(taskPaths.map(async fullPath => {
+    // Convert the full filesystem path to a module path
+    const modulePath = fullPath.split('/tasks/')[1];
+    const task = (await import(`@/tasks/${modulePath}`)).default;
+    return task;
+  }));
+  return tasks;
+}

package/dist/server/lib/getTaskDetails.js

@@ -0,0 +1,17 @@
+import { notFound } from 'next/navigation';
+import { importTaskModule } from "../../importTaskModule";
+export default async function getTaskDetails({
+  params
+}) {
+  try {
+    let taskConfig = await importTaskModule(params.slug);
+    return {
+      ...taskConfig.default,
+      // _folderPath: folderPath
+      _folderPath: '1.intro'
+    };
+  } catch (e) {
+    if (e.code === 'MODULE_NOT_FOUND') notFound();
+    throw e;
+  }
+}

package/dist/server/lib/loadSchedules.js

@@ -0,0 +1,77 @@
+import cron from "node-cron";
+import getAllTasks from "./getAllTasks";
+import async from 'async';
+import executeRun from "./executeRun";
+import microlightDB from "../database/microlight";
+import tasks from "../tasks";
+async function executeTask({
+  inputs,
+  task
+}) {
+  const workflow = {
+    createRun: async function () {
+      let run = await microlightDB.Runs.create({
+        task: task.slug,
+        logs: {},
+        inputs: inputs,
+        triggered_by: 'schedule',
+        status: 'pending'
+      }, {
+        returning: true
+      });
+      return run.toJSON();
+    },
+    startRun: ['createRun', async function (results) {
+      process.nextTick(() => executeRun(results.createRun));
+      return;
+    }]
+  };
+  try {
+    const results = await async.auto(workflow);
+    return {
+      success: true,
+      run: results.createRun
+    };
+  } catch (e) {
+    return {
+      success: false,
+      error: e
+    };
+  }
+}
+export default async function loadSchedules() {
+  // const tasks = await getAllTasks();
+  let schedules = [];
+  tasks.forEach(function (task) {
+    // Check if task has schedules
+    if (task.is_enabled && task.schedules && Array.isArray(task.schedules)) {
+      task.schedules.forEach(scheduleConfig => {
+        if (scheduleConfig.is_enabled && scheduleConfig.schedule) {
+          // Create cron job
+          const job = cron.schedule(scheduleConfig.schedule, async () => {
+            try {
+              console.log('trigger the task');
+              // Execute task with schedule-specific inputs
+              await executeTask({
+                inputs: scheduleConfig.inputs || {},
+                task
+              });
+            } catch (error) {
+              console.error(`Error executing scheduled task ${task.slug}:`, error);
+            }
+          }, {
+            timezone: scheduleConfig.timezone || process.env.ML_CRON_TIMEZONE
+          });
+          schedules.push({
+            task,
+            schedule: scheduleConfig,
+            job
+          });
+        }
+      });
+    }
+  });
+  console.log('Count of schedules :' + schedules.length);
+  console.log(schedules);
+  return schedules;
+}

package/dist/server/tasks/1.intro/hello_world2.task.js

@@ -0,0 +1,21 @@
+const task = {
+  slug: 'hello_world',
+  name: 'Hello World',
+  is_enabled: true,
+  description: 'Simply prints Hello world',
+  inputs: {
+    name: {
+      name: "Name",
+      description: "Name of the person",
+      // default: new Date(Date.now() - 86400000).toISOString().substring(0,10),
+      default: 'World',
+      placeholder: 'Your name',
+      type: 'string',
+      required: false
+    }
+  },
+  fn: async function (ml, inputs) {
+    ml.log('hello world');
+  }
+};
+export default task;

package/dist/server/tasks/1.intro/microlight.folder.js

@@ -0,0 +1,5 @@
+export default {
+  name: 'Introduction to Microlight',
+  is_enabled: true,
+  description: "Demo jobs that microlight can do"
+};

package/dist/server/tasks/1.intro/ml.task.js

@@ -0,0 +1,31 @@
+const task = {
+  slug: 'ml',
+  name: 'Microlight display functions',
+  is_enabled: true,
+  description: 'Simply prints Hello world',
+  inputs: {},
+  fn: async function (ml, inputs) {
+    await ml.log('Microlight exposes a number of display functions. Using this you can display feedback to the user');
+    await ml.log('This is a log. Call this with ml.log("Hello world")');
+    await ml.log('ml.json(data)');
+    var data = {
+      name: 'microlight',
+      author: 'Alex J V',
+      github: 'https://github.com/IMGears/microlight'
+    };
+    await ml.json(data);
+    await ml.log('ml.markdown(text)');
+    await ml.markdown(`
+      #### Process FG Details:
+      - **Deleted:** 10
+      - **Created:** 10
+    `);
+    await ml.log("ml.error(new Error('test error'))");
+    await ml.error(new Error('test error'));
+    await ml.danger('this is a danger message');
+    await ml.warn('this is a warning message');
+    await ml.info('this is a info message');
+    await ml.success('this is a success message');
+  }
+};
+export default task;

package/dist/server/tasks/1.intro/scheduled.task.js

@@ -0,0 +1,18 @@
+import { schedule } from "node-cron";
+const task = {
+  slug: 'scheduled',
+  name: 'Scheduled task',
+  is_enabled: true,
+  description: 'This task runs every 2 mins',
+  inputs: {},
+  // maybe should be called inputFields
+  schedules: [{
+    schedule: '*/2 * * * *',
+    is_enabled: true,
+    inputs: {}
+  }],
+  fn: async function (ml, inputs) {
+    await ml.log('This is a scheduled task. Will run every 2 mins');
+  }
+};
+export default task;

package/dist/server/tasks/1.intro/takes_time.task.js

@@ -0,0 +1,28 @@
+const task = {
+  slug: 'takes_time',
+  name: 'Task that takes time to run',
+  is_enabled: true,
+  description: 'Simply prints Hello world',
+  inputs: {},
+  fn: async function (ml, inputs) {
+    await ml.log('This task is going to take time complete execution');
+    await ml.wait(1000); // Wait for 1 seconds
+    await ml.log('Doing step 1');
+    await ml.wait(2000); // Wait for 2 seconds
+    await ml.log('Doing step 2');
+    await ml.wait(2000); // Wait for 2 seconds
+    await ml.log('Doing step 3');
+    await ml.wait(3000); // Wait for 3 seconds
+    await ml.log('Doing step 4');
+    await ml.wait(1000); // Wait for 1 seconds
+    await ml.log('Doing step 5');
+    await ml.wait(100); // Wait for 0.1 seconds
+    await ml.log('Doing step 6');
+    await ml.wait(100); // Wait for 0.1 seconds
+    await ml.log('Doing step 7');
+    await ml.wait(100); // Wait for 0.1 seconds
+    await ml.wait(2000); // Wait for 2 seconds
+    await ml.log('all done');
+  }
+};
+export default task;

package/dist/server/tasks/1.intro/test/microlight.folder.js

@@ -0,0 +1,5 @@
+export default {
+  name: 'Test folder',
+  is_enabled: true,
+  description: "Microlight supports nested folders"
+};

package/dist/server/tasks/1.intro/test/takes_time2.task.js

@@ -0,0 +1,28 @@
+const task = {
+  slug: 'takes_time2',
+  name: 'Task that takes time to run',
+  is_enabled: true,
+  description: 'Simply prints Hello world',
+  inputs: {},
+  fn: async function (ml, inputs) {
+    await ml.log('This task is going to take time complete execution');
+    await ml.wait(1000); // Wait for 1 seconds
+    await ml.log('Doing step 1');
+    await ml.wait(2000); // Wait for 2 seconds
+    await ml.log('Doing step 2');
+    await ml.wait(2000); // Wait for 2 seconds
+    await ml.log('Doing step 3');
+    await ml.wait(3000); // Wait for 3 seconds
+    await ml.log('Doing step 4');
+    await ml.wait(1000); // Wait for 1 seconds
+    await ml.log('Doing step 5');
+    await ml.wait(100); // Wait for 0.1 seconds
+    await ml.log('Doing step 6');
+    await ml.wait(100); // Wait for 0.1 seconds
+    await ml.log('Doing step 7');
+    await ml.wait(100); // Wait for 0.1 seconds
+    await ml.wait(2000); // Wait for 2 seconds
+    await ml.log('all done');
+  }
+};
+export default task;

package/dist/server/tasks/index.js

@@ -0,0 +1,33 @@
+import { glob } from 'glob';
+import path from 'path';
+import { fileURLToPath } from 'url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+console.log(__filename);
+console.log(__dirname);
+
+// Find all task files and folder files
+const taskFiles = glob.sync(['**/*.task.js', '**/microlight.folder.js'], {
+  cwd: __dirname,
+  absolute: true
+});
+console.log(taskFiles);
+
+// Import all task files dynamically
+const tasks = await Promise.all(taskFiles.map(async filePath => {
+  const task = await import(filePath);
+  const taskName = path.basename(filePath, '.task.js');
+  // return [taskName, task.default];
+  return [task?.default?.slug, {
+    ...task?.default,
+    ...{
+      file_name: taskName
+    }
+  }];
+}));
+console.log(tasks);
+
+// Convert array of entries to an object
+const taskMap = Object.fromEntries(tasks);
+console.log(taskMap);
+export default taskMap;

package/dist/server/tasks/microlight.folder.js

@@ -0,0 +1,5 @@
+export default {
+  name: 'Library',
+  is_enabled: true,
+  description: "All executable tasks"
+};

package/index.js

@@ -0,0 +1 @@
+console.log('came here');