@dnax/core 0.73.0 → 0.73.2

package/define/index.ts

@@ -13,11 +13,11 @@ import type {
   routeCtx,
   sessionCtx,
   Script,
+  ETLCfg,
   TimeSeriesCollection,
 } from "../types";
 import { Cfg } from "../config/";
 import { deepMerge, freeze } from "../utils";
-import { config } from "valibot";
 
 function Config(
   config: Omit<
@@ -85,6 +85,10 @@ function Task(config: cronConfig) {
   return config;
 }
 
+function Etl(config: ETLCfg) {
+  return config;
+}
+
 function Permission(config: permissionSchema): permissionSchema {
   return config;
 }
@@ -136,6 +140,7 @@ const define = {
   Route,
   Script,
   TimeSeriesCollection,
+  Etl,
 };
 
 export default define;

package/driver/mongo/rest.ts

@@ -41,6 +41,7 @@ import {
 import { Cfg } from "../../config";
 import cleanDeep from "clean-deep";
 import { useCache } from "../../lib/bento";
+import consola from "consola";
 
 type options = {
   tenant_id: string;
@@ -2549,27 +2550,47 @@ class useRest {
       .dropIndex(indexName);
   }
 
-  startTransaction(options?: { forceNew?: boolean }) {
-    if (options?.forceNew) {
-      this.#session = null;
-    }
-    if (!this.#session) {
-      this.#session = this.#tenant.database.client?.startSession();
-      this.#session?.startTransaction();
+  startTransaction(options?: { forceNew?: boolean; silent?: boolean }) {
+    try {
+      if (options?.forceNew) {
+        this.#session = null;
+      }
+      if (!this.#session) {
+        this.#session = this.#tenant.database.client?.startSession();
+        this.#session?.startTransaction();
+      }
+    } catch (err: any) {
+      if (!options?.silent) {
+        consola.error(err?.message | err);
+      }
     }
   }
 
   async abortTransaction() {
-    if (this.#session) {
-      await this.#session?.abortTransaction();
-    }
+    return new Promise(async (resolve, reject) => {
+      try {
+        if (this.#session) {
+          await this.#session?.abortTransaction();
+          return resolve(true);
+        }
+      } catch (err: any) {
+        return reject(err?.message || err);
+      }
+    });
   }
 
   async endSession() {
-    if (this.#session) {
-      await this.#session?.endSession();
-      this.#session = null;
-    }
+    return new Promise(async (resolve, reject) => {
+      try {
+        if (this.#session) {
+          await this.#session?.endSession();
+          this.#session = null;
+        }
+        return resolve(true);
+      } catch (err: any) {
+        return reject(err?.message || err);
+      }
+    });
   }
 
   async commitTransaction(options?: { closeSession?: boolean }) {
@@ -2585,8 +2606,8 @@ class useRest {
             }
             resolve(true);
           })
-          .catch((err) => {
-            reject(err);
+          .catch((err: any) => {
+            reject(err?.message || err);
           });
       }
     });

package/lib/collection.ts

@@ -66,8 +66,40 @@ async function syncCollectionDatabase() {
         ?.listCollections()
         .toArray();
 
-      //t.database.db.
+      // System collections
+      t.database.db
+        ?.collection("_etl_")
+        .createIndex({
+          createdAt: -1,
+          updatedAt: -1,
+        })
+        .catch();
 
+      t.database.db
+        ?.collection("_activity_")
+        .createIndex({
+          createdAt: -1,
+          updatedAt: -1,
+        })
+        .catch();
+
+      t.database.db
+        ?.collection("_activity_")
+        .createIndex({
+          "operation.type": 1,
+        })
+        .catch();
+
+      t.database.db
+        ?.collection("_activity_")
+        .createIndex({
+          "operation.target": 1,
+        })
+        .catch();
+
+      //__________
+
+      // local Collections
       if (t.database.driver == "mongodb") {
         const collections: (Collection & TimeSeriesCollection[]) | undefined =
           Cfg.collections?.filter((cn) => cn.tenant_id == t.id);

package/lib/connectors/index.ts

@@ -0,0 +1,119 @@
+import { Cron } from "croner";
+import { Glob } from "bun";
+import { Cfg } from "../../config";
+import path from "path";
+import * as aq from "arquero";
+import type { ETLCfg } from "../../types";
+import cleanDeep from "clean-deep";
+import { useRest } from "../../driver/mongo";
+import { v4 } from "uuid";
+import moment from "moment";
+import { formatData } from "../../driver/mongo/utils";
+import { getRecentDate } from "../../utils";
+async function initETL() {
+  if (Cfg?.tenants?.length) {
+    for await (let t of Cfg.tenants) {
+      let tenantPath = `${t.dir}/connectors/**/**.etl.{ts,js}`;
+      const glob = new Glob(tenantPath);
+      for await (let file of glob.scan({
+        cwd: Cfg.cwd,
+      })) {
+        let fullPathFile = path.join(Cfg.cwd || "", file);
+        await import(fullPathFile)
+          .then((inject) => {
+            let etlOpt = inject.default as ETLCfg;
+            const rest = new useRest({
+              tenant_id: t.id,
+            });
+
+            if (
+              typeof etlOpt.exec == "function" &&
+              etlOpt.name &&
+              etlOpt.pattern
+            ) {
+              let lockExecution = false;
+              const cronETL = new Cron(etlOpt.pattern, async () => {
+                if (lockExecution) return;
+                lockExecution = true;
+                try {
+                  let lastRuntAt = new Date();
+                  let [c] = (await rest.db
+                    ?.collection("_etl_")
+                    .find({
+                      name: etlOpt.name,
+                      ...(etlOpt.filter || {}),
+                    })
+                    .limit(1)
+                    .sort({
+                      createdAt: -1,
+                    })
+                    .toArray()) as Object;
+
+                  let traceId = v4();
+                  await etlOpt.exec({
+                    extractLatest: getRecentDate,
+                    rest,
+                    aq: aq,
+                    commit: async (op) => {
+                      return new Promise(async (resolve, reject) => {
+                        try {
+                          await rest.db?.collection("_etl_").updateOne(
+                            {
+                              name: etlOpt.name,
+                              tenantId: t.id,
+                              ...(etlOpt.filter || {}),
+                            },
+                            {
+                              $set: {
+                                ...formatData({
+                                  tenantId: t.id,
+                                  name: etlOpt.name,
+                                  ts: op?.ts ? new Date(op.ts) : null,
+                                  data: op?.data,
+                                  comment: op?.comment,
+                                  tags: op?.tags || [],
+                                  version: op?.version || "1",
+                                  context: op?.context || {},
+                                  lastRuntAt: lastRuntAt,
+                                  traceId: traceId,
+                                  commitAt: c?.commitAt
+                                    ? new Date(c?.commitAt)
+                                    : new Date(),
+                                  createdAt: new Date(),
+                                  updatedAt: new Date(),
+                                }),
+                              },
+                            },
+                            {
+                              upsert: true,
+                            }
+                          );
+                          resolve(true);
+                        } catch (error: any) {
+                          reject(error?.message || error);
+                        }
+                      });
+                    },
+                    lastCommitAt: c?.commitAt || null,
+                    ts: c?.ts || null,
+                    lastRuntAt: lastRuntAt,
+                    traceId,
+                  });
+                } catch (err: any) {
+                  console.error(err?.message || err);
+                }
+                lockExecution = false;
+              });
+            }
+          })
+          .catch((err) => {
+            console.error(err);
+          });
+      }
+    }
+  }
+
+  //Cfg.endpoints = endpoints;
+}
+
+export { initETL };

package/lib/crypto.ts

@@ -63,8 +63,6 @@ class AES {
   }
 }
 
-const message = "Bonjour, comment ça va ?";
-
 const crypt = {
   AES,
 };

package/lib/index.ts

@@ -3,6 +3,7 @@ import { connectTenantsDatabase } from "../driver";
 import { loadEndpoints } from "./endpoint";
 import { loadServices } from "./service";
 import { initCron } from "./cron";
+import { initETL } from "./connectors";
 import { loadPermissions } from "./permissions";
 import { loadSocket } from "../lib/socket";
 import { loadAutoRoutes } from "./routes";
@@ -49,7 +50,11 @@ async function init(cf = { app: null }) {
   await loadAutoRoutes();
   //consola.success("Loaded auto routes");
 
-  await initCron();
+  // Init Cron
+  initCron();
+
+  // Init ETL
+  initETL();
 }
 
 export { init };

package/lib/session/index.ts

@@ -1,28 +1 @@
-import { v4 } from "uuid";
-import { sessionCache } from "../bento";
-
-const localSession = {
-  get: async (id: string) => {
-    return await sessionCache.get(id);
-  },
-  set: (
-    id: string,
-    data: object,
-    options = {
-      ttl: "7d",
-    }
-  ) => {
-    sessionCache.set({
-      key: id,
-      value: data,
-      //  ttl: "1m",
-      ttl: options?.ttl || "7d",
-    });
-
-    return {
-      data: data,
-    };
-  },
-};
-
-export { localSession };
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dnax/core",
-  "version": "0.73.0",
+  "version": "0.73.2",
   "module": "index.ts",
   "type": "module",
   "bin": {},
@@ -21,6 +21,7 @@
   "dependencies": {
     "@colors/colors": "^1.6.0",
     "@lukeed/ms": "^2.0.2",
+    "arquero": "^8.0.3",
     "bentocache": "^1.5.0",
     "boxen": "^7.1.1",
     "chokidar": "3.6.0",
@@ -33,7 +34,7 @@
     "dot-object": "2.1.5",
     "fs-extra": "^11.2.0",
     "generate-unique-id": "^2.0.3",
-    "hono": "4.8.9",
+    "hono": "4.9.5",
     "joi": "17.13.3",
     "json-joy": "16.8.0",
     "jsonwebtoken": "^9.0.2",

package/types/index.ts

@@ -1,5 +1,6 @@
 import type { AnySchema } from "joi";
 import { Cron } from "croner";
+import * as aq from "arquero";
 //import type{ updateParams } from "./../driver/mongo/@types";
 //import * as v from "valibot";
 import type { Db, MongoClient } from "mongodb";
@@ -12,7 +13,7 @@ import type { Context, Hono } from "hono";
 
 import type { socketIoType } from "../lib/socket/instance";
 
-import { fn } from "../utils";
+import { fn, getRecentDate } from "../utils";
 import type {
   findOneParam,
   findParam,
@@ -893,3 +894,29 @@ export type permissionSchema = {
 export type CoreType = {
   listCollections: () => Array<Collection>;
 };
+
+
+export type ETLCfg ={ // Extract, Transform, Load
+  name:string;
+  pattern:string; // Cron Expression
+  filter?:{ // Filter to get last record of ETL execution
+    [key: string]: any;
+  },
+  exec:(ctx:{
+    rest:InstanceType<typeof useRest>;
+    lastCommitAt:Date|null;
+    ts:Date;
+    lastRuntAt:Date;
+    traceId:string;
+    aq:typeof aq;
+    extractLatest:typeof getRecentDate;
+    commit:(options:{
+      comment?:string;
+      tags?:Array<string>;
+      data?:any;
+      context?:object;
+      version?:number;
+      ts:Date;
+    })=>Promise<boolean|any>
+  })=>any;
+}

package/utils/index.ts

@@ -19,6 +19,7 @@ import { applyPatch, createPatch, Pointer } from "rfc6902";
 
 import * as _ from "radash";
 import { email } from "../lib/mail";
+import { getPidUsage } from "./os";
 
 import { deepEqual } from "json-joy/lib/json-equal/deepEqual";
 import type { SupportedCryptoAlgorithms } from "bun";
@@ -570,6 +571,29 @@ function useKey(
   return n;
 }
 
+function getRecentDate<T extends Record<string, any>>(
+  items: T[],
+  key: string = "createdAt" // default key to get recent date
+): { data: T | null; date: Date | null } {
+  if (!items || items.length === 0) return { data: null, date: null };
+
+  let latest: Date | null = null;
+  let latestItem: T | null = null;
+
+  for (const item of items) {
+    const dateStr = item[key];
+    if (dateStr) {
+      const d = new Date(dateStr);
+      if (!isNaN(d.getTime()) && (!latest || d > latest)) {
+        latest = d;
+        latestItem = item;
+      }
+    }
+  }
+
+  return { data: latestItem, date: latest };
+}
+
 const contextError = ContextError;
 export {
   file,
@@ -607,4 +631,6 @@ export {
   createPatch,
   containsForbiddenOperators,
   useKey,
+  getRecentDate,
+  getPidUsage,
 };

package/utils/os.ts

@@ -15,8 +15,8 @@ export async function getPidUsage(pids = [process.pid]) {
       ram: usage.memory / 1024 / 1024, // en MB
       elapsed: usage.elapsed / 1000, // uptime en secondes
     }));
-  } catch (err) {
-    console.error("Erreur getPidUsage:", err.message);
+  } catch (err: any) {
+    console.error("Erreur getPidUsage:", err?.message);
     return [];
   }
 }

package/lib/workflow/index.ts

@@ -1,84 +0,0 @@
-import { useRest } from "../../driver/mongo";
-import { v4 } from "uuid";
-
-type WorkFlowConfig = {
-  tenant_id: string;
-  version?: string;
-};
-
-type StepMeta = {
-  ms?: number;
-  [key: string]: any;
-};
-
-type StepStatus = "done" | "failed";
-
-type StepFunction<TData = any> = (ctx: {
-  rest: useRest;
-  traceId: string;
-  data: TData;
-  meta?: Record<string, any>;
-  output: any;
-}) => Promise<void>;
-
-type Step = {
-  name: string;
-  handler: StepFunction;
-};
-
-class Workflow<Tdata> {
-  name: string;
-  config: WorkFlowConfig;
-  #rest: useRest;
-  traceId?: string;
-  #steps: Step[] = [];
-  constructor(name: string, config: WorkFlowConfig) {
-    this.name = name;
-    this.config = config;
-    this.#rest = new useRest({
-      tenant_id: config.tenant_id,
-    });
-  }
-
-  step(name: string, handler: StepFunction<Tdata>) {
-    this.#steps.push({
-      name,
-      handler,
-    });
-    return this;
-  }
-
-  async run(data: Tdata, traceId?: string): Promise<void> {
-    this.traceId = traceId || v4();
-    let output: any = {};
-    for (const step of this.#steps) {
-      let startAt = Date.now();
-      let ev = {
-        name: step.name,
-        traceId: this.traceId,
-        duration: 0,
-        status: "pending",
-        startedAt: new Date(),
-        endedAt: new Date(),
-      };
-      output = await step.handler({
-        rest: this.#rest,
-        traceId: this.traceId,
-        data,
-        output,
-      });
-      let endAt = Date.now();
-      // duration en ms
-      let duration = endAt - startAt;
-      ev.duration = duration;
-      ev.status = "done";
-      ev.endedAt = new Date();
-    }
-  }
-}
-
-function useWorkflow<T>(name: string, config: WorkFlowConfig) {
-  return new Workflow<T>(name, config);
-}
-
-export { useWorkflow };

package/lib/workflow/types.ts

@@ -1,64 +0,0 @@
-import { useRest } from "../../driver/mongo";
-
-export type StepActions = {
-  done: (comment?: string) => void;
-  fail: (comment?: string) => void;
-  abort: (comment?: string) => void;
-};
-
-export type Step<Name extends string = string> = {
-  name: Name;
-  description?: string;
-  exec: (ctx: { data: any; rest: useRest; action?: StepActions }) => void;
-};
-
-export type stepData = {
-  name: string;
-  data: any;
-  at: Date;
-  comment?: string;
-  description?: string;
-  status: "done" | "failed" | "aborted";
-  traceId: string;
-  ms: number;
-};
-
-export type EventStep = {
-  status: "failed" | "aborted" | "done";
-  data: any;
-  at: Date;
-  name: string;
-};
-
-export type WorkflowData = {
-  _id: string;
-  name: string;
-  traceId: string;
-  currentStep: string;
-  steps: stepData[];
-  events: [];
-  status:
-    | "processing"
-    | "completed"
-    | "failed"
-    | "aborted"
-    | "cancelled"
-    | "initialized";
-};
-
-export type WorkflowType = {
-  traceId: string;
-  _id: string;
-  name: string;
-  currentStep: string;
-  failedAtStep?: string;
-  steps: Step[];
-  events: [];
-  completedIf: () => boolean;
-  failedIf: () => boolean;
-  abortedIf: () => boolean;
-};
-
-export type StepInstance = {
-  run: (stepName: string, data?: any) => Promise<void>;
-};