@nmakarov/cli-toolkit 0.33.0 → 0.37.0

package/dist/tasks.js

@@ -88,6 +88,91 @@ import os from "os";
 // src/tasks/taskUtils.js
 import { randomUUID } from "crypto";
 
+// src/db/ensure.js
+var dbLabel = (db) => db?.config?.name ?? "db";
+async function ensureExtension(db, name, options = {}) {
+  const action = `CREATE EXTENSION IF NOT EXISTS "${name}"`;
+  if (!options.dryRun) {
+    await db.raw(action);
+  }
+  options.logger?.silly?.(`[ensure] ${dbLabel(db)}: ${action}${options.dryRun ? " (dryRun)" : ""}`);
+  return { action };
+}
+async function ensureTable(db, tableName, spec, options = {}) {
+  const { dryRun = false, logger } = options;
+  const actions = [];
+  const exists = await db.tableExists(tableName);
+  if (!exists) {
+    actions.push(`CREATE TABLE ${tableName} (${Object.keys(spec.columns).length} columns)`);
+    if (!dryRun) {
+      await db.schema.createTable(tableName, (t) => {
+        for (const define of Object.values(spec.columns)) {
+          define(t, db);
+        }
+      });
+    }
+  } else {
+    const missing = [];
+    for (const column of Object.keys(spec.columns)) {
+      if (!await db.schema.hasColumn(tableName, column)) {
+        missing.push(column);
+      }
+    }
+    if (missing.length > 0) {
+      actions.push(`ALTER TABLE ${tableName} ADD COLUMN ${missing.join(", ")}`);
+      if (!dryRun) {
+        await db.schema.alterTable(tableName, (t) => {
+          for (const column of missing) {
+            spec.columns[column](t, db);
+          }
+        });
+      }
+    }
+  }
+  for (const index of spec.indexes ?? []) {
+    const indexActions = await ensureIndex(db, tableName, index, options);
+    actions.push(...indexActions);
+  }
+  for (const action of actions) {
+    logger?.silly?.(`[ensure] ${dbLabel(db)}: ${action}${dryRun ? " (dryRun)" : ""}`);
+  }
+  return actions;
+}
+async function ensureIndex(db, tableName, index, options = {}) {
+  const { dryRun = false } = options;
+  const kind = index.unique ? "UNIQUE INDEX" : "INDEX";
+  const cols = index.columns.map((c) => `"${c}"`).join(", ");
+  const isPg = String(db?.config?.connectionString ?? "").startsWith("postgresql");
+  if (isPg) {
+    const sql2 = `CREATE ${kind} IF NOT EXISTS "${index.name}" ON "${tableName}" (${cols})`;
+    const { rows } = await db.raw(`SELECT 1 FROM pg_indexes WHERE indexname = ?`, [index.name]);
+    if (rows.length > 0) return [];
+    if (!dryRun) await db.raw(sql2);
+    return [sql2];
+  }
+  const sql = `CREATE ${kind} ${index.name} ON ${tableName} (${cols})`;
+  if (!dryRun) {
+    try {
+      await db.raw(sql);
+    } catch (error) {
+      if (!/already exists|duplicate/i.test(error?.message ?? "")) throw error;
+      return [];
+    }
+  }
+  return [sql];
+}
+async function ensureSchema(db, spec, options = {}) {
+  const actions = [];
+  for (const extension of spec.extensions ?? []) {
+    const { action } = await ensureExtension(db, extension, options);
+    if (options.dryRun) actions.push(action);
+  }
+  for (const [tableName, tableSpec] of Object.entries(spec.tables ?? {})) {
+    actions.push(...await ensureTable(db, tableName, tableSpec, options));
+  }
+  return { database: dbLabel(db), actions };
+}
+
 // src/tasks/time-matcher.js
 var RANGES = ["0-59", "0-59", "0-23", "1-31", "1-12", "0-6"];
 function resolveAsterisks(field, range) {
@@ -172,29 +257,90 @@ function queueToTableNames(queueName) {
     registryTable: `${queueName}_services_registry`
   };
 }
-function defineTasksTable(t, db, tableNameForIndex) {
-  t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()"));
-  t.timestamp("created_at").notNullable().defaultTo(db.fn.now());
-  t.timestamp("started_at");
-  t.timestamp("completed_at");
-  t.integer("priority").notNullable().defaultTo(50);
-  t.text("schedule");
-  t.timestamp("next_run_at").defaultTo(null);
-  t.timestamp("past_due").defaultTo(null);
-  t.text("name").notNullable();
-  t.text("opid");
-  t.jsonb("params");
-  t.text("service_group");
-  t.integer("instance_number");
-  t.text("service_name");
-  t.text("server_name");
-  t.text("status").notNullable().defaultTo("idle");
-  t.timestamp("status_changed_at").defaultTo(null);
-  t.text("progress");
-  t.boolean("success");
-  t.jsonb("results");
-  t.index(["service_group", "status", "priority", "created_at"], `${tableNameForIndex}_claim_idx`);
-  t.index(["service_group", "name"], `${tableNameForIndex}_group_name_idx`);
+function tasksTableSpec(tableNameForIndex) {
+  return {
+    columns: {
+      id: (t, db) => t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()")),
+      created_at: (t, db) => t.timestamp("created_at").notNullable().defaultTo(db.fn.now()),
+      started_at: (t) => t.timestamp("started_at"),
+      completed_at: (t) => t.timestamp("completed_at"),
+      /*
+       * Priority: lower number = claimed first (see claimNextRunnableTask: ORDER BY priority ASC).
+       * Suggested range 0–100: 0 = most urgent, 100 = least; default 50 for normal work.
+       */
+      priority: (t) => t.integer("priority").notNullable().defaultTo(50),
+      schedule: (t) => t.text("schedule"),
+      next_run_at: (t) => t.timestamp("next_run_at").defaultTo(null),
+      past_due: (t) => t.timestamp("past_due").defaultTo(null),
+      name: (t) => t.text("name").notNullable(),
+      opid: (t) => t.text("opid"),
+      params: (t) => t.jsonb("params"),
+      // those are target identifiers, kind of who is going to run a task.
+      service_group: (t) => t.text("service_group"),
+      // harvester, loader, photos, ...
+      instance_number: (t) => t.integer("instance_number"),
+      service_name: (t) => t.text("service_name"),
+      // that's a "<server_name>_<service_group>_<instance_number>"
+      server_name: (t) => t.text("server_name"),
+      // filled by runner when registering, auto.
+      status: (t) => t.text("status").notNullable().defaultTo("idle"),
+      // idle, running, completed, failed, paused
+      status_changed_at: (t) => t.timestamp("status_changed_at").defaultTo(null),
+      progress: (t) => t.text("progress"),
+      success: (t) => t.boolean("success"),
+      results: (t) => t.jsonb("results")
+    },
+    indexes: [
+      {
+        columns: ["service_group", "status", "priority", "created_at"],
+        name: `${tableNameForIndex}_claim_idx`
+      },
+      { columns: ["service_group", "name"], name: `${tableNameForIndex}_group_name_idx` }
+    ]
+  };
+}
+function registryTableSpec(registryTable) {
+  return {
+    columns: {
+      id: (t, db) => t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()")),
+      queue_name: (t) => t.text("queue_name").notNullable(),
+      service_group: (t) => t.text("service_group").notNullable(),
+      // harvester, loader, photos, ...
+      instance_number: (t) => t.integer("instance_number").notNullable().defaultTo(1),
+      service_name: (t) => t.text("service_name").notNullable(),
+      // that's a "<server_name>_<service_group>_<instance_number>"
+      server_name: (t) => t.text("server_name").notNullable(),
+      // filled by runner when registering, auto.
+      pid: (t) => t.integer("pid"),
+      metadata: (t) => t.json("metadata"),
+      created_at: (t, db) => t.timestamp("created_at").notNullable().defaultTo(db.fn.now()),
+      last_seen_at: (t, db) => t.timestamp("last_seen_at").notNullable().defaultTo(db.fn.now())
+    },
+    indexes: [
+      {
+        columns: ["queue_name", "service_name"],
+        name: `${registryTable}_queue_name_service_name_uniq`,
+        unique: true
+      },
+      {
+        columns: ["queue_name", "service_group", "last_seen_at"],
+        name: `${registryTable}_queue_group_seen_idx`
+      },
+      { columns: ["queue_name", "last_seen_at"], name: `${registryTable}_queue_seen_idx` }
+    ]
+    // TODO: a reference is needed - task_history entry should reference the registry entry, so that we can easily find all executed tasks for a given service and calculate the average workload or identify if there's a bottleneck. Also may be used for the load balancing.
+  };
+}
+function tasksSchemaSpec(queueName = "tasks") {
+  const { tasksTable, historyTable, registryTable } = queueToTableNames(queueName);
+  return {
+    extensions: ["uuid-ossp"],
+    tables: {
+      [tasksTable]: tasksTableSpec(tasksTable),
+      [historyTable]: tasksTableSpec(historyTable),
+      [registryTable]: registryTableSpec(registryTable)
+    }
+  };
 }
 function taskHistoryInsertFromQueueRow(row, overrides) {
   const { id, ...snapshot } = row;
@@ -208,60 +354,38 @@ async function ensureTaskTables(context, options = {}) {
   const queueName = options.queueName ?? "tasks";
   const recreate = options.recreate ?? false;
   const dryRun = options.dryRun ?? false;
-  const db = getDb(context);
+  const databases = options.databases ?? [getDb(context)];
   const log = context.logger ?? console;
   const { tasksTable, historyTable, registryTable } = queueToTableNames(queueName);
-  const needsTasks = recreate ? true : !await db.tableExists(tasksTable);
-  const needsHistory = recreate ? true : !await db.tableExists(historyTable);
-  const needsRegistry = recreate ? true : !await db.tableExists(registryTable);
-  if (dryRun) {
-    const plan = [];
+  const spec = tasksSchemaSpec(queueName);
+  for (const db of databases) {
+    const label = db?.config?.name ?? "db";
     if (recreate) {
-      plan.push(`DROP TABLE IF EXISTS ${historyTable}, ${tasksTable}, ${registryTable}`);
+      if (dryRun) {
+        log.info?.(
+          `[tasks-schema] dryRun \u2014 ${label}: DROP TABLE IF EXISTS ${historyTable}, ${tasksTable}, ${registryTable}`
+        );
+      } else {
+        await db.schema.dropTableIfExists(historyTable);
+        await db.schema.dropTableIfExists(tasksTable);
+        await db.schema.dropTableIfExists(registryTable);
+      }
     }
-    if (needsTasks) plan.push(`CREATE TABLE ${tasksTable} (tasks queue)`);
-    if (needsHistory) plan.push(`CREATE TABLE ${historyTable} (history mirror)`);
-    if (needsRegistry) plan.push(`CREATE TABLE ${registryTable} (services registry)`);
-    if (plan.length === 0) {
-      log.info?.(`[tasks-schema] dryRun \u2014 queue "${queueName}" already up to date; no DDL`);
-    } else {
-      log.info?.(`[tasks-schema] dryRun \u2014 would run ${plan.length} statement(s) for queue "${queueName}":`);
-      for (const s of plan) log.info?.(`  - ${s}`);
+    const { actions } = await ensureSchema(db, spec, { dryRun, logger: context.logger });
+    if (dryRun) {
+      if (actions.length === 0) {
+        log.info?.(`[tasks-schema] dryRun \u2014 ${label}: queue "${queueName}" already up to date; no DDL`);
+      } else {
+        log.info?.(
+          `[tasks-schema] dryRun \u2014 ${label}: would run ${actions.length} statement(s) for queue "${queueName}":`
+        );
+        for (const s of actions) log.info?.(`  - ${s}`);
+      }
+    } else if (actions.length > 0) {
+      log.info?.(
+        `[tasks-schema] ${label}: applied ${actions.length} DDL statement(s) for queue "${queueName}"`
+      );
     }
-    return;
-  }
-  if (recreate) {
-    await db.schema.dropTableIfExists(historyTable);
-    await db.schema.dropTableIfExists(tasksTable);
-    await db.schema.dropTableIfExists(registryTable);
-  }
-  if (needsTasks) {
-    await db.raw(`CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`);
-    await db.schema.createTable(tasksTable, (t) => {
-      defineTasksTable(t, db, tasksTable);
-    });
-  }
-  if (needsHistory) {
-    await db.schema.createTable(historyTable, (t) => {
-      defineTasksTable(t, db, historyTable);
-    });
-  }
-  if (needsRegistry) {
-    await db.schema.createTable(registryTable, (t) => {
-      t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()"));
-      t.text("queue_name").notNullable();
-      t.text("service_group").notNullable();
-      t.integer("instance_number").notNullable().defaultTo(1);
-      t.text("service_name").notNullable();
-      t.text("server_name").notNullable();
-      t.integer("pid");
-      t.json("metadata");
-      t.timestamp("created_at").notNullable().defaultTo(db.fn.now());
-      t.timestamp("last_seen_at").notNullable().defaultTo(db.fn.now());
-      t.unique(["queue_name", "service_name"], `${registryTable}_queue_name_service_name_uniq`);
-      t.index(["queue_name", "service_group", "last_seen_at"], `${registryTable}_queue_group_seen_idx`);
-      t.index(["queue_name", "last_seen_at"], `${registryTable}_queue_seen_idx`);
-    });
   }
 }
 async function enqueueTask(context, options) {