@nmakarov/cli-toolkit 0.33.0 → 0.37.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1868,6 +1868,49 @@ var Params = class _Params {
1868
1868
  module: moduleName ?? this._currentModule
1869
1869
  });
1870
1870
  }
1871
+ /**
1872
+ * Report a param value that a component RESOLVED ON ITS OWN — outside
1873
+ * Params.get() — e.g. discovered by combining its config files (the way
1874
+ * blueprints merge defaults/aggregator/feed data). Without this, every
1875
+ * key such a component probed for an override shows up in the
1876
+ * --showUsedParams dump as "undefined (default)"; reporting upgrades the
1877
+ * entry to the value the component actually works with.
1878
+ *
1879
+ * Attribution rules:
1880
+ * - entries figured from an explicit input (cli/env/options/…) are left
1881
+ * untouched — the component merely confirmed them, the origin stands;
1882
+ * - "default" entries (Params had nothing) and earlier reports are
1883
+ * replaced by the report;
1884
+ * - keys never seen by Params are appended as new entries.
1885
+ * The LATEST report wins — a component may resolve the same key several
1886
+ * times with increasing specificity (e.g. a blueprint re-merged for a
1887
+ * concrete resource) and the dump should show what it settled on.
1888
+ * Later params.get() probes that find nothing ("default") never shadow a
1889
+ * reported value — see {@link getFiguredByModule}.
1890
+ *
1891
+ * @param {string} key
1892
+ * @param {*} value - the value the component actually uses
1893
+ * @param {string} [source="discovered"] - short origin label, e.g. "blueprint"
1894
+ * @param {string} [moduleName] - dump section; defaults to the current module
1895
+ */
1896
+ reportResolved(key, value, source = "discovered", moduleName) {
1897
+ const mod = moduleName ?? this._currentModule;
1898
+ const mine = this.trackedParams.filter((e) => e.key === key && e.module === mod);
1899
+ if (mine.some((e) => e.source !== "default" && !e.reported)) {
1900
+ return;
1901
+ }
1902
+ this.trackedParams = this.trackedParams.filter(
1903
+ (e) => !(e.key === key && e.module === mod)
1904
+ );
1905
+ this.trackedParams.push({
1906
+ key,
1907
+ definition: "reported",
1908
+ value,
1909
+ source,
1910
+ module: mod,
1911
+ reported: true
1912
+ });
1913
+ }
1871
1914
  /**
1872
1915
  * Get all tracked parameters (for --stopAfter=init)
1873
1916
  */
@@ -1892,12 +1935,24 @@ var Params = class _Params {
1892
1935
  /**
1893
1936
  * Get figured parameters grouped by module name.
1894
1937
  * Same param can appear in multiple modules (e.g. source, resource).
1938
+ * Last occurrence per key wins, EXCEPT that an empty probe — a
1939
+ * params.get() that found nothing ("default", undefined) — never shadows
1940
+ * a value reported via {@link reportResolved}: components probe for
1941
+ * overrides on every resolution cycle, and those misses say nothing about
1942
+ * the value the component actually uses.
1895
1943
  */
1896
1944
  getFiguredByModule() {
1945
+ const reported = /* @__PURE__ */ new Set();
1946
+ for (const param of this.trackedParams) {
1947
+ if (param.reported) reported.add(`${param.module}\0${param.key}`);
1948
+ }
1897
1949
  const byModule = {};
1898
1950
  for (const param of this.trackedParams) {
1899
1951
  const mod = param.module;
1900
1952
  if (!byModule[mod]) byModule[mod] = {};
1953
+ if (!param.reported && param.source === "default" && reported.has(`${mod}\0${param.key}`)) {
1954
+ continue;
1955
+ }
1901
1956
  byModule[mod][param.key] = { value: param.value, source: param.source };
1902
1957
  }
1903
1958
  return byModule;
@@ -2660,13 +2715,100 @@ async function init(flow2, opts = {}) {
2660
2715
 
2661
2716
  // src/db/index.js
2662
2717
  var import_knex = __toESM(require("knex"), 1);
2718
+
2719
+ // src/db/ensure.js
2720
+ var dbLabel = (db) => db?.config?.name ?? "db";
2721
+ async function ensureExtension(db, name, options = {}) {
2722
+ const action = `CREATE EXTENSION IF NOT EXISTS "${name}"`;
2723
+ if (!options.dryRun) {
2724
+ await db.raw(action);
2725
+ }
2726
+ options.logger?.silly?.(`[ensure] ${dbLabel(db)}: ${action}${options.dryRun ? " (dryRun)" : ""}`);
2727
+ return { action };
2728
+ }
2729
+ async function ensureTable(db, tableName, spec, options = {}) {
2730
+ const { dryRun = false, logger } = options;
2731
+ const actions = [];
2732
+ const exists = await db.tableExists(tableName);
2733
+ if (!exists) {
2734
+ actions.push(`CREATE TABLE ${tableName} (${Object.keys(spec.columns).length} columns)`);
2735
+ if (!dryRun) {
2736
+ await db.schema.createTable(tableName, (t) => {
2737
+ for (const define of Object.values(spec.columns)) {
2738
+ define(t, db);
2739
+ }
2740
+ });
2741
+ }
2742
+ } else {
2743
+ const missing = [];
2744
+ for (const column of Object.keys(spec.columns)) {
2745
+ if (!await db.schema.hasColumn(tableName, column)) {
2746
+ missing.push(column);
2747
+ }
2748
+ }
2749
+ if (missing.length > 0) {
2750
+ actions.push(`ALTER TABLE ${tableName} ADD COLUMN ${missing.join(", ")}`);
2751
+ if (!dryRun) {
2752
+ await db.schema.alterTable(tableName, (t) => {
2753
+ for (const column of missing) {
2754
+ spec.columns[column](t, db);
2755
+ }
2756
+ });
2757
+ }
2758
+ }
2759
+ }
2760
+ for (const index of spec.indexes ?? []) {
2761
+ const indexActions = await ensureIndex(db, tableName, index, options);
2762
+ actions.push(...indexActions);
2763
+ }
2764
+ for (const action of actions) {
2765
+ logger?.silly?.(`[ensure] ${dbLabel(db)}: ${action}${dryRun ? " (dryRun)" : ""}`);
2766
+ }
2767
+ return actions;
2768
+ }
2769
+ async function ensureIndex(db, tableName, index, options = {}) {
2770
+ const { dryRun = false } = options;
2771
+ const kind = index.unique ? "UNIQUE INDEX" : "INDEX";
2772
+ const cols = index.columns.map((c) => `"${c}"`).join(", ");
2773
+ const isPg = String(db?.config?.connectionString ?? "").startsWith("postgresql");
2774
+ if (isPg) {
2775
+ const sql2 = `CREATE ${kind} IF NOT EXISTS "${index.name}" ON "${tableName}" (${cols})`;
2776
+ const { rows } = await db.raw(`SELECT 1 FROM pg_indexes WHERE indexname = ?`, [index.name]);
2777
+ if (rows.length > 0) return [];
2778
+ if (!dryRun) await db.raw(sql2);
2779
+ return [sql2];
2780
+ }
2781
+ const sql = `CREATE ${kind} ${index.name} ON ${tableName} (${cols})`;
2782
+ if (!dryRun) {
2783
+ try {
2784
+ await db.raw(sql);
2785
+ } catch (error) {
2786
+ if (!/already exists|duplicate/i.test(error?.message ?? "")) throw error;
2787
+ return [];
2788
+ }
2789
+ }
2790
+ return [sql];
2791
+ }
2792
+ async function ensureSchema(db, spec, options = {}) {
2793
+ const actions = [];
2794
+ for (const extension of spec.extensions ?? []) {
2795
+ const { action } = await ensureExtension(db, extension, options);
2796
+ if (options.dryRun) actions.push(action);
2797
+ }
2798
+ for (const [tableName, tableSpec] of Object.entries(spec.tables ?? {})) {
2799
+ actions.push(...await ensureTable(db, tableName, tableSpec, options));
2800
+ }
2801
+ return { database: dbLabel(db), actions };
2802
+ }
2803
+
2804
+ // src/db/index.js
2663
2805
  var KNEX_DEFAULTS = {
2664
2806
  testConnection: true,
2665
2807
  pool: { min: 2, max: 10 },
2666
2808
  acquireConnectionTimeout: 1e4,
2667
2809
  ssl: { rejectUnauthorized: false }
2668
2810
  };
2669
- var Db = class {
2811
+ var Db = class _Db {
2670
2812
  static async init(context, options = {}) {
2671
2813
  const buildConfig = async () => {
2672
2814
  const defs2 = {
@@ -2735,6 +2877,177 @@ var Db = class {
2735
2877
  const config2 = context?.params?.runWithModuleAsync ? await context.params.runWithModuleAsync("db", buildConfig) : await buildConfig();
2736
2878
  return dbConnect(context, config2);
2737
2879
  }
2880
+ /**
2881
+ * Initialize a SIBLING database handler: a database that lives on the same
2882
+ * server with the same credentials/options as an existing ("base") one, and
2883
+ * differs only by its database name. Typical use: a per-tenant / per-subject
2884
+ * database alongside a main database that keeps the shared tables.
2885
+ *
2886
+ * context.db = await Db.init(context); // main
2887
+ * const sub = await Db.initSibling(context, "src_bright"); // sibling
2888
+ *
2889
+ * LOCATION-AGNOSTIC BY DESIGN — the call gracefully falls back to the main
2890
+ * database, so callers can use it for all subject data without knowing what
2891
+ * has been migrated where:
2892
+ * - empty `siblingName` → the main handler (same as Db.init)
2893
+ * - sibling DB does not exist yet → the main handler (data not migrated;
2894
+ * it still lives in the main database)
2895
+ * Callers that must not fall back can check `handler === context.db`.
2896
+ *
2897
+ * Connection string resolution for an actual sibling, in order:
2898
+ * 1. Explicit override — param `dbConnectionStringSib<SiblingName>` (env
2899
+ * `DB_CONNECTION_STRING_SIB_<SIBLING_NAME>`, e.g. `src_bright` →
2900
+ * `DB_CONNECTION_STRING_SIB_SRC_BRIGHT`). Nobody needs this on day
2901
+ * one; it is the escape hatch for when a sibling later moves to its
2902
+ * own server — one env var, no code changes (the `SIB_` namespace
2903
+ * both overrides the connection and declares that the sibling
2904
+ * exists, so it never falls back to main).
2905
+ * 2. Derived — take the base connection string and swap the database
2906
+ * name (after confirming the database exists on that server). The
2907
+ * base is `options.baseDb` (a Db handler), then
2908
+ * `options.baseConnectionString`, then `context.db`.
2909
+ *
2910
+ * Handlers are cached per name on the context (`context.siblingDbs`), so
2911
+ * any number of components asking for the same sibling share one pool —
2912
+ * including the "falls back to main" answer, which is remembered for the
2913
+ * lifetime of the process (a mid-run migration is picked up on restart).
2914
+ * Disconnect is registered via `context.registerCleanup`, same as the
2915
+ * main handler.
2916
+ *
2917
+ * @param {object} context - context with params/logger (and usually .db)
2918
+ * @param {string} [siblingName] - the sibling's database name (e.g. "src_bright")
2919
+ * @param {object} [options] - { baseDb, baseConnectionString, dbProfile }
2920
+ * @returns {Promise<Db>} connected handler (same proxy shape as Db.init)
2921
+ */
2922
+ static async initSibling(context, siblingName, options = {}) {
2923
+ if (!siblingName) {
2924
+ return resolveMainHandler(context, options);
2925
+ }
2926
+ if (!/^[a-zA-Z0-9_]+$/.test(siblingName)) {
2927
+ throw new ParamError(
2928
+ `Db.initSibling: invalid sibling database name "${siblingName}" (letters, digits and _ only)`
2929
+ );
2930
+ }
2931
+ if (!context.siblingDbs) {
2932
+ context.siblingDbs = /* @__PURE__ */ new Map();
2933
+ }
2934
+ const cached = context.siblingDbs.get(siblingName);
2935
+ if (cached) {
2936
+ return cached;
2937
+ }
2938
+ const overrideParam = `dbConnectionStringSib${camelizeDbName(siblingName)}`;
2939
+ let connectionString = await context?.params?.get?.(overrideParam, "string");
2940
+ if (connectionString) {
2941
+ context.logger?.debug?.(
2942
+ `[Db] sibling "${siblingName}": using override param "${overrideParam}"`
2943
+ );
2944
+ } else {
2945
+ const baseHandle = options.baseDb ?? context.db;
2946
+ const base = baseHandle?.config?.connectionString ?? options.baseConnectionString;
2947
+ if (!base) {
2948
+ throw new ParamError(
2949
+ `Db.initSibling: no base connection to derive "${siblingName}" from \u2014 init the main Db first (context.db = await Db.init(context)), or pass options.baseDb / options.baseConnectionString, or set the ${overrideParam} param (env ${toEnvKey(overrideParam)})`
2950
+ );
2951
+ }
2952
+ const exists = await databaseExistsOnServer(baseHandle, base, siblingName);
2953
+ if (!exists) {
2954
+ context.logger?.debug?.(
2955
+ `[Db] sibling "${siblingName}" does not exist \u2014 falling back to the main database`
2956
+ );
2957
+ const main = await resolveMainHandler(context, options);
2958
+ context.siblingDbs.set(siblingName, main);
2959
+ return main;
2960
+ }
2961
+ connectionString = replaceDatabaseName(base, siblingName);
2962
+ context.logger?.debug?.(
2963
+ `[Db] sibling "${siblingName}": derived from base (${formatConnectionEndpoint(base) ?? "?"})`
2964
+ );
2965
+ }
2966
+ const config2 = {
2967
+ ...KNEX_DEFAULTS,
2968
+ connectionString,
2969
+ name: siblingName,
2970
+ profile: !!options.dbProfile,
2971
+ logger: context.logger
2972
+ };
2973
+ const handler = await dbConnect(context, config2);
2974
+ context.siblingDbs.set(siblingName, handler);
2975
+ return handler;
2976
+ }
2977
+ /**
2978
+ * Discover the sibling databases that are "currently in use", by name.
2979
+ * Two sources, merged (env wins on duplicates):
2980
+ *
2981
+ * 1. ENV-DECLARED — every `DB_CONNECTION_STRING_SIB_<NAME>` env var
2982
+ * (the dedicated `SIB_` namespace) whose decoded name matches. This
2983
+ * covers siblings that moved to their own server: the same var that
2984
+ * overrides the connection also *registers* the database, so `.env`
2985
+ * stays the single convenient list.
2986
+ * 2. SAME-SERVER SCAN — `SELECT datname FROM pg_database` on the base
2987
+ * server (PostgreSQL only), filtered the same way.
2988
+ *
2989
+ * The caller says what "matches": `{ prefix: "src_" }` or
2990
+ * `{ match: /^src_/ }` — the toolkit does not guess a naming convention.
2991
+ *
2992
+ * @param {object} context - needs context.db (or options.baseDb) for the server scan
2993
+ * @param {{ prefix?: string, match?: RegExp, baseDb?: object, env?: object }} options
2994
+ * @returns {Promise<Array<{ name: string, origin: "env"|"server" }>>} sorted by name
2995
+ */
2996
+ static async discoverSiblings(context, options = {}) {
2997
+ const { prefix, match, env = process.env } = options;
2998
+ if (!prefix && !match) {
2999
+ throw new ParamError(`Db.discoverSiblings: pass { prefix: "..." } or { match: /.../ }`);
3000
+ }
3001
+ const matches = match instanceof RegExp ? (n) => match.test(n) : (n) => n.startsWith(prefix);
3002
+ const found = /* @__PURE__ */ new Map();
3003
+ for (const key of Object.keys(env)) {
3004
+ const m = /^DB_CONNECTION_STRING_SIB_(.+)$/.exec(key);
3005
+ if (!m || !env[key]) continue;
3006
+ const name = m[1].toLowerCase();
3007
+ if (matches(name)) {
3008
+ found.set(name, "env");
3009
+ }
3010
+ }
3011
+ const base = options.baseDb ?? context?.db;
3012
+ if (base) {
3013
+ const connectionString = String(base.config?.connectionString ?? "");
3014
+ if (connectionString.startsWith("postgresql")) {
3015
+ const { rows } = await base.raw(
3016
+ "SELECT datname FROM pg_database WHERE datistemplate = false"
3017
+ );
3018
+ for (const { datname } of rows) {
3019
+ if (matches(datname) && !found.has(datname)) {
3020
+ found.set(datname, "server");
3021
+ }
3022
+ }
3023
+ } else {
3024
+ context?.logger?.warn?.(
3025
+ "[Db] discoverSiblings: server scan supported for PostgreSQL only; using env-declared siblings"
3026
+ );
3027
+ }
3028
+ }
3029
+ return [...found].map(([name, origin]) => ({ name, origin })).sort((a, b) => a.name.localeCompare(b.name));
3030
+ }
3031
+ /**
3032
+ * Discover + connect: one handler per active sibling (cached, pooled —
3033
+ * see initSibling). Pass `includeMain: true` to get `[context.db, ...]`,
3034
+ * which is the usual shape for "apply this DDL everywhere" loops:
3035
+ *
3036
+ * const dbs = await Db.initAllSiblings(context, { prefix: "src_", includeMain: true });
3037
+ * await ensureSchemaEverywhere(dbs, spec, { logger: context.logger });
3038
+ *
3039
+ * @param {object} context
3040
+ * @param {{ prefix?: string, match?: RegExp, includeMain?: boolean, baseDb?: object, env?: object }} options
3041
+ * @returns {Promise<Function[]>} connected handlers
3042
+ */
3043
+ static async initAllSiblings(context, options = {}) {
3044
+ const discovered = await _Db.discoverSiblings(context, options);
3045
+ const handlers = [];
3046
+ for (const { name } of discovered) {
3047
+ handlers.push(await _Db.initSibling(context, name, options));
3048
+ }
3049
+ return options.includeMain && context.db ? [context.db, ...handlers] : handlers;
3050
+ }
2738
3051
  constructor(config2) {
2739
3052
  if (!config2 || !config2.connectionString) {
2740
3053
  throw new ParamError("Db: connectionString is required");
@@ -2995,6 +3308,54 @@ var Db = class {
2995
3308
  function capitalizeFirstLetter(str) {
2996
3309
  return str.charAt(0).toUpperCase() + str.slice(1);
2997
3310
  }
3311
+ async function resolveMainHandler(context, options = {}) {
3312
+ if (options.baseDb) {
3313
+ return options.baseDb;
3314
+ }
3315
+ if (!context.db) {
3316
+ context.db = await Db.init(context);
3317
+ }
3318
+ return context.db;
3319
+ }
3320
+ async function databaseExistsOnServer(baseHandle, baseConnectionString, databaseName) {
3321
+ if (!String(baseConnectionString).startsWith("postgresql")) {
3322
+ return true;
3323
+ }
3324
+ const query = "SELECT 1 FROM pg_database WHERE datname = ?";
3325
+ if (baseHandle) {
3326
+ const { rows } = await baseHandle.raw(query, [databaseName]);
3327
+ return rows.length > 0;
3328
+ }
3329
+ const shortLived = (0, import_knex.default)({
3330
+ client: "pg",
3331
+ connection: { connectionString: baseConnectionString },
3332
+ pool: { min: 0, max: 1 }
3333
+ });
3334
+ try {
3335
+ const { rows } = await shortLived.raw(query, [databaseName]);
3336
+ return rows.length > 0;
3337
+ } finally {
3338
+ await shortLived.destroy();
3339
+ }
3340
+ }
3341
+ function camelizeDbName(name) {
3342
+ return name.split(/[_-]+/).filter(Boolean).map(capitalizeFirstLetter).join("");
3343
+ }
3344
+ function toEnvKey(key) {
3345
+ return key.replace(/([A-Z])/g, "_$1").toUpperCase();
3346
+ }
3347
+ function replaceDatabaseName(connectionString, databaseName) {
3348
+ let url;
3349
+ try {
3350
+ url = new URL(connectionString);
3351
+ } catch {
3352
+ throw new ParamError(
3353
+ `Db: cannot parse connection string to derive a sibling database from it`
3354
+ );
3355
+ }
3356
+ url.pathname = `/${databaseName}`;
3357
+ return url.toString();
3358
+ }
2998
3359
  function resolveDbDisplayName(dbName, connectionParam, argsEnv, mergedName) {
2999
3360
  if (dbName) {
3000
3361
  return dbName;
@@ -3247,29 +3608,90 @@ function queueToTableNames(queueName) {
3247
3608
  registryTable: `${queueName}_services_registry`
3248
3609
  };
3249
3610
  }
3250
- function defineTasksTable(t, db, tableNameForIndex) {
3251
- t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()"));
3252
- t.timestamp("created_at").notNullable().defaultTo(db.fn.now());
3253
- t.timestamp("started_at");
3254
- t.timestamp("completed_at");
3255
- t.integer("priority").notNullable().defaultTo(50);
3256
- t.text("schedule");
3257
- t.timestamp("next_run_at").defaultTo(null);
3258
- t.timestamp("past_due").defaultTo(null);
3259
- t.text("name").notNullable();
3260
- t.text("opid");
3261
- t.jsonb("params");
3262
- t.text("service_group");
3263
- t.integer("instance_number");
3264
- t.text("service_name");
3265
- t.text("server_name");
3266
- t.text("status").notNullable().defaultTo("idle");
3267
- t.timestamp("status_changed_at").defaultTo(null);
3268
- t.text("progress");
3269
- t.boolean("success");
3270
- t.jsonb("results");
3271
- t.index(["service_group", "status", "priority", "created_at"], `${tableNameForIndex}_claim_idx`);
3272
- t.index(["service_group", "name"], `${tableNameForIndex}_group_name_idx`);
3611
+ function tasksTableSpec(tableNameForIndex) {
3612
+ return {
3613
+ columns: {
3614
+ id: (t, db) => t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()")),
3615
+ created_at: (t, db) => t.timestamp("created_at").notNullable().defaultTo(db.fn.now()),
3616
+ started_at: (t) => t.timestamp("started_at"),
3617
+ completed_at: (t) => t.timestamp("completed_at"),
3618
+ /*
3619
+ * Priority: lower number = claimed first (see claimNextRunnableTask: ORDER BY priority ASC).
3620
+ * Suggested range 0–100: 0 = most urgent, 100 = least; default 50 for normal work.
3621
+ */
3622
+ priority: (t) => t.integer("priority").notNullable().defaultTo(50),
3623
+ schedule: (t) => t.text("schedule"),
3624
+ next_run_at: (t) => t.timestamp("next_run_at").defaultTo(null),
3625
+ past_due: (t) => t.timestamp("past_due").defaultTo(null),
3626
+ name: (t) => t.text("name").notNullable(),
3627
+ opid: (t) => t.text("opid"),
3628
+ params: (t) => t.jsonb("params"),
3629
+ // those are target identifiers, kind of who is going to run a task.
3630
+ service_group: (t) => t.text("service_group"),
3631
+ // harvester, loader, photos, ...
3632
+ instance_number: (t) => t.integer("instance_number"),
3633
+ service_name: (t) => t.text("service_name"),
3634
+ // that's a "<server_name>_<service_group>_<instance_number>"
3635
+ server_name: (t) => t.text("server_name"),
3636
+ // filled by runner when registering, auto.
3637
+ status: (t) => t.text("status").notNullable().defaultTo("idle"),
3638
+ // idle, running, completed, failed, paused
3639
+ status_changed_at: (t) => t.timestamp("status_changed_at").defaultTo(null),
3640
+ progress: (t) => t.text("progress"),
3641
+ success: (t) => t.boolean("success"),
3642
+ results: (t) => t.jsonb("results")
3643
+ },
3644
+ indexes: [
3645
+ {
3646
+ columns: ["service_group", "status", "priority", "created_at"],
3647
+ name: `${tableNameForIndex}_claim_idx`
3648
+ },
3649
+ { columns: ["service_group", "name"], name: `${tableNameForIndex}_group_name_idx` }
3650
+ ]
3651
+ };
3652
+ }
3653
+ function registryTableSpec(registryTable) {
3654
+ return {
3655
+ columns: {
3656
+ id: (t, db) => t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()")),
3657
+ queue_name: (t) => t.text("queue_name").notNullable(),
3658
+ service_group: (t) => t.text("service_group").notNullable(),
3659
+ // harvester, loader, photos, ...
3660
+ instance_number: (t) => t.integer("instance_number").notNullable().defaultTo(1),
3661
+ service_name: (t) => t.text("service_name").notNullable(),
3662
+ // that's a "<server_name>_<service_group>_<instance_number>"
3663
+ server_name: (t) => t.text("server_name").notNullable(),
3664
+ // filled by runner when registering, auto.
3665
+ pid: (t) => t.integer("pid"),
3666
+ metadata: (t) => t.json("metadata"),
3667
+ created_at: (t, db) => t.timestamp("created_at").notNullable().defaultTo(db.fn.now()),
3668
+ last_seen_at: (t, db) => t.timestamp("last_seen_at").notNullable().defaultTo(db.fn.now())
3669
+ },
3670
+ indexes: [
3671
+ {
3672
+ columns: ["queue_name", "service_name"],
3673
+ name: `${registryTable}_queue_name_service_name_uniq`,
3674
+ unique: true
3675
+ },
3676
+ {
3677
+ columns: ["queue_name", "service_group", "last_seen_at"],
3678
+ name: `${registryTable}_queue_group_seen_idx`
3679
+ },
3680
+ { columns: ["queue_name", "last_seen_at"], name: `${registryTable}_queue_seen_idx` }
3681
+ ]
3682
+ // TODO: a reference is needed - task_history entry should reference the registry entry, so that we can easily find all executed tasks for a given service and calculate the average workload or identify if there's a bottleneck. Also may be used for the load balancing.
3683
+ };
3684
+ }
3685
+ function tasksSchemaSpec(queueName = "tasks") {
3686
+ const { tasksTable, historyTable, registryTable } = queueToTableNames(queueName);
3687
+ return {
3688
+ extensions: ["uuid-ossp"],
3689
+ tables: {
3690
+ [tasksTable]: tasksTableSpec(tasksTable),
3691
+ [historyTable]: tasksTableSpec(historyTable),
3692
+ [registryTable]: registryTableSpec(registryTable)
3693
+ }
3694
+ };
3273
3695
  }
3274
3696
  function taskHistoryInsertFromQueueRow(row, overrides) {
3275
3697
  const { id, ...snapshot } = row;
@@ -3283,60 +3705,38 @@ async function ensureTaskTables(context, options = {}) {
3283
3705
  const queueName = options.queueName ?? "tasks";
3284
3706
  const recreate = options.recreate ?? false;
3285
3707
  const dryRun = options.dryRun ?? false;
3286
- const db = getDb(context);
3708
+ const databases = options.databases ?? [getDb(context)];
3287
3709
  const log = context.logger ?? console;
3288
3710
  const { tasksTable, historyTable, registryTable } = queueToTableNames(queueName);
3289
- const needsTasks = recreate ? true : !await db.tableExists(tasksTable);
3290
- const needsHistory = recreate ? true : !await db.tableExists(historyTable);
3291
- const needsRegistry = recreate ? true : !await db.tableExists(registryTable);
3292
- if (dryRun) {
3293
- const plan = [];
3711
+ const spec = tasksSchemaSpec(queueName);
3712
+ for (const db of databases) {
3713
+ const label = db?.config?.name ?? "db";
3294
3714
  if (recreate) {
3295
- plan.push(`DROP TABLE IF EXISTS ${historyTable}, ${tasksTable}, ${registryTable}`);
3715
+ if (dryRun) {
3716
+ log.info?.(
3717
+ `[tasks-schema] dryRun \u2014 ${label}: DROP TABLE IF EXISTS ${historyTable}, ${tasksTable}, ${registryTable}`
3718
+ );
3719
+ } else {
3720
+ await db.schema.dropTableIfExists(historyTable);
3721
+ await db.schema.dropTableIfExists(tasksTable);
3722
+ await db.schema.dropTableIfExists(registryTable);
3723
+ }
3296
3724
  }
3297
- if (needsTasks) plan.push(`CREATE TABLE ${tasksTable} (tasks queue)`);
3298
- if (needsHistory) plan.push(`CREATE TABLE ${historyTable} (history mirror)`);
3299
- if (needsRegistry) plan.push(`CREATE TABLE ${registryTable} (services registry)`);
3300
- if (plan.length === 0) {
3301
- log.info?.(`[tasks-schema] dryRun \u2014 queue "${queueName}" already up to date; no DDL`);
3302
- } else {
3303
- log.info?.(`[tasks-schema] dryRun \u2014 would run ${plan.length} statement(s) for queue "${queueName}":`);
3304
- for (const s of plan) log.info?.(` - ${s}`);
3725
+ const { actions } = await ensureSchema(db, spec, { dryRun, logger: context.logger });
3726
+ if (dryRun) {
3727
+ if (actions.length === 0) {
3728
+ log.info?.(`[tasks-schema] dryRun \u2014 ${label}: queue "${queueName}" already up to date; no DDL`);
3729
+ } else {
3730
+ log.info?.(
3731
+ `[tasks-schema] dryRun \u2014 ${label}: would run ${actions.length} statement(s) for queue "${queueName}":`
3732
+ );
3733
+ for (const s of actions) log.info?.(` - ${s}`);
3734
+ }
3735
+ } else if (actions.length > 0) {
3736
+ log.info?.(
3737
+ `[tasks-schema] ${label}: applied ${actions.length} DDL statement(s) for queue "${queueName}"`
3738
+ );
3305
3739
  }
3306
- return;
3307
- }
3308
- if (recreate) {
3309
- await db.schema.dropTableIfExists(historyTable);
3310
- await db.schema.dropTableIfExists(tasksTable);
3311
- await db.schema.dropTableIfExists(registryTable);
3312
- }
3313
- if (needsTasks) {
3314
- await db.raw(`CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`);
3315
- await db.schema.createTable(tasksTable, (t) => {
3316
- defineTasksTable(t, db, tasksTable);
3317
- });
3318
- }
3319
- if (needsHistory) {
3320
- await db.schema.createTable(historyTable, (t) => {
3321
- defineTasksTable(t, db, historyTable);
3322
- });
3323
- }
3324
- if (needsRegistry) {
3325
- await db.schema.createTable(registryTable, (t) => {
3326
- t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()"));
3327
- t.text("queue_name").notNullable();
3328
- t.text("service_group").notNullable();
3329
- t.integer("instance_number").notNullable().defaultTo(1);
3330
- t.text("service_name").notNullable();
3331
- t.text("server_name").notNullable();
3332
- t.integer("pid");
3333
- t.json("metadata");
3334
- t.timestamp("created_at").notNullable().defaultTo(db.fn.now());
3335
- t.timestamp("last_seen_at").notNullable().defaultTo(db.fn.now());
3336
- t.unique(["queue_name", "service_name"], `${registryTable}_queue_name_service_name_uniq`);
3337
- t.index(["queue_name", "service_group", "last_seen_at"], `${registryTable}_queue_group_seen_idx`);
3338
- t.index(["queue_name", "last_seen_at"], `${registryTable}_queue_seen_idx`);
3339
- });
3340
3740
  }
3341
3741
  }
3342
3742
  async function updateTaskProgress(context, tasksTable, taskId, progress) {