@nmakarov/cli-toolkit 0.33.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-runner.cjs +473 -73
- package/dist/cli-runner.cjs.map +1 -1
- package/dist/cli-runner.js +473 -73
- package/dist/cli-runner.js.map +1 -1
- package/dist/db.cjs +332 -3
- package/dist/db.cjs.map +1 -1
- package/dist/db.js +325 -2
- package/dist/db.js.map +1 -1
- package/dist/index.cjs +498 -73
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +492 -73
- package/dist/index.js.map +1 -1
- package/dist/init.cjs +55 -0
- package/dist/init.cjs.map +1 -1
- package/dist/init.js +55 -0
- package/dist/init.js.map +1 -1
- package/dist/params.cjs +55 -0
- package/dist/params.cjs.map +1 -1
- package/dist/params.js +55 -0
- package/dist/params.js.map +1 -1
- package/dist/tasks.cjs +196 -72
- package/dist/tasks.cjs.map +1 -1
- package/dist/tasks.js +196 -72
- package/dist/tasks.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1129,8 +1129,13 @@ __export(src_exports, {
|
|
|
1129
1129
|
enqueueTask: () => enqueueTask,
|
|
1130
1130
|
ensureDeployKeyOnRemote: () => ensureDeployKeyOnRemote,
|
|
1131
1131
|
ensureEnvOnRemote: () => ensureEnvOnRemote,
|
|
1132
|
+
ensureExtension: () => ensureExtension,
|
|
1133
|
+
ensureIndex: () => ensureIndex,
|
|
1132
1134
|
ensureRemoteRepo: () => ensureRemoteRepo,
|
|
1133
1135
|
ensureRepoDependencies: () => ensureRepoDependencies,
|
|
1136
|
+
ensureSchema: () => ensureSchema,
|
|
1137
|
+
ensureSchemaEverywhere: () => ensureSchemaEverywhere,
|
|
1138
|
+
ensureTable: () => ensureTable,
|
|
1134
1139
|
ensureTaskTables: () => ensureTaskTables,
|
|
1135
1140
|
flushTaskIpcLogs: () => flushTaskIpcLogs,
|
|
1136
1141
|
getArgsInstance: () => getArgsInstance,
|
|
@@ -1169,6 +1174,7 @@ __export(src_exports, {
|
|
|
1169
1174
|
releaseDir: () => releaseDir,
|
|
1170
1175
|
releaseStamp: () => releaseStamp,
|
|
1171
1176
|
reloadPm2: () => reloadPm2,
|
|
1177
|
+
replaceDatabaseName: () => replaceDatabaseName,
|
|
1172
1178
|
resolveAsterisks: () => resolveAsterisks,
|
|
1173
1179
|
resolveIpcFileLogsDir: () => resolveIpcFileLogsDir,
|
|
1174
1180
|
resolveNextVersion: () => resolveNextVersion,
|
|
@@ -1966,6 +1972,49 @@ var Params = class _Params {
|
|
|
1966
1972
|
module: moduleName ?? this._currentModule
|
|
1967
1973
|
});
|
|
1968
1974
|
}
|
|
1975
|
+
/**
|
|
1976
|
+
* Report a param value that a component RESOLVED ON ITS OWN — outside
|
|
1977
|
+
* Params.get() — e.g. discovered by combining its config files (the way
|
|
1978
|
+
* blueprints merge defaults/aggregator/feed data). Without this, every
|
|
1979
|
+
* key such a component probed for an override shows up in the
|
|
1980
|
+
* --showUsedParams dump as "undefined (default)"; reporting upgrades the
|
|
1981
|
+
* entry to the value the component actually works with.
|
|
1982
|
+
*
|
|
1983
|
+
* Attribution rules:
|
|
1984
|
+
* - entries figured from an explicit input (cli/env/options/…) are left
|
|
1985
|
+
* untouched — the component merely confirmed them, the origin stands;
|
|
1986
|
+
* - "default" entries (Params had nothing) and earlier reports are
|
|
1987
|
+
* replaced by the report;
|
|
1988
|
+
* - keys never seen by Params are appended as new entries.
|
|
1989
|
+
* The LATEST report wins — a component may resolve the same key several
|
|
1990
|
+
* times with increasing specificity (e.g. a blueprint re-merged for a
|
|
1991
|
+
* concrete resource) and the dump should show what it settled on.
|
|
1992
|
+
* Later params.get() probes that find nothing ("default") never shadow a
|
|
1993
|
+
* reported value — see {@link getFiguredByModule}.
|
|
1994
|
+
*
|
|
1995
|
+
* @param {string} key
|
|
1996
|
+
* @param {*} value - the value the component actually uses
|
|
1997
|
+
* @param {string} [source="discovered"] - short origin label, e.g. "blueprint"
|
|
1998
|
+
* @param {string} [moduleName] - dump section; defaults to the current module
|
|
1999
|
+
*/
|
|
2000
|
+
reportResolved(key, value, source = "discovered", moduleName) {
|
|
2001
|
+
const mod = moduleName ?? this._currentModule;
|
|
2002
|
+
const mine = this.trackedParams.filter((e) => e.key === key && e.module === mod);
|
|
2003
|
+
if (mine.some((e) => e.source !== "default" && !e.reported)) {
|
|
2004
|
+
return;
|
|
2005
|
+
}
|
|
2006
|
+
this.trackedParams = this.trackedParams.filter(
|
|
2007
|
+
(e) => !(e.key === key && e.module === mod)
|
|
2008
|
+
);
|
|
2009
|
+
this.trackedParams.push({
|
|
2010
|
+
key,
|
|
2011
|
+
definition: "reported",
|
|
2012
|
+
value,
|
|
2013
|
+
source,
|
|
2014
|
+
module: mod,
|
|
2015
|
+
reported: true
|
|
2016
|
+
});
|
|
2017
|
+
}
|
|
1969
2018
|
/**
|
|
1970
2019
|
* Get all tracked parameters (for --stopAfter=init)
|
|
1971
2020
|
*/
|
|
@@ -1990,12 +2039,24 @@ var Params = class _Params {
|
|
|
1990
2039
|
/**
|
|
1991
2040
|
* Get figured parameters grouped by module name.
|
|
1992
2041
|
* Same param can appear in multiple modules (e.g. source, resource).
|
|
2042
|
+
* Last occurrence per key wins, EXCEPT that an empty probe — a
|
|
2043
|
+
* params.get() that found nothing ("default", undefined) — never shadows
|
|
2044
|
+
* a value reported via {@link reportResolved}: components probe for
|
|
2045
|
+
* overrides on every resolution cycle, and those misses say nothing about
|
|
2046
|
+
* the value the component actually uses.
|
|
1993
2047
|
*/
|
|
1994
2048
|
getFiguredByModule() {
|
|
2049
|
+
const reported = /* @__PURE__ */ new Set();
|
|
2050
|
+
for (const param of this.trackedParams) {
|
|
2051
|
+
if (param.reported) reported.add(`${param.module}\0${param.key}`);
|
|
2052
|
+
}
|
|
1995
2053
|
const byModule = {};
|
|
1996
2054
|
for (const param of this.trackedParams) {
|
|
1997
2055
|
const mod = param.module;
|
|
1998
2056
|
if (!byModule[mod]) byModule[mod] = {};
|
|
2057
|
+
if (!param.reported && param.source === "default" && reported.has(`${mod}\0${param.key}`)) {
|
|
2058
|
+
continue;
|
|
2059
|
+
}
|
|
1999
2060
|
byModule[mod][param.key] = { value: param.value, source: param.source };
|
|
2000
2061
|
}
|
|
2001
2062
|
return byModule;
|
|
@@ -3471,13 +3532,113 @@ function listSources(basePath) {
|
|
|
3471
3532
|
|
|
3472
3533
|
// src/db/index.js
|
|
3473
3534
|
var import_knex = __toESM(require("knex"), 1);
|
|
3535
|
+
|
|
3536
|
+
// src/db/ensure.js
|
|
3537
|
+
var dbLabel = (db) => db?.config?.name ?? "db";
|
|
3538
|
+
async function ensureExtension(db, name, options = {}) {
|
|
3539
|
+
const action = `CREATE EXTENSION IF NOT EXISTS "${name}"`;
|
|
3540
|
+
if (!options.dryRun) {
|
|
3541
|
+
await db.raw(action);
|
|
3542
|
+
}
|
|
3543
|
+
options.logger?.silly?.(`[ensure] ${dbLabel(db)}: ${action}${options.dryRun ? " (dryRun)" : ""}`);
|
|
3544
|
+
return { action };
|
|
3545
|
+
}
|
|
3546
|
+
async function ensureTable(db, tableName, spec, options = {}) {
|
|
3547
|
+
const { dryRun = false, logger } = options;
|
|
3548
|
+
const actions = [];
|
|
3549
|
+
const exists = await db.tableExists(tableName);
|
|
3550
|
+
if (!exists) {
|
|
3551
|
+
actions.push(`CREATE TABLE ${tableName} (${Object.keys(spec.columns).length} columns)`);
|
|
3552
|
+
if (!dryRun) {
|
|
3553
|
+
await db.schema.createTable(tableName, (t) => {
|
|
3554
|
+
for (const define of Object.values(spec.columns)) {
|
|
3555
|
+
define(t, db);
|
|
3556
|
+
}
|
|
3557
|
+
});
|
|
3558
|
+
}
|
|
3559
|
+
} else {
|
|
3560
|
+
const missing = [];
|
|
3561
|
+
for (const column of Object.keys(spec.columns)) {
|
|
3562
|
+
if (!await db.schema.hasColumn(tableName, column)) {
|
|
3563
|
+
missing.push(column);
|
|
3564
|
+
}
|
|
3565
|
+
}
|
|
3566
|
+
if (missing.length > 0) {
|
|
3567
|
+
actions.push(`ALTER TABLE ${tableName} ADD COLUMN ${missing.join(", ")}`);
|
|
3568
|
+
if (!dryRun) {
|
|
3569
|
+
await db.schema.alterTable(tableName, (t) => {
|
|
3570
|
+
for (const column of missing) {
|
|
3571
|
+
spec.columns[column](t, db);
|
|
3572
|
+
}
|
|
3573
|
+
});
|
|
3574
|
+
}
|
|
3575
|
+
}
|
|
3576
|
+
}
|
|
3577
|
+
for (const index of spec.indexes ?? []) {
|
|
3578
|
+
const indexActions = await ensureIndex(db, tableName, index, options);
|
|
3579
|
+
actions.push(...indexActions);
|
|
3580
|
+
}
|
|
3581
|
+
for (const action of actions) {
|
|
3582
|
+
logger?.silly?.(`[ensure] ${dbLabel(db)}: ${action}${dryRun ? " (dryRun)" : ""}`);
|
|
3583
|
+
}
|
|
3584
|
+
return actions;
|
|
3585
|
+
}
|
|
3586
|
+
async function ensureIndex(db, tableName, index, options = {}) {
|
|
3587
|
+
const { dryRun = false } = options;
|
|
3588
|
+
const kind = index.unique ? "UNIQUE INDEX" : "INDEX";
|
|
3589
|
+
const cols = index.columns.map((c) => `"${c}"`).join(", ");
|
|
3590
|
+
const isPg = String(db?.config?.connectionString ?? "").startsWith("postgresql");
|
|
3591
|
+
if (isPg) {
|
|
3592
|
+
const sql2 = `CREATE ${kind} IF NOT EXISTS "${index.name}" ON "${tableName}" (${cols})`;
|
|
3593
|
+
const { rows } = await db.raw(`SELECT 1 FROM pg_indexes WHERE indexname = ?`, [index.name]);
|
|
3594
|
+
if (rows.length > 0) return [];
|
|
3595
|
+
if (!dryRun) await db.raw(sql2);
|
|
3596
|
+
return [sql2];
|
|
3597
|
+
}
|
|
3598
|
+
const sql = `CREATE ${kind} ${index.name} ON ${tableName} (${cols})`;
|
|
3599
|
+
if (!dryRun) {
|
|
3600
|
+
try {
|
|
3601
|
+
await db.raw(sql);
|
|
3602
|
+
} catch (error) {
|
|
3603
|
+
if (!/already exists|duplicate/i.test(error?.message ?? "")) throw error;
|
|
3604
|
+
return [];
|
|
3605
|
+
}
|
|
3606
|
+
}
|
|
3607
|
+
return [sql];
|
|
3608
|
+
}
|
|
3609
|
+
async function ensureSchema(db, spec, options = {}) {
|
|
3610
|
+
const actions = [];
|
|
3611
|
+
for (const extension of spec.extensions ?? []) {
|
|
3612
|
+
const { action } = await ensureExtension(db, extension, options);
|
|
3613
|
+
if (options.dryRun) actions.push(action);
|
|
3614
|
+
}
|
|
3615
|
+
for (const [tableName, tableSpec] of Object.entries(spec.tables ?? {})) {
|
|
3616
|
+
actions.push(...await ensureTable(db, tableName, tableSpec, options));
|
|
3617
|
+
}
|
|
3618
|
+
return { database: dbLabel(db), actions };
|
|
3619
|
+
}
|
|
3620
|
+
async function ensureSchemaEverywhere(dbs, spec, options = {}) {
|
|
3621
|
+
const reports = [];
|
|
3622
|
+
for (const db of dbs) {
|
|
3623
|
+
const report = await ensureSchema(db, spec, options);
|
|
3624
|
+
if (report.actions.length > 0) {
|
|
3625
|
+
options.logger?.info?.(
|
|
3626
|
+
`[ensure] ${report.database}: ${options.dryRun ? "would apply" : "applied"} ${report.actions.length} DDL statement(s)`
|
|
3627
|
+
);
|
|
3628
|
+
}
|
|
3629
|
+
reports.push(report);
|
|
3630
|
+
}
|
|
3631
|
+
return reports;
|
|
3632
|
+
}
|
|
3633
|
+
|
|
3634
|
+
// src/db/index.js
|
|
3474
3635
|
var KNEX_DEFAULTS = {
|
|
3475
3636
|
testConnection: true,
|
|
3476
3637
|
pool: { min: 2, max: 10 },
|
|
3477
3638
|
acquireConnectionTimeout: 1e4,
|
|
3478
3639
|
ssl: { rejectUnauthorized: false }
|
|
3479
3640
|
};
|
|
3480
|
-
var Db = class {
|
|
3641
|
+
var Db = class _Db {
|
|
3481
3642
|
static async init(context, options = {}) {
|
|
3482
3643
|
const buildConfig = async () => {
|
|
3483
3644
|
const defs = {
|
|
@@ -3546,6 +3707,177 @@ var Db = class {
|
|
|
3546
3707
|
const config2 = context?.params?.runWithModuleAsync ? await context.params.runWithModuleAsync("db", buildConfig) : await buildConfig();
|
|
3547
3708
|
return dbConnect(context, config2);
|
|
3548
3709
|
}
|
|
3710
|
+
/**
|
|
3711
|
+
* Initialize a SIBLING database handler: a database that lives on the same
|
|
3712
|
+
* server with the same credentials/options as an existing ("base") one, and
|
|
3713
|
+
* differs only by its database name. Typical use: a per-tenant / per-subject
|
|
3714
|
+
* database alongside a main database that keeps the shared tables.
|
|
3715
|
+
*
|
|
3716
|
+
* context.db = await Db.init(context); // main
|
|
3717
|
+
* const sub = await Db.initSibling(context, "src_bright"); // sibling
|
|
3718
|
+
*
|
|
3719
|
+
* LOCATION-AGNOSTIC BY DESIGN — the call gracefully falls back to the main
|
|
3720
|
+
* database, so callers can use it for all subject data without knowing what
|
|
3721
|
+
* has been migrated where:
|
|
3722
|
+
* - empty `siblingName` → the main handler (same as Db.init)
|
|
3723
|
+
* - sibling DB does not exist yet → the main handler (data not migrated;
|
|
3724
|
+
* it still lives in the main database)
|
|
3725
|
+
* Callers that must not fall back can check `handler === context.db`.
|
|
3726
|
+
*
|
|
3727
|
+
* Connection string resolution for an actual sibling, in order:
|
|
3728
|
+
* 1. Explicit override — param `dbConnectionStringSib<SiblingName>` (env
|
|
3729
|
+
* `DB_CONNECTION_STRING_SIB_<SIBLING_NAME>`, e.g. `src_bright` →
|
|
3730
|
+
* `DB_CONNECTION_STRING_SIB_SRC_BRIGHT`). Nobody needs this on day
|
|
3731
|
+
* one; it is the escape hatch for when a sibling later moves to its
|
|
3732
|
+
* own server — one env var, no code changes (the `SIB_` namespace
|
|
3733
|
+
* both overrides the connection and declares that the sibling
|
|
3734
|
+
* exists, so it never falls back to main).
|
|
3735
|
+
* 2. Derived — take the base connection string and swap the database
|
|
3736
|
+
* name (after confirming the database exists on that server). The
|
|
3737
|
+
* base is `options.baseDb` (a Db handler), then
|
|
3738
|
+
* `options.baseConnectionString`, then `context.db`.
|
|
3739
|
+
*
|
|
3740
|
+
* Handlers are cached per name on the context (`context.siblingDbs`), so
|
|
3741
|
+
* any number of components asking for the same sibling share one pool —
|
|
3742
|
+
* including the "falls back to main" answer, which is remembered for the
|
|
3743
|
+
* lifetime of the process (a mid-run migration is picked up on restart).
|
|
3744
|
+
* Disconnect is registered via `context.registerCleanup`, same as the
|
|
3745
|
+
* main handler.
|
|
3746
|
+
*
|
|
3747
|
+
* @param {object} context - context with params/logger (and usually .db)
|
|
3748
|
+
* @param {string} [siblingName] - the sibling's database name (e.g. "src_bright")
|
|
3749
|
+
* @param {object} [options] - { baseDb, baseConnectionString, dbProfile }
|
|
3750
|
+
* @returns {Promise<Db>} connected handler (same proxy shape as Db.init)
|
|
3751
|
+
*/
|
|
3752
|
+
static async initSibling(context, siblingName, options = {}) {
|
|
3753
|
+
if (!siblingName) {
|
|
3754
|
+
return resolveMainHandler(context, options);
|
|
3755
|
+
}
|
|
3756
|
+
if (!/^[a-zA-Z0-9_]+$/.test(siblingName)) {
|
|
3757
|
+
throw new ParamError(
|
|
3758
|
+
`Db.initSibling: invalid sibling database name "${siblingName}" (letters, digits and _ only)`
|
|
3759
|
+
);
|
|
3760
|
+
}
|
|
3761
|
+
if (!context.siblingDbs) {
|
|
3762
|
+
context.siblingDbs = /* @__PURE__ */ new Map();
|
|
3763
|
+
}
|
|
3764
|
+
const cached = context.siblingDbs.get(siblingName);
|
|
3765
|
+
if (cached) {
|
|
3766
|
+
return cached;
|
|
3767
|
+
}
|
|
3768
|
+
const overrideParam = `dbConnectionStringSib${camelizeDbName(siblingName)}`;
|
|
3769
|
+
let connectionString = await context?.params?.get?.(overrideParam, "string");
|
|
3770
|
+
if (connectionString) {
|
|
3771
|
+
context.logger?.debug?.(
|
|
3772
|
+
`[Db] sibling "${siblingName}": using override param "${overrideParam}"`
|
|
3773
|
+
);
|
|
3774
|
+
} else {
|
|
3775
|
+
const baseHandle = options.baseDb ?? context.db;
|
|
3776
|
+
const base = baseHandle?.config?.connectionString ?? options.baseConnectionString;
|
|
3777
|
+
if (!base) {
|
|
3778
|
+
throw new ParamError(
|
|
3779
|
+
`Db.initSibling: no base connection to derive "${siblingName}" from \u2014 init the main Db first (context.db = await Db.init(context)), or pass options.baseDb / options.baseConnectionString, or set the ${overrideParam} param (env ${toEnvKey(overrideParam)})`
|
|
3780
|
+
);
|
|
3781
|
+
}
|
|
3782
|
+
const exists = await databaseExistsOnServer(baseHandle, base, siblingName);
|
|
3783
|
+
if (!exists) {
|
|
3784
|
+
context.logger?.debug?.(
|
|
3785
|
+
`[Db] sibling "${siblingName}" does not exist \u2014 falling back to the main database`
|
|
3786
|
+
);
|
|
3787
|
+
const main = await resolveMainHandler(context, options);
|
|
3788
|
+
context.siblingDbs.set(siblingName, main);
|
|
3789
|
+
return main;
|
|
3790
|
+
}
|
|
3791
|
+
connectionString = replaceDatabaseName(base, siblingName);
|
|
3792
|
+
context.logger?.debug?.(
|
|
3793
|
+
`[Db] sibling "${siblingName}": derived from base (${formatConnectionEndpoint(base) ?? "?"})`
|
|
3794
|
+
);
|
|
3795
|
+
}
|
|
3796
|
+
const config2 = {
|
|
3797
|
+
...KNEX_DEFAULTS,
|
|
3798
|
+
connectionString,
|
|
3799
|
+
name: siblingName,
|
|
3800
|
+
profile: !!options.dbProfile,
|
|
3801
|
+
logger: context.logger
|
|
3802
|
+
};
|
|
3803
|
+
const handler = await dbConnect(context, config2);
|
|
3804
|
+
context.siblingDbs.set(siblingName, handler);
|
|
3805
|
+
return handler;
|
|
3806
|
+
}
|
|
3807
|
+
/**
|
|
3808
|
+
* Discover the sibling databases that are "currently in use", by name.
|
|
3809
|
+
* Two sources, merged (env wins on duplicates):
|
|
3810
|
+
*
|
|
3811
|
+
* 1. ENV-DECLARED — every `DB_CONNECTION_STRING_SIB_<NAME>` env var
|
|
3812
|
+
* (the dedicated `SIB_` namespace) whose decoded name matches. This
|
|
3813
|
+
* covers siblings that moved to their own server: the same var that
|
|
3814
|
+
* overrides the connection also *registers* the database, so `.env`
|
|
3815
|
+
* stays the single convenient list.
|
|
3816
|
+
* 2. SAME-SERVER SCAN — `SELECT datname FROM pg_database` on the base
|
|
3817
|
+
* server (PostgreSQL only), filtered the same way.
|
|
3818
|
+
*
|
|
3819
|
+
* The caller says what "matches": `{ prefix: "src_" }` or
|
|
3820
|
+
* `{ match: /^src_/ }` — the toolkit does not guess a naming convention.
|
|
3821
|
+
*
|
|
3822
|
+
* @param {object} context - needs context.db (or options.baseDb) for the server scan
|
|
3823
|
+
* @param {{ prefix?: string, match?: RegExp, baseDb?: object, env?: object }} options
|
|
3824
|
+
* @returns {Promise<Array<{ name: string, origin: "env"|"server" }>>} sorted by name
|
|
3825
|
+
*/
|
|
3826
|
+
static async discoverSiblings(context, options = {}) {
|
|
3827
|
+
const { prefix, match, env = process.env } = options;
|
|
3828
|
+
if (!prefix && !match) {
|
|
3829
|
+
throw new ParamError(`Db.discoverSiblings: pass { prefix: "..." } or { match: /.../ }`);
|
|
3830
|
+
}
|
|
3831
|
+
const matches = match instanceof RegExp ? (n) => match.test(n) : (n) => n.startsWith(prefix);
|
|
3832
|
+
const found = /* @__PURE__ */ new Map();
|
|
3833
|
+
for (const key of Object.keys(env)) {
|
|
3834
|
+
const m = /^DB_CONNECTION_STRING_SIB_(.+)$/.exec(key);
|
|
3835
|
+
if (!m || !env[key]) continue;
|
|
3836
|
+
const name = m[1].toLowerCase();
|
|
3837
|
+
if (matches(name)) {
|
|
3838
|
+
found.set(name, "env");
|
|
3839
|
+
}
|
|
3840
|
+
}
|
|
3841
|
+
const base = options.baseDb ?? context?.db;
|
|
3842
|
+
if (base) {
|
|
3843
|
+
const connectionString = String(base.config?.connectionString ?? "");
|
|
3844
|
+
if (connectionString.startsWith("postgresql")) {
|
|
3845
|
+
const { rows } = await base.raw(
|
|
3846
|
+
"SELECT datname FROM pg_database WHERE datistemplate = false"
|
|
3847
|
+
);
|
|
3848
|
+
for (const { datname } of rows) {
|
|
3849
|
+
if (matches(datname) && !found.has(datname)) {
|
|
3850
|
+
found.set(datname, "server");
|
|
3851
|
+
}
|
|
3852
|
+
}
|
|
3853
|
+
} else {
|
|
3854
|
+
context?.logger?.warn?.(
|
|
3855
|
+
"[Db] discoverSiblings: server scan supported for PostgreSQL only; using env-declared siblings"
|
|
3856
|
+
);
|
|
3857
|
+
}
|
|
3858
|
+
}
|
|
3859
|
+
return [...found].map(([name, origin]) => ({ name, origin })).sort((a, b) => a.name.localeCompare(b.name));
|
|
3860
|
+
}
|
|
3861
|
+
/**
|
|
3862
|
+
* Discover + connect: one handler per active sibling (cached, pooled —
|
|
3863
|
+
* see initSibling). Pass `includeMain: true` to get `[context.db, ...]`,
|
|
3864
|
+
* which is the usual shape for "apply this DDL everywhere" loops:
|
|
3865
|
+
*
|
|
3866
|
+
* const dbs = await Db.initAllSiblings(context, { prefix: "src_", includeMain: true });
|
|
3867
|
+
* await ensureSchemaEverywhere(dbs, spec, { logger: context.logger });
|
|
3868
|
+
*
|
|
3869
|
+
* @param {object} context
|
|
3870
|
+
* @param {{ prefix?: string, match?: RegExp, includeMain?: boolean, baseDb?: object, env?: object }} options
|
|
3871
|
+
* @returns {Promise<Function[]>} connected handlers
|
|
3872
|
+
*/
|
|
3873
|
+
static async initAllSiblings(context, options = {}) {
|
|
3874
|
+
const discovered = await _Db.discoverSiblings(context, options);
|
|
3875
|
+
const handlers = [];
|
|
3876
|
+
for (const { name } of discovered) {
|
|
3877
|
+
handlers.push(await _Db.initSibling(context, name, options));
|
|
3878
|
+
}
|
|
3879
|
+
return options.includeMain && context.db ? [context.db, ...handlers] : handlers;
|
|
3880
|
+
}
|
|
3549
3881
|
constructor(config2) {
|
|
3550
3882
|
if (!config2 || !config2.connectionString) {
|
|
3551
3883
|
throw new ParamError("Db: connectionString is required");
|
|
@@ -3806,6 +4138,54 @@ var Db = class {
|
|
|
3806
4138
|
function capitalizeFirstLetter(str) {
|
|
3807
4139
|
return str.charAt(0).toUpperCase() + str.slice(1);
|
|
3808
4140
|
}
|
|
4141
|
+
async function resolveMainHandler(context, options = {}) {
|
|
4142
|
+
if (options.baseDb) {
|
|
4143
|
+
return options.baseDb;
|
|
4144
|
+
}
|
|
4145
|
+
if (!context.db) {
|
|
4146
|
+
context.db = await Db.init(context);
|
|
4147
|
+
}
|
|
4148
|
+
return context.db;
|
|
4149
|
+
}
|
|
4150
|
+
async function databaseExistsOnServer(baseHandle, baseConnectionString, databaseName) {
|
|
4151
|
+
if (!String(baseConnectionString).startsWith("postgresql")) {
|
|
4152
|
+
return true;
|
|
4153
|
+
}
|
|
4154
|
+
const query = "SELECT 1 FROM pg_database WHERE datname = ?";
|
|
4155
|
+
if (baseHandle) {
|
|
4156
|
+
const { rows } = await baseHandle.raw(query, [databaseName]);
|
|
4157
|
+
return rows.length > 0;
|
|
4158
|
+
}
|
|
4159
|
+
const shortLived = (0, import_knex.default)({
|
|
4160
|
+
client: "pg",
|
|
4161
|
+
connection: { connectionString: baseConnectionString },
|
|
4162
|
+
pool: { min: 0, max: 1 }
|
|
4163
|
+
});
|
|
4164
|
+
try {
|
|
4165
|
+
const { rows } = await shortLived.raw(query, [databaseName]);
|
|
4166
|
+
return rows.length > 0;
|
|
4167
|
+
} finally {
|
|
4168
|
+
await shortLived.destroy();
|
|
4169
|
+
}
|
|
4170
|
+
}
|
|
4171
|
+
function camelizeDbName(name) {
|
|
4172
|
+
return name.split(/[_-]+/).filter(Boolean).map(capitalizeFirstLetter).join("");
|
|
4173
|
+
}
|
|
4174
|
+
function toEnvKey(key) {
|
|
4175
|
+
return key.replace(/([A-Z])/g, "_$1").toUpperCase();
|
|
4176
|
+
}
|
|
4177
|
+
function replaceDatabaseName(connectionString, databaseName) {
|
|
4178
|
+
let url;
|
|
4179
|
+
try {
|
|
4180
|
+
url = new URL(connectionString);
|
|
4181
|
+
} catch {
|
|
4182
|
+
throw new ParamError(
|
|
4183
|
+
`Db: cannot parse connection string to derive a sibling database from it`
|
|
4184
|
+
);
|
|
4185
|
+
}
|
|
4186
|
+
url.pathname = `/${databaseName}`;
|
|
4187
|
+
return url.toString();
|
|
4188
|
+
}
|
|
3809
4189
|
function resolveDbDisplayName(dbName, connectionParam, argsEnv, mergedName) {
|
|
3810
4190
|
if (dbName) {
|
|
3811
4191
|
return dbName;
|
|
@@ -5895,29 +6275,90 @@ function queueToTableNames(queueName) {
|
|
|
5895
6275
|
registryTable: `${queueName}_services_registry`
|
|
5896
6276
|
};
|
|
5897
6277
|
}
|
|
5898
|
-
function
|
|
5899
|
-
|
|
5900
|
-
|
|
5901
|
-
|
|
5902
|
-
|
|
5903
|
-
|
|
5904
|
-
|
|
5905
|
-
|
|
5906
|
-
|
|
5907
|
-
|
|
5908
|
-
|
|
5909
|
-
|
|
5910
|
-
|
|
5911
|
-
|
|
5912
|
-
|
|
5913
|
-
|
|
5914
|
-
|
|
5915
|
-
|
|
5916
|
-
|
|
5917
|
-
|
|
5918
|
-
|
|
5919
|
-
|
|
5920
|
-
|
|
6278
|
+
function tasksTableSpec(tableNameForIndex) {
|
|
6279
|
+
return {
|
|
6280
|
+
columns: {
|
|
6281
|
+
id: (t, db) => t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()")),
|
|
6282
|
+
created_at: (t, db) => t.timestamp("created_at").notNullable().defaultTo(db.fn.now()),
|
|
6283
|
+
started_at: (t) => t.timestamp("started_at"),
|
|
6284
|
+
completed_at: (t) => t.timestamp("completed_at"),
|
|
6285
|
+
/*
|
|
6286
|
+
* Priority: lower number = claimed first (see claimNextRunnableTask: ORDER BY priority ASC).
|
|
6287
|
+
* Suggested range 0–100: 0 = most urgent, 100 = least; default 50 for normal work.
|
|
6288
|
+
*/
|
|
6289
|
+
priority: (t) => t.integer("priority").notNullable().defaultTo(50),
|
|
6290
|
+
schedule: (t) => t.text("schedule"),
|
|
6291
|
+
next_run_at: (t) => t.timestamp("next_run_at").defaultTo(null),
|
|
6292
|
+
past_due: (t) => t.timestamp("past_due").defaultTo(null),
|
|
6293
|
+
name: (t) => t.text("name").notNullable(),
|
|
6294
|
+
opid: (t) => t.text("opid"),
|
|
6295
|
+
params: (t) => t.jsonb("params"),
|
|
6296
|
+
// those are target identifiers, kind of who is going to run a task.
|
|
6297
|
+
service_group: (t) => t.text("service_group"),
|
|
6298
|
+
// harvester, loader, photos, ...
|
|
6299
|
+
instance_number: (t) => t.integer("instance_number"),
|
|
6300
|
+
service_name: (t) => t.text("service_name"),
|
|
6301
|
+
// that's a "<server_name>_<service_group>_<instance_number>"
|
|
6302
|
+
server_name: (t) => t.text("server_name"),
|
|
6303
|
+
// filled by runner when registering, auto.
|
|
6304
|
+
status: (t) => t.text("status").notNullable().defaultTo("idle"),
|
|
6305
|
+
// idle, running, completed, failed, paused
|
|
6306
|
+
status_changed_at: (t) => t.timestamp("status_changed_at").defaultTo(null),
|
|
6307
|
+
progress: (t) => t.text("progress"),
|
|
6308
|
+
success: (t) => t.boolean("success"),
|
|
6309
|
+
results: (t) => t.jsonb("results")
|
|
6310
|
+
},
|
|
6311
|
+
indexes: [
|
|
6312
|
+
{
|
|
6313
|
+
columns: ["service_group", "status", "priority", "created_at"],
|
|
6314
|
+
name: `${tableNameForIndex}_claim_idx`
|
|
6315
|
+
},
|
|
6316
|
+
{ columns: ["service_group", "name"], name: `${tableNameForIndex}_group_name_idx` }
|
|
6317
|
+
]
|
|
6318
|
+
};
|
|
6319
|
+
}
|
|
6320
|
+
function registryTableSpec(registryTable) {
|
|
6321
|
+
return {
|
|
6322
|
+
columns: {
|
|
6323
|
+
id: (t, db) => t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()")),
|
|
6324
|
+
queue_name: (t) => t.text("queue_name").notNullable(),
|
|
6325
|
+
service_group: (t) => t.text("service_group").notNullable(),
|
|
6326
|
+
// harvester, loader, photos, ...
|
|
6327
|
+
instance_number: (t) => t.integer("instance_number").notNullable().defaultTo(1),
|
|
6328
|
+
service_name: (t) => t.text("service_name").notNullable(),
|
|
6329
|
+
// that's a "<server_name>_<service_group>_<instance_number>"
|
|
6330
|
+
server_name: (t) => t.text("server_name").notNullable(),
|
|
6331
|
+
// filled by runner when registering, auto.
|
|
6332
|
+
pid: (t) => t.integer("pid"),
|
|
6333
|
+
metadata: (t) => t.json("metadata"),
|
|
6334
|
+
created_at: (t, db) => t.timestamp("created_at").notNullable().defaultTo(db.fn.now()),
|
|
6335
|
+
last_seen_at: (t, db) => t.timestamp("last_seen_at").notNullable().defaultTo(db.fn.now())
|
|
6336
|
+
},
|
|
6337
|
+
indexes: [
|
|
6338
|
+
{
|
|
6339
|
+
columns: ["queue_name", "service_name"],
|
|
6340
|
+
name: `${registryTable}_queue_name_service_name_uniq`,
|
|
6341
|
+
unique: true
|
|
6342
|
+
},
|
|
6343
|
+
{
|
|
6344
|
+
columns: ["queue_name", "service_group", "last_seen_at"],
|
|
6345
|
+
name: `${registryTable}_queue_group_seen_idx`
|
|
6346
|
+
},
|
|
6347
|
+
{ columns: ["queue_name", "last_seen_at"], name: `${registryTable}_queue_seen_idx` }
|
|
6348
|
+
]
|
|
6349
|
+
// TODO: a reference is needed - task_history entry should reference the registry entry, so that we can easily find all executed tasks for a given service and calculate the average workload or identify if there's a bottleneck. Also may be used for the load balancing.
|
|
6350
|
+
};
|
|
6351
|
+
}
|
|
6352
|
+
function tasksSchemaSpec(queueName = "tasks") {
|
|
6353
|
+
const { tasksTable, historyTable, registryTable } = queueToTableNames(queueName);
|
|
6354
|
+
return {
|
|
6355
|
+
extensions: ["uuid-ossp"],
|
|
6356
|
+
tables: {
|
|
6357
|
+
[tasksTable]: tasksTableSpec(tasksTable),
|
|
6358
|
+
[historyTable]: tasksTableSpec(historyTable),
|
|
6359
|
+
[registryTable]: registryTableSpec(registryTable)
|
|
6360
|
+
}
|
|
6361
|
+
};
|
|
5921
6362
|
}
|
|
5922
6363
|
function taskHistoryInsertFromQueueRow(row, overrides) {
|
|
5923
6364
|
const { id, ...snapshot } = row;
|
|
@@ -5931,60 +6372,38 @@ async function ensureTaskTables(context, options = {}) {
|
|
|
5931
6372
|
const queueName = options.queueName ?? "tasks";
|
|
5932
6373
|
const recreate = options.recreate ?? false;
|
|
5933
6374
|
const dryRun = options.dryRun ?? false;
|
|
5934
|
-
const
|
|
6375
|
+
const databases = options.databases ?? [getDb(context)];
|
|
5935
6376
|
const log = context.logger ?? console;
|
|
5936
6377
|
const { tasksTable, historyTable, registryTable } = queueToTableNames(queueName);
|
|
5937
|
-
const
|
|
5938
|
-
const
|
|
5939
|
-
|
|
5940
|
-
if (dryRun) {
|
|
5941
|
-
const plan = [];
|
|
6378
|
+
const spec = tasksSchemaSpec(queueName);
|
|
6379
|
+
for (const db of databases) {
|
|
6380
|
+
const label = db?.config?.name ?? "db";
|
|
5942
6381
|
if (recreate) {
|
|
5943
|
-
|
|
6382
|
+
if (dryRun) {
|
|
6383
|
+
log.info?.(
|
|
6384
|
+
`[tasks-schema] dryRun \u2014 ${label}: DROP TABLE IF EXISTS ${historyTable}, ${tasksTable}, ${registryTable}`
|
|
6385
|
+
);
|
|
6386
|
+
} else {
|
|
6387
|
+
await db.schema.dropTableIfExists(historyTable);
|
|
6388
|
+
await db.schema.dropTableIfExists(tasksTable);
|
|
6389
|
+
await db.schema.dropTableIfExists(registryTable);
|
|
6390
|
+
}
|
|
5944
6391
|
}
|
|
5945
|
-
|
|
5946
|
-
if (
|
|
5947
|
-
|
|
5948
|
-
|
|
5949
|
-
|
|
5950
|
-
|
|
5951
|
-
|
|
5952
|
-
|
|
6392
|
+
const { actions } = await ensureSchema(db, spec, { dryRun, logger: context.logger });
|
|
6393
|
+
if (dryRun) {
|
|
6394
|
+
if (actions.length === 0) {
|
|
6395
|
+
log.info?.(`[tasks-schema] dryRun \u2014 ${label}: queue "${queueName}" already up to date; no DDL`);
|
|
6396
|
+
} else {
|
|
6397
|
+
log.info?.(
|
|
6398
|
+
`[tasks-schema] dryRun \u2014 ${label}: would run ${actions.length} statement(s) for queue "${queueName}":`
|
|
6399
|
+
);
|
|
6400
|
+
for (const s of actions) log.info?.(` - ${s}`);
|
|
6401
|
+
}
|
|
6402
|
+
} else if (actions.length > 0) {
|
|
6403
|
+
log.info?.(
|
|
6404
|
+
`[tasks-schema] ${label}: applied ${actions.length} DDL statement(s) for queue "${queueName}"`
|
|
6405
|
+
);
|
|
5953
6406
|
}
|
|
5954
|
-
return;
|
|
5955
|
-
}
|
|
5956
|
-
if (recreate) {
|
|
5957
|
-
await db.schema.dropTableIfExists(historyTable);
|
|
5958
|
-
await db.schema.dropTableIfExists(tasksTable);
|
|
5959
|
-
await db.schema.dropTableIfExists(registryTable);
|
|
5960
|
-
}
|
|
5961
|
-
if (needsTasks) {
|
|
5962
|
-
await db.raw(`CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`);
|
|
5963
|
-
await db.schema.createTable(tasksTable, (t) => {
|
|
5964
|
-
defineTasksTable(t, db, tasksTable);
|
|
5965
|
-
});
|
|
5966
|
-
}
|
|
5967
|
-
if (needsHistory) {
|
|
5968
|
-
await db.schema.createTable(historyTable, (t) => {
|
|
5969
|
-
defineTasksTable(t, db, historyTable);
|
|
5970
|
-
});
|
|
5971
|
-
}
|
|
5972
|
-
if (needsRegistry) {
|
|
5973
|
-
await db.schema.createTable(registryTable, (t) => {
|
|
5974
|
-
t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()"));
|
|
5975
|
-
t.text("queue_name").notNullable();
|
|
5976
|
-
t.text("service_group").notNullable();
|
|
5977
|
-
t.integer("instance_number").notNullable().defaultTo(1);
|
|
5978
|
-
t.text("service_name").notNullable();
|
|
5979
|
-
t.text("server_name").notNullable();
|
|
5980
|
-
t.integer("pid");
|
|
5981
|
-
t.json("metadata");
|
|
5982
|
-
t.timestamp("created_at").notNullable().defaultTo(db.fn.now());
|
|
5983
|
-
t.timestamp("last_seen_at").notNullable().defaultTo(db.fn.now());
|
|
5984
|
-
t.unique(["queue_name", "service_name"], `${registryTable}_queue_name_service_name_uniq`);
|
|
5985
|
-
t.index(["queue_name", "service_group", "last_seen_at"], `${registryTable}_queue_group_seen_idx`);
|
|
5986
|
-
t.index(["queue_name", "last_seen_at"], `${registryTable}_queue_seen_idx`);
|
|
5987
|
-
});
|
|
5988
6407
|
}
|
|
5989
6408
|
}
|
|
5990
6409
|
async function enqueueTask(context, options) {
|
|
@@ -8144,8 +8563,13 @@ var TasksManager = class _TasksManager {
|
|
|
8144
8563
|
enqueueTask,
|
|
8145
8564
|
ensureDeployKeyOnRemote,
|
|
8146
8565
|
ensureEnvOnRemote,
|
|
8566
|
+
ensureExtension,
|
|
8567
|
+
ensureIndex,
|
|
8147
8568
|
ensureRemoteRepo,
|
|
8148
8569
|
ensureRepoDependencies,
|
|
8570
|
+
ensureSchema,
|
|
8571
|
+
ensureSchemaEverywhere,
|
|
8572
|
+
ensureTable,
|
|
8149
8573
|
ensureTaskTables,
|
|
8150
8574
|
flushTaskIpcLogs,
|
|
8151
8575
|
getArgsInstance,
|
|
@@ -8184,6 +8608,7 @@ var TasksManager = class _TasksManager {
|
|
|
8184
8608
|
releaseDir,
|
|
8185
8609
|
releaseStamp,
|
|
8186
8610
|
reloadPm2,
|
|
8611
|
+
replaceDatabaseName,
|
|
8187
8612
|
resolveAsterisks,
|
|
8188
8613
|
resolveIpcFileLogsDir,
|
|
8189
8614
|
resolveNextVersion,
|