@starci/hfs 4.11.0 → 4.11.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -0
- package/README.md +40 -1
- package/db/drift.mjs +8 -6
- package/db/image.mjs +9 -4
- package/db/migrate.mjs +3 -3
- package/package.json +2 -1
- package/runtime/knowledge/hfs/canon-pins.yaml +13 -8
- package/runtime/knowledge/hfs/rules.yaml +29 -12
- package/runtime/knowledge/hfs/slots.yaml +48 -37
- package/runtime/knowledge/patterns/be/event-bus.yaml +12 -10
- package/runtime/knowledge/patterns/be/queues.yaml +2 -3
- package/runtime/knowledge/patterns/be/reactors.yaml +1 -1
- package/runtime/modules/kernel/failure-codes.yaml +17 -17
- package/runtime/modules/schemas/application-stacks.schema.yaml +4 -2
- package/runtime/scripts/gates/starcistacks.mjs +6 -2
- package/runtime/scripts/guards/push-scan.mjs +1 -1
- package/runtime/scripts/hfs/ci-declaration.mjs +3 -2
- package/runtime/scripts/hfs/db-baseline-verb.mjs +50 -10
- package/runtime/scripts/hfs/db-errors.mjs +2 -0
- package/runtime/scripts/hfs/db-plan-verb.mjs +17 -5
- package/runtime/scripts/hfs/db-squash.mjs +514 -0
- package/runtime/scripts/hfs/declaration-shape.mjs +3 -15
- package/runtime/scripts/hfs/example-secrets.mjs +4 -3
- package/runtime/scripts/hfs/rule-params-shape.mjs +11 -9
- package/runtime/scripts/hfs/rules/a15-findings.mjs +1 -1
- package/runtime/scripts/hfs/rules/database-config.mjs +1 -1
- package/runtime/scripts/hfs/rules/database-constants.mjs +16 -1
- package/runtime/scripts/hfs/rules/database-queue-sources.mjs +2 -110
- package/runtime/scripts/hfs/rules/database-queues.mjs +53 -239
- package/runtime/scripts/hfs/rules/database-sql.mjs +2 -0
- package/runtime/scripts/hfs/rules/database-transaction.mjs +45 -0
- package/runtime/scripts/hfs/rules/deploy-scan.mjs +1 -1
- package/runtime/scripts/hfs/rules/edition.mjs +1 -1
- package/runtime/scripts/hfs/rules/env-parity.mjs +1 -1
- package/runtime/scripts/hfs/rules/lite-secrets.mjs +3 -3
- package/runtime/scripts/hfs/rules/script-hygiene.mjs +1 -1
- package/runtime/scripts/hfs/rules/secret-custody-only.mjs +2 -2
- package/runtime/scripts/hfs/rules/secrets.mjs +1 -1
- package/runtime/scripts/hfs/rules/stack-tooling.mjs +1 -1
- package/runtime/scripts/hfs/rules/stacks-deploy.mjs +2 -1
- package/runtime/scripts/hfs/rules/stacks-lite-backing.mjs +2 -2
- package/runtime/scripts/hfs/rules/stacks-runtime.mjs +2 -2
- package/runtime/scripts/hfs/rules/stacks-swarm.mjs +1 -1
- package/runtime/scripts/hfs/rules/stacks.mjs +3 -3
- package/runtime/scripts/hfs/sealed-members.mjs +10 -6
- package/runtime/scripts/hfs/secret-files.mjs +16 -0
- package/runtime/scripts/hfs/secret-init.mjs +120 -21
- package/runtime/scripts/hfs/secret-output.mjs +2 -1
- package/runtime/scripts/hfs/secret-prompt.mjs +128 -0
- package/runtime/scripts/hfs/secret-sops.mjs +93 -9
- package/runtime/scripts/hfs/secret-standard.mjs +315 -0
- package/runtime/scripts/hfs/secret-store.mjs +23 -9
- package/runtime/scripts/hfs/secret.mjs +110 -46
- package/runtime/scripts/hfs/slots.mjs +8 -1
- package/runtime/scripts/hfs/stack-deploy-config.mjs +20 -6
- package/runtime/scripts/hfs/stacks-gate.mjs +9 -2
- package/runtime/scripts/hfs/starcistacks-runtime.mjs +1 -1
- package/runtime/scripts/lib/sealed-member.mjs +36 -0
- package/runtime/scripts/lib/secret-patterns.mjs +1 -1
- package/runtime/scripts/lib/swarm-stack.mjs +2 -2
- package/scaffold/add-inbox-lite.mjs +2 -2
- package/scaffold/app.mjs +2 -2
- package/src/main.mjs +5 -4
- package/stack/build.mjs +1 -1
- package/stack/config.mjs +2 -1
- package/stack/docker-transport.mjs +2 -1
- package/stack/remote.mjs +4 -3
- package/stack/sealed.mjs +9 -8
- package/stack/target-binding.mjs +2 -1
- package/sync/hygiene.mjs +1 -1
- package/sync/index.mjs +33 -10
- package/templates/app/db-bootstrap/db/bootstrap.sql +7 -4
- package/templates/app/db-image/starcistacks/images/db/Dockerfile +18 -0
- package/templates/app/gitignore +2 -1
- package/templates/app/gitignore-lite +3 -1
- package/templates/app/package-scripts/db.json +1 -1
- package/templates/app/skeleton-lite/.starcistacks/application-stacks.yaml +1 -1
- package/templates/app/skeleton-lite/.starcistacks/dev/README.md +1 -1
- package/templates/app/skeleton-lite/.starcistacks/production/README.md +1 -1
- package/templates/app/skeleton-lite/README.md +2 -2
- package/templates/be/inbox/delivery-recorded.event.ts.tpl +7 -5
- package/templates/be/inbox/events.json.tpl +1 -1
- package/templates/be/inbox/platform/event-bus/event-bus.contracts.ts.tpl +58 -11
- package/templates/be/inbox/platform/event-bus/event-bus.decorators.ts.tpl +13 -1
- package/templates/be/inbox/platform/event-bus/event-bus.log-events.ts.tpl +9 -0
- package/templates/be/inbox/platform/event-bus/event-bus.module-definition.ts.tpl +1 -1
- package/templates/be/inbox/platform/event-bus/event-bus.module.ts.tpl +11 -5
- package/templates/be/inbox/platform/event-bus/event-bus.options.ts.tpl +9 -2
- package/templates/be/inbox/platform/event-bus/event-bus.port.ts.tpl +18 -5
- package/templates/be/inbox/platform/event-bus/event-bus.service.spec.ts.tpl +37 -16
- package/templates/be/inbox/platform/event-bus/event-bus.service.ts.tpl +33 -5
- package/templates/be/inbox/platform/event-bus/event-consumer.service.spec.ts.tpl +143 -0
- package/templates/be/inbox/platform/event-bus/event-consumer.service.ts.tpl +119 -0
- package/templates/be/inbox/platform/event-bus/index.ts.tpl +13 -4
- package/templates/be/inbox/platform/inbox/inbox.service.spec.ts.tpl +4 -4
- package/templates/be/inbox/platform/inbox/inbox.service.ts.tpl +4 -4
- package/templates/be/inbox/platform/migration.sql.tpl +97 -34
- package/templates/be/inbox/service.spec.ts.tpl +1 -1
- package/templates/be/inbox/webhook-door.ts.tpl +2 -2
- package/templates/be/patterns/jobs/platform-lite/job-claim.service.spec.ts.tpl +4 -4
- package/templates/be/patterns/jobs/platform-lite/job-claim.service.ts.tpl +5 -5
- package/templates/be/patterns/jobs/platform-lite/job-tick.service.spec.ts.tpl +1 -1
- package/templates/be/patterns/jobs/platform-lite/job-tick.service.ts.tpl +1 -1
- package/templates/be/patterns/queues/platform-lite/queue-outbox.service.spec.ts.tpl +1 -1
- package/templates/be/patterns/queues/platform-lite/queue-outbox.service.ts.tpl +1 -1
- package/templates/be/patterns/queues/platform-lite/queue-relay.service.spec.ts.tpl +2 -2
- package/templates/be/patterns/queues/platform-lite/queue-relay.service.ts.tpl +2 -2
- package/templates/be/skeleton/apps/__app__/src/__app__.env.ts +4 -0
- package/templates/be/skeleton/apps/__app__/src/__app__.options.ts +3 -0
- package/templates/be/skeleton/apps/__app__/src/compose/platform.ts +1 -1
- package/templates/be/skeleton/src/modules/platform/http/fetch-http-client.client.spec.ts +37 -2
- package/templates/be/skeleton/src/modules/platform/http/fetch-http-client.client.ts +36 -3
- package/templates/be/skeleton/src/modules/platform/http/http.config.ts +11 -0
- package/templates/be/skeleton/src/modules/platform/http/http.decorators.ts +7 -0
- package/templates/be/skeleton/src/modules/platform/http/http.module-definition.ts +5 -3
- package/templates/be/skeleton/src/modules/platform/http/http.options.ts +9 -2
- package/templates/be/skeleton/src/modules/platform/http/index.ts +2 -0
- package/templates/be/skeleton-lite/src/modules/platform/database/database.client.spec.ts +96 -0
- package/templates/be/skeleton-lite/src/modules/platform/database/database.client.ts +93 -0
- package/templates/be/skeleton-lite/src/modules/platform/database/index.ts +1 -1
- package/runtime/scripts/hfs/rules/database-queue-outbox.mjs +0 -102
- package/templates/app/db-image/db/Dockerfile +0 -8
- package/templates/be/skeleton-lite/src/modules/platform/database/database.functions.ts +0 -24
|
@@ -36,11 +36,7 @@ const formsOk = (v) => Array.isArray(v) && v.length > 0 && v.every((x) => FORM_N
|
|
|
36
36
|
const doubleOk = (v) => isPlainObject(v) && /^[A-Za-z][A-Za-z0-9]*$/.test(String(v.double)) && formsOk(v.forms);
|
|
37
37
|
const regexOk = (v) => { try { return typeof v === 'string' && v.length > 0 && Boolean(new RegExp(v)); } catch { return false; } };
|
|
38
38
|
const dbExtensionsOk = (value) => Array.isArray(value) && value.every((name) => typeof name === 'string' && name.length > 0) && new Set(value).size === value.length;
|
|
39
|
-
const
|
|
40
|
-
const queueOutboxOk = (value) => isPlainObject(value) && Object.keys(value).sort(byCodeUnit).join() === 'columns,table'
|
|
41
|
-
&& typeof value.table === 'string' && value.table.split('.').length === 2 && value.table.split('.').every((part) => SQL_NAME.test(part))
|
|
42
|
-
&& nonemptyUniqueList(value.columns, (column) => SQL_NAME.test(String(column))) && value.columns.includes('queue');
|
|
43
|
-
const DB_PARAMS = { extensions: dbExtensionsOk, queueOutbox: queueOutboxOk };
|
|
39
|
+
const DB_PARAMS = { extensions: dbExtensionsOk };
|
|
44
40
|
const dbParamsOk = (db) => db === undefined || (isPlainObject(db) && Object.entries(db).every(([key, value]) => Object.hasOwn(DB_PARAMS, key) && DB_PARAMS[key](value)));
|
|
45
41
|
|
|
46
42
|
function beSpecificProblems(be, manifest) {
|
|
@@ -62,8 +58,14 @@ function beSpecificProblems(be, manifest) {
|
|
|
62
58
|
return bad;
|
|
63
59
|
}
|
|
64
60
|
|
|
65
|
-
/**
|
|
66
|
-
|
|
61
|
+
/** A database function name (R334): `schema.name` outside pg_catalog, or, where `bare` admits it, a pg built-in named alone. */
|
|
62
|
+
const dbFunctionNameOk = (name, bare) => typeof name === 'string' && (/^[a-z_][a-z0-9_]*\.[a-z_][a-z0-9_]*$/.test(name) ? !name.startsWith('pg_catalog.') : bare && /^[a-z_][a-z0-9_]*$/.test(name));
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* True for an absent list or a list of {name, reason} database functions, names unique and schema-qualified; `bare` also admits
|
|
66
|
+
* a bare name, which only the canon's list (ruleParams.be.dbFunctions) uses, for the pg built-ins its helpers call.
|
|
67
|
+
*/
|
|
68
|
+
export const dbFunctionsOk = (list, { bare = false } = {}) => list === undefined || (Array.isArray(list) && list.every((entry) => isPlainObject(entry) && Object.keys(entry).sort().join() === 'name,reason' && dbFunctionNameOk(entry.name, bare) && typeof entry.reason === 'string' && entry.reason.trim().length > 0) && new Set(list.map((entry) => entry.name)).size === list.length);
|
|
67
69
|
|
|
68
70
|
/** The shape problems of `m.ruleParams`. */
|
|
69
71
|
export function ruleParamsProblems(m) {
|
|
@@ -74,10 +76,10 @@ export function ruleParamsProblems(m) {
|
|
|
74
76
|
// common holds the parameters both sides share; a side may restate one of them (a valid value) as an override, nothing else.
|
|
75
77
|
// Edition lite: a side's optional `lite` map overrides keys of the same side (never `lite` itself) when the app declares edition lite.
|
|
76
78
|
if (!sideOk(rp.common, []) || !fileLinesOk(rp.common.fileLines) || !blockOk(rp.common.duplicateBlock)) bad.push('ruleParams.common needs fileLines {soft, hardGrowth} and duplicateBlock {lines >= 2, tokens >= 1}; optional functionLines and featureFiles must be positive integers');
|
|
77
|
-
if (!sideOk(rp.be, BE_OWN, ['schemaAuthority', 'lite', 'dbFunctions']) || (rp.be.schemaAuthority !== undefined && !SCHEMA_AUTHORITIES.includes(rp.be.schemaAuthority)) || !dbFunctionsOk(rp.be.dbFunctions) || !liteOk(rp.be.lite, [...BE_OWN, ...SHARED_KEYS, 'schemaAuthority', 'dbFunctions'])) bad.push('ruleParams.be needs infraOwners, eventBus, specDoubles, paramNames, suffixes, bannedSuffixes, contractShape {helper}, unitRoles, logicRoles, thinRoles, patternScenarios, kindPatterns, addKinds and backing (shared parameters live in ruleParams.common, a side may override them); optional: schemaAuthority (' + SCHEMA_AUTHORITIES.join(' | ') + '), dbFunctions ([{name, reason}]) and lite (overrides of the same keys)'); else bad.push(...unitRolesProblems(rp.be), ...roleListProblems(rp.be));
|
|
79
|
+
if (!sideOk(rp.be, BE_OWN, ['schemaAuthority', 'lite', 'dbFunctions']) || (rp.be.schemaAuthority !== undefined && !SCHEMA_AUTHORITIES.includes(rp.be.schemaAuthority)) || !dbFunctionsOk(rp.be.dbFunctions, { bare: true }) || !liteOk(rp.be.lite, [...BE_OWN, ...SHARED_KEYS, 'schemaAuthority', 'dbFunctions'])) bad.push('ruleParams.be needs infraOwners, eventBus, specDoubles, paramNames, suffixes, bannedSuffixes, contractShape {helper}, unitRoles, logicRoles, thinRoles, patternScenarios, kindPatterns, addKinds and backing (shared parameters live in ruleParams.common, a side may override them); optional: schemaAuthority (' + SCHEMA_AUTHORITIES.join(' | ') + '), dbFunctions ([{name, reason}]: a pg built-in bare, every other function schema.name) and lite (overrides of the same keys)'); else bad.push(...unitRolesProblems(rp.be), ...roleListProblems(rp.be));
|
|
78
80
|
bad.push(...beSpecificProblems(rp.be, m));
|
|
79
81
|
if (rp.fe !== undefined && (!sideOk(rp.fe, [], ['lite']) || !liteOk(rp.fe.lite, SHARED_KEYS))) bad.push('ruleParams.fe may only restate the shared parameters of ruleParams.common, and carry a lite override of them');
|
|
80
|
-
if (!dbParamsOk(rp.db)) bad.push('ruleParams.db may contain only extensions (a list of unique non-empty extension names)
|
|
82
|
+
if (!dbParamsOk(rp.db)) bad.push('ruleParams.db may contain only extensions (a list of unique non-empty extension names)');
|
|
81
83
|
}
|
|
82
84
|
return bad;
|
|
83
85
|
}
|
|
@@ -48,7 +48,7 @@ export const A15_CODES = [
|
|
|
48
48
|
'HFS_WORKFLOW_CONTEXT',
|
|
49
49
|
'CI_STANDARD',
|
|
50
50
|
'CI_SECRET_SEALED', 'APP_SECRET_IN_CI', 'CI_DECLARATION_DRIFT',
|
|
51
|
-
'BACKUP_PROOF_MISSING', 'BACKUP_UNDECLARED', 'BE_APP_BOOT_PROOF_MISSING', 'BOOT_PROOF_TIMEOUT', 'BE_APP_ENV_REQUIRED', 'BE_APP_MODULE_LIST_INLINE', 'BE_LITE_PROOF_LANE_MISSING', 'DB_ARCHIVE_MUTATED', 'DB_BASELINE_SHAPE', '
|
|
51
|
+
'BACKUP_PROOF_MISSING', 'BACKUP_UNDECLARED', 'BE_APP_BOOT_PROOF_MISSING', 'BOOT_PROOF_TIMEOUT', 'BE_APP_ENV_REQUIRED', 'BE_APP_MODULE_LIST_INLINE', 'BE_LITE_PROOF_LANE_MISSING', 'DB_ARCHIVE_MUTATED', 'DB_BASELINE_SHAPE', 'DB_EXTENSION_ALLOWED', 'DB_MIGRATION_DESTRUCTIVE', 'DB_PGMQ_DECLARED', 'DB_VAULT_WRITE', 'ENV_DUPLICATE', 'ENV_EXAMPLE_DRIFT', 'ENV_PUBLIC_SEALED', 'ENV_REQUIRED_UNSET', 'ENV_SEALED_UNREAD', 'ENV_VALUE_SHAPE', 'FE_STACK_CORE_MISSING', 'FE_STACK_UNLISTED', 'HFS_AGE_KEY_CUSTODY', 'HFS_APP_LEXICON', 'HFS_APP_WORKFLOW_TRIGGER', 'HFS_ASSET_KIND', 'HFS_ASSET_OVERSIZE', 'HFS_AUDIT_STEP_MISSING', 'HFS_CANON_COPY_DRIFT', 'HFS_CI_TOOL_INSTALL', 'HFS_COVERAGE_BELOW', 'HFS_DB_DEPLOY_UNGUARDED', 'HFS_DEPENDABOT_MISSING', 'HFS_DEPLOY_COMPOSE_ON_VPS', 'HFS_DEPLOY_NO_LOGIC', 'HFS_DEPLOY_SCHEDULER', 'HFS_DEPLOY_WORKFLOW', 'HFS_DOC_COMMAND_UNKNOWN', 'HFS_DOC_ENV_UNKNOWN', 'HFS_DOC_PATH_MISSING', 'HFS_DOC_RULE_UNKNOWN', 'HFS_FEATURE_FILE_BUDGET', 'HFS_HOST_CRON', 'HFS_RELEASE_TAG_MISMATCH', 'HFS_SBOM_MISSING', 'HFS_SCHEDULE_UNDECLARED', 'HFS_SCRIPT_PLAINTEXT_LOAD', 'HFS_SCRIPT_XTRACE', 'HFS_SECRET_OUTSIDE_ENC', 'HFS_SECRET_PLAINTEXT_FILE', 'HFS_SEED_PII', 'HFS_UNUSED_SYMBOL', 'HFS_WORKFLOW_PIN', 'STACKS_DEPLOY_CONFIG_INVALID', 'STACKS_DNS_UNDECLARED', 'STACKS_HEALTHCHECK_MISSING', 'STACKS_HOST_UNROUTED', 'STACKS_INGRESS_UNDECLARED', 'STACKS_INGRESS_UNKNOWN', 'STACKS_LITE_REDIS_PERSISTENCE', 'STACKS_LITE_SELFHOSTED_DB', 'STACKS_MODEL_UNAVAILABLE', 'STACKS_PLACEMENT_MISSING', 'STACKS_PORT_PUBLISHED', 'STACKS_REALTIME_SCALED', 'STACKS_RELEASE_UNDECLARED', 'STACKS_SECOND_INGRESS', 'STACKS_SECRET_ENV_LITERAL', 'STACKS_SECRET_NOT_EXTERNAL', 'STACKS_SECRET_UNVERSIONED', 'STACKS_SIDECAR_DB_UNSUPPORTED', 'STACKS_SIDECAR_VOLUME_UNDECLARED', 'STACKS_TLS_UNDECLARED', 'STACKS_UPDATE_POLICY_MISSING', 'STACK_TOOLING_REIMPLEMENTED',
|
|
52
52
|
// codes-end
|
|
53
53
|
];
|
|
54
54
|
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
import { databaseExecutableTokens, databaseSqlStatements, databaseSqlTokens, destructiveMigrationFindings } from './database-migrations.mjs';
|
|
8
8
|
import { byCodeUnit } from '../../lib/list.mjs';
|
|
9
9
|
|
|
10
|
-
const DEFAULT_EXTENSIONS = ['pgcrypto', 'pg_trgm', 'vector', 'uuid-ossp', 'unaccent', 'btree_gist'];
|
|
10
|
+
const DEFAULT_EXTENSIONS = ['pgcrypto', 'pg_trgm', 'vector', 'uuid-ossp', 'unaccent', 'btree_gist', 'pgmq'];
|
|
11
11
|
const word = (token, value) => token?.t === 'word' && token.v === value;
|
|
12
12
|
const identifier = (token) => token && ['word', 'ident'].includes(token.t) ? token.v : null;
|
|
13
13
|
|
|
@@ -9,7 +9,6 @@ export const LITE_DB_MIGRATION = 'LITE_DB_MIGRATION';
|
|
|
9
9
|
export const DB_BASELINE_SHAPE = 'DB_BASELINE_SHAPE';
|
|
10
10
|
export const DB_ARCHIVE_MUTATED = 'DB_ARCHIVE_MUTATED';
|
|
11
11
|
export const DB_PGMQ_DECLARED = 'DB_PGMQ_DECLARED';
|
|
12
|
-
export const DB_CRON_OWNED = 'DB_CRON_OWNED';
|
|
13
12
|
export const DB_EXTENSION_ALLOWED = 'DB_EXTENSION_ALLOWED';
|
|
14
13
|
export const DB_MIGRATION_DESTRUCTIVE = 'DB_MIGRATION_DESTRUCTIVE';
|
|
15
14
|
export const HFS_SEED_PII = 'HFS_SEED_PII';
|
|
@@ -22,3 +21,19 @@ export const MIGRATIONS_ARCHIVE_DIR = `${MIGRATIONS_DIR}/archive`;
|
|
|
22
21
|
/** True for a migration `starci app db migrate` applies: a path under db/migrations/ outside the archive. */
|
|
23
22
|
export const isActiveMigration = (file) => file.startsWith(`${MIGRATIONS_DIR}/`) && !file.startsWith(`${MIGRATIONS_ARCHIVE_DIR}/`);
|
|
24
23
|
export const SEED_FILE = `${DB_DIR}/seed.sql`;
|
|
24
|
+
/** The managed pre-migration bootstrap (slot app.db.bootstrap): `starci app sync` renders it, and `db baseline --squash` owns one marked section. */
|
|
25
|
+
export const BOOTSTRAP_FILE = `${DB_DIR}/bootstrap.sql`;
|
|
26
|
+
export const BOOTSTRAP_SQUASH_BEGIN = '-- >>> starci app db baseline --squash: roles, role settings and grants a per-database dump loses (regenerate with the verb; do not edit) >>>';
|
|
27
|
+
export const BOOTSTRAP_SQUASH_END = '-- <<< starci app db baseline --squash <<<';
|
|
28
|
+
/**
|
|
29
|
+
* db/bootstrap.sql split into the part sync renders (`head`) and the squash section (`section`, from its begin line through its end line, or
|
|
30
|
+
* null). A begin marker without its end marker is no section, so the whole text stays the rendered part and differs from the render.
|
|
31
|
+
*/
|
|
32
|
+
export function bootstrapSection(text) {
|
|
33
|
+
const start = text.indexOf(BOOTSTRAP_SQUASH_BEGIN);
|
|
34
|
+
const end = start < 0 ? -1 : text.indexOf(BOOTSTRAP_SQUASH_END, start);
|
|
35
|
+
if (start < 0 || end < 0 || (start > 0 && text[start - 1] !== '\n')) return { head: text, section: null };
|
|
36
|
+
const newline = text.indexOf('\n', end);
|
|
37
|
+
const stop = newline < 0 ? text.length : newline + 1;
|
|
38
|
+
return { head: text.slice(0, start) + text.slice(stop), section: text.slice(start, stop) };
|
|
39
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// TypeScript queue constants
|
|
1
|
+
// TypeScript queue constants and event-bus topic queue constants for migration parity (R258).
|
|
2
2
|
import pg from 'libpg-query';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { fileURLToPath } from 'node:url';
|
|
@@ -24,60 +24,20 @@ export function sourceValue(ts, node) {
|
|
|
24
24
|
return node;
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
function sourceImports(ts, source) {
|
|
28
|
-
const imports = new Map();
|
|
29
|
-
for (const statement of source.statements) {
|
|
30
|
-
if (!ts.isImportDeclaration(statement) || !ts.isStringLiteral(statement.moduleSpecifier)) continue;
|
|
31
|
-
const module = statement.moduleSpecifier.text;
|
|
32
|
-
const clause = statement.importClause;
|
|
33
|
-
if (clause?.name) imports.set(clause.name.text, { name: 'default', module });
|
|
34
|
-
const named = clause?.namedBindings;
|
|
35
|
-
if (named && ts.isNamespaceImport(named)) imports.set(named.name.text, { name: '*', module });
|
|
36
|
-
for (const item of named?.elements ?? []) imports.set(item.name.text, { name: (item.propertyName ?? item.name).text, module });
|
|
37
|
-
}
|
|
38
|
-
return imports;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
function sourceExports(ts, source) {
|
|
42
|
-
const exports = new Map();
|
|
43
|
-
const stars = [];
|
|
44
|
-
for (const statement of source.statements) {
|
|
45
|
-
if (!ts.isExportDeclaration(statement)) continue;
|
|
46
|
-
const module = statement.moduleSpecifier?.text ?? null;
|
|
47
|
-
if (!statement.exportClause && module) stars.push(module);
|
|
48
|
-
for (const item of statement.exportClause?.elements ?? []) exports.set(item.name.text, { name: (item.propertyName ?? item.name).text, module });
|
|
49
|
-
}
|
|
50
|
-
return { exports, stars };
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
function sourceBindings(ts, source) {
|
|
54
|
-
const bindings = new Map();
|
|
55
|
-
const visit = (node) => {
|
|
56
|
-
if ((ts.isVariableDeclaration(node) || ts.isFunctionDeclaration(node)) && node.name && ts.isIdentifier(node.name)) bindings.set(node.name.text, node.initializer ?? node.body);
|
|
57
|
-
if (ts.isExportAssignment(node)) bindings.set('default', node.expression);
|
|
58
|
-
ts.forEachChild(node, visit);
|
|
59
|
-
};
|
|
60
|
-
visit(source);
|
|
61
|
-
return { source, bindings, imports: sourceImports(ts, source), ...sourceExports(ts, source) };
|
|
62
|
-
}
|
|
63
|
-
|
|
64
27
|
export function queueSourceFacts(state, files, inspectSql) {
|
|
65
28
|
const ts = loadTypescript(state.repoRoot, HERE);
|
|
66
|
-
const sources = new Map();
|
|
67
29
|
if (!ts) {
|
|
68
30
|
if (files.some((file) => QUEUE_FILE.test(file) || TOPIC_FILE.test(file))) queueProblem(state, DB_PGMQ_DECLARED, 'hfs.json', 'queue declarations require the TypeScript parser');
|
|
69
|
-
return
|
|
31
|
+
return;
|
|
70
32
|
}
|
|
71
33
|
for (const file of files.filter((item) => SOURCE_FILE.test(item) && !TEST_FILE.test(item))) {
|
|
72
34
|
const text = readText(state.repoRoot, file);
|
|
73
35
|
if (text === null) continue;
|
|
74
36
|
const source = ts.createSourceFile(file, text, ts.ScriptTarget.Latest, true);
|
|
75
|
-
sources.set(file, sourceBindings(ts, source));
|
|
76
37
|
if (QUEUE_FILE.test(file)) addLiteralConstants(state, state.constants, ts, source, file, /(?:^QUEUE$|_QUEUE(?:_NAME)?$|^QUEUE_)/, 'queue names');
|
|
77
38
|
if (TOPIC_FILE.test(file)) addLiteralConstants(state, state.topics, ts, source, file, /_TOPIC_QUEUE$/, 'event-bus topic queue names');
|
|
78
39
|
addSourceSql(state, ts, source, file, inspectSql);
|
|
79
40
|
}
|
|
80
|
-
return { ts, sources };
|
|
81
41
|
}
|
|
82
42
|
|
|
83
43
|
function addLiteralConstants(state, into, ts, source, file, name, what) {
|
|
@@ -108,71 +68,3 @@ function addSourceSql(state, ts, source, file, inspectSql) {
|
|
|
108
68
|
};
|
|
109
69
|
visit(source);
|
|
110
70
|
}
|
|
111
|
-
|
|
112
|
-
function importedFile(files, from, specifier) {
|
|
113
|
-
const prefix = from.startsWith('be/') ? 'be/' : '';
|
|
114
|
-
let target = null;
|
|
115
|
-
if (specifier.startsWith('@modules/')) target = `${prefix}src/modules/${specifier.slice(9)}`;
|
|
116
|
-
else if (specifier.startsWith('.')) target = path.posix.normalize(path.posix.join(path.posix.dirname(from), specifier));
|
|
117
|
-
if (target === null) return null;
|
|
118
|
-
return [target, target.replace(/\.js$/, '.ts'), `${target}.ts`, `${target}/index.ts`].find((file) => files.has(file)) ?? null;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
function schedulerBinding(ts, sources, file, name, seen) {
|
|
122
|
-
const key = `binding:${file}:${name}`;
|
|
123
|
-
if (seen.has(key)) return [];
|
|
124
|
-
seen.add(key);
|
|
125
|
-
const info = sources.get(file);
|
|
126
|
-
const local = info?.bindings.get(name);
|
|
127
|
-
if (local) return schedulerValues(ts, sources, file, local, seen);
|
|
128
|
-
const imported = info?.imports.get(name) ?? info?.exports.get(name);
|
|
129
|
-
if (imported) {
|
|
130
|
-
const target = imported.module ? importedFile(sources, file, imported.module) : file;
|
|
131
|
-
return target ? schedulerBinding(ts, sources, target, imported.name, seen) : [];
|
|
132
|
-
}
|
|
133
|
-
const values = [];
|
|
134
|
-
for (const module of info?.stars ?? []) {
|
|
135
|
-
const target = importedFile(sources, file, module);
|
|
136
|
-
if (target) values.push(...schedulerBinding(ts, sources, target, name, seen));
|
|
137
|
-
}
|
|
138
|
-
return values;
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
function schedulerMember(ts, sources, file, node, seen) {
|
|
142
|
-
if (!ts.isPropertyAccessExpression(node) || !ts.isIdentifier(node.expression)) return null;
|
|
143
|
-
const imported = sources.get(file)?.imports.get(node.expression.text);
|
|
144
|
-
if (imported?.name !== '*') return null;
|
|
145
|
-
const target = importedFile(sources, file, imported.module);
|
|
146
|
-
return target ? schedulerBinding(ts, sources, target, node.name.text, seen) : [];
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
function schedulerValues(ts, sources, file, node, seen = new Set()) {
|
|
150
|
-
node = sourceValue(ts, node);
|
|
151
|
-
if (!node) return [];
|
|
152
|
-
const key = `${file}:${node.pos}`;
|
|
153
|
-
if (seen.has(key)) return [];
|
|
154
|
-
seen.add(key);
|
|
155
|
-
if (ts.isIdentifier(node)) return schedulerBinding(ts, sources, file, node.text, seen);
|
|
156
|
-
const member = schedulerMember(ts, sources, file, node, seen);
|
|
157
|
-
if (member) return member;
|
|
158
|
-
const values = [];
|
|
159
|
-
if (ts.isStringLiteralLike(node)) values.push(node.text);
|
|
160
|
-
ts.forEachChild(node, (child) => { values.push(...schedulerValues(ts, sources, file, child, seen)); });
|
|
161
|
-
return values;
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
export function queueSchedulers({ ts, sources }) {
|
|
165
|
-
const values = new Map();
|
|
166
|
-
if (!ts) return values;
|
|
167
|
-
for (const [file, info] of sources) {
|
|
168
|
-
if (!/^(?:be\/)?apps\//.test(file)) continue;
|
|
169
|
-
const visit = (node) => {
|
|
170
|
-
if ((ts.isPropertyAssignment(node) || ts.isShorthandPropertyAssignment(node)) && node.name.getText(info.source).replaceAll('"', '').replaceAll("'", '') === 'schedulers') {
|
|
171
|
-
for (const value of schedulerValues(ts, sources, file, node.initializer ?? node.name)) values.set(value, file);
|
|
172
|
-
}
|
|
173
|
-
ts.forEachChild(node, visit);
|
|
174
|
-
};
|
|
175
|
-
visit(info.source);
|
|
176
|
-
}
|
|
177
|
-
return values;
|
|
178
|
-
}
|
|
@@ -1,25 +1,22 @@
|
|
|
1
|
-
// R258:
|
|
2
|
-
// every pgmq queue they leave is an event-bus topic queue named by a literal *_TOPIC_QUEUE constant under
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// migration may still hold; the live job never enqueues that way.
|
|
1
|
+
// R258: pgmq is the event bus (ruling:lite-pgmq-bus-bullmq-jobs) and jobs run on BullMQ, so the active migrations leave no pgmq queue
|
|
2
|
+
// for a queue constant, and every pgmq queue they leave is an event-bus topic queue named by a literal *_TOPIC_QUEUE constant under
|
|
3
|
+
// platform/event-bus, beside its dead-letter queue `<queue>__dead`. A topic binding names a declared queue and a literal pattern,
|
|
4
|
+
// and pgmq grants reach only the q_ and a_ tables of declared queues, for the application role alone.
|
|
6
5
|
// Initialize PostgreSQL's WASM parser at module load; the exported finder itself is synchronous.
|
|
7
6
|
// SQL and TypeScript ASTs keep comments, quoted identifiers and inert string literals out of the inventory.
|
|
8
7
|
import pg from 'libpg-query';
|
|
9
8
|
import { sqlStringParts as strings, walkSqlAst as walk, sqlParser, sqlCallArgument as argument, sqlStringLiteral as sqlLiteral } from '../../lib/sql-ast.mjs';
|
|
10
|
-
import { queueSourceFacts,
|
|
9
|
+
import { queueSourceFacts, queueProblem as problem } from './database-queue-sources.mjs';
|
|
11
10
|
import { readJson, readText } from './read.mjs';
|
|
12
|
-
import {
|
|
13
|
-
import { loadSlotManifest } from '../slots.mjs';
|
|
14
|
-
import { queueOutboxDeclaration, outboxEnqueueQueueIndex } from './database-queue-outbox.mjs';
|
|
11
|
+
import { DB_PGMQ_DECLARED, isActiveMigration } from './database-constants.mjs';
|
|
15
12
|
|
|
16
13
|
await pg.loadModule();
|
|
17
14
|
|
|
18
|
-
|
|
15
|
+
/** The suffix of a consumer queue's dead-letter queue (templates/be/inbox/platform/event-bus DEAD_QUEUE_SUFFIX). */
|
|
16
|
+
const DEAD_SUFFIX = '__dead';
|
|
19
17
|
const TABLE_PRIVILEGES = ['select', 'insert', 'update', 'delete', 'truncate', 'references', 'trigger', 'maintain'];
|
|
20
18
|
const sqlName = (nodes) => strings(nodes).join('.');
|
|
21
19
|
const callName = (call) => sqlName(call.funcname);
|
|
22
|
-
const qualified = (name, schema = 'public') => name.includes('.') ? name : `${schema}.${name}`;
|
|
23
20
|
|
|
24
21
|
const sqlTree = sqlParser(pg);
|
|
25
22
|
const walkExecutable = (tree, visit) => walk(tree, visit, true);
|
|
@@ -36,7 +33,8 @@ function inspectCalls(state, tree, file, migration) {
|
|
|
36
33
|
});
|
|
37
34
|
for (const call of callsOf(tree)) {
|
|
38
35
|
inspectQueueCall(state, call, file, migration);
|
|
39
|
-
|
|
36
|
+
inspectBindingCall(state, call, file, migration);
|
|
37
|
+
recordSchedule(state, call, migration);
|
|
40
38
|
}
|
|
41
39
|
}
|
|
42
40
|
|
|
@@ -54,36 +52,30 @@ function inspectQueueCall(state, call, file, migration) {
|
|
|
54
52
|
}
|
|
55
53
|
}
|
|
56
54
|
|
|
57
|
-
|
|
55
|
+
/** A topic binding routes the events of a name to a consumer group's queue: both literal, in an active migration, the queue declared. */
|
|
56
|
+
function inspectBindingCall(state, call, file, migration) {
|
|
57
|
+
const name = callName(call);
|
|
58
|
+
if (name !== 'pgmq.bind_topic' && name !== 'pgmq.unbind_topic') return;
|
|
59
|
+
const pattern = sqlLiteral(argument(call, 0, ['pattern']));
|
|
60
|
+
const queue = sqlLiteral(argument(call, 1, ['queue_name']));
|
|
61
|
+
if (!migration || pattern === null || queue === null) problem(state, DB_PGMQ_DECLARED, file, `${name} must name a literal pattern and queue in an active migration`);
|
|
62
|
+
else if (name === 'pgmq.bind_topic' && !state.created.has(queue)) problem(state, DB_PGMQ_DECLARED, file, `pgmq.bind_topic binds ${queue}, which no earlier pgmq.create declares`);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The pg_cron jobs an archived chain still schedules, which `starci app db baseline` carries over. pg_cron is not an allowed extension
|
|
67
|
+
* (R259), so no rule judges them here.
|
|
68
|
+
*/
|
|
69
|
+
function recordSchedule(state, call, migration) {
|
|
70
|
+
if (!migration) return;
|
|
71
|
+
const name = callName(call);
|
|
58
72
|
const job = sqlLiteral(argument(call, 0, ['job_name']));
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
if (job !== null) state.scheduledNames.add(job);
|
|
73
|
+
if (job === null) return;
|
|
74
|
+
if (name === 'cron.unschedule') state.activeCron.delete(job);
|
|
75
|
+
if (name !== 'cron.schedule') return;
|
|
63
76
|
const schedule = sqlLiteral(argument(call, 1, ['schedule']));
|
|
64
77
|
const command = sqlLiteral(argument(call, 2, ['command']));
|
|
65
|
-
if (
|
|
66
|
-
state.activeCron.set(job, { name: job, schedule, command });
|
|
67
|
-
state.liveSchedules.set(job, entry);
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
function inspectCronCall(state, call, file, migration) {
|
|
72
|
-
const name = callName(call);
|
|
73
|
-
if (name === 'cron.schedule' || name === 'cron.schedule_in_database') {
|
|
74
|
-
if (!migration || name !== 'cron.schedule') problem(state, DB_CRON_OWNED, file, 'cron jobs must be declared by cron.schedule in an active migration');
|
|
75
|
-
else recordSchedule(state, call, file);
|
|
76
|
-
return;
|
|
77
|
-
}
|
|
78
|
-
if (name !== 'cron.unschedule' || !migration) return;
|
|
79
|
-
const job = sqlLiteral(argument(call, 0, ['job_name']));
|
|
80
|
-
if (job !== null) {
|
|
81
|
-
state.unschedules.push({ name: job, file });
|
|
82
|
-
state.activeCron.delete(job);
|
|
83
|
-
state.scheduledNames.delete(job);
|
|
84
|
-
state.liveSchedules.delete(job);
|
|
85
|
-
}
|
|
86
|
-
else problem(state, DB_CRON_OWNED, file, 'cron.unschedule must identify the migration-owned job by literal name');
|
|
78
|
+
if (schedule !== null && command !== null) state.activeCron.set(job, { name: job, schedule, command });
|
|
87
79
|
}
|
|
88
80
|
|
|
89
81
|
function plpgsqlTree(body, parameters = []) {
|
|
@@ -103,8 +95,7 @@ function dynamicOwnership(state, expression, file, migration) {
|
|
|
103
95
|
const tree = sqlTree(sql.replace(/%[ILs]/g, 'NULL'));
|
|
104
96
|
walkExecutable(tree, (node) => { if (node.GrantStmt || node.AlterDefaultPrivilegesStmt) recordGrant(state, node, file, migration); });
|
|
105
97
|
for (const call of callsOf(tree)) {
|
|
106
|
-
if (
|
|
107
|
-
if (callName(call).startsWith('cron.')) problem(state, DB_CRON_OWNED, file, 'dynamic SQL cannot own a cron declaration; write the call as static migration SQL');
|
|
98
|
+
if (['pgmq.create', 'pgmq.bind_topic'].includes(callName(call))) problem(state, DB_PGMQ_DECLARED, file, 'dynamic SQL cannot own a queue declaration or binding; write the call as static migration SQL');
|
|
108
99
|
}
|
|
109
100
|
}
|
|
110
101
|
}
|
|
@@ -155,121 +146,6 @@ function updateQueuePrivilege(acl, grant, privilege) {
|
|
|
155
146
|
else if (acl.has(privilege)) acl.set(privilege, false);
|
|
156
147
|
}
|
|
157
148
|
|
|
158
|
-
function migrationStatement(state, node, file) {
|
|
159
|
-
if (node.CreateFunctionStmt) {
|
|
160
|
-
const fn = node.CreateFunctionStmt;
|
|
161
|
-
state.functions.set(qualified(sqlName(fn.funcname)), { node: fn, file });
|
|
162
|
-
inspectFunctionDeclarations(state, fn, file);
|
|
163
|
-
}
|
|
164
|
-
if (node.DropStmt?.removeType === 'OBJECT_FUNCTION') {
|
|
165
|
-
for (const item of node.DropStmt.objects ?? []) {
|
|
166
|
-
const name = qualified(sqlName(item.ObjectWithArgs?.objname));
|
|
167
|
-
state.drops.push({ name, file });
|
|
168
|
-
state.functions.delete(name);
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
if (node.DoStmt) inspectDo(state, node.DoStmt, file);
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
function migrationFacts(state, files) {
|
|
175
|
-
for (const file of files.filter((item) => item.endsWith('.sql') && !item.startsWith('db/migrations/archive/')).sort((left, right) => left.localeCompare(right))) {
|
|
176
|
-
const text = readText(state.repoRoot, file);
|
|
177
|
-
if (text === null) continue;
|
|
178
|
-
const tree = sqlTree(text);
|
|
179
|
-
const active = isActiveMigration(file);
|
|
180
|
-
const scheduleStart = state.schedules.length;
|
|
181
|
-
for (const entry of tree?.stmts ?? []) {
|
|
182
|
-
inspectCalls(state, { stmts: [entry] }, file, active);
|
|
183
|
-
if (active) migrationStatement(state, entry.stmt, file);
|
|
184
|
-
}
|
|
185
|
-
completeScheduleInventory(state, scheduleStart);
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
function completeScheduleInventory(state, start) {
|
|
190
|
-
for (const schedule of state.schedules.slice(start)) {
|
|
191
|
-
for (const [queue, file] of state.created) if (!schedule.created.has(queue)) schedule.created.set(queue, file);
|
|
192
|
-
for (const [name, fn] of state.functions) if (!schedule.functions.has(name)) schedule.functions.set(name, fn);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
function singleCall(tree) {
|
|
197
|
-
if (tree?.stmts?.length !== 1) return null;
|
|
198
|
-
if (tree.stmts[0].stmt?.ReturnStmt) return tree.stmts[0].stmt.ReturnStmt.returnval?.FuncCall ?? null;
|
|
199
|
-
const select = tree.stmts[0].stmt?.SelectStmt;
|
|
200
|
-
if (!select || select.targetList?.length !== 1 || !/^SETOP_NONE$/.test(select.op) || Object.keys(select).some((key) => !['targetList', 'op', 'limitOption'].includes(key))) return null;
|
|
201
|
-
return select.targetList[0].ResTarget?.val?.FuncCall ?? null;
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
/** The historical pgmq.send of an existing queue; only a job's original body may hold it. */
|
|
205
|
-
function enqueue(state, call) {
|
|
206
|
-
if (callName(call) !== 'pgmq.send') return null;
|
|
207
|
-
const queue = sqlLiteral(argument(call, 0, ['queue_name']));
|
|
208
|
-
const safe = pureArguments(call);
|
|
209
|
-
return queue !== null && state.created.has(queue) && safe ? queue : null;
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
function pureCall(call) {
|
|
213
|
-
return uuidConstructor(call) || PURE_CALLS.has(callName(call).replace(/^pg_catalog\./, ''));
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
function pureArguments(call) {
|
|
217
|
-
return pureExpressions(call.args);
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
function pureExpressions(nodes) {
|
|
221
|
-
let safe = callsOf(nodes).every(pureCall);
|
|
222
|
-
walk(nodes, (node) => {
|
|
223
|
-
if (['SubLink', 'SelectStmt', 'InsertStmt', 'UpdateStmt', 'DeleteStmt', 'MergeStmt'].some((key) => node[key])) safe = false;
|
|
224
|
-
});
|
|
225
|
-
return safe;
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
/** The queue outbox table and the columns an enqueue may write, from ruleParams.db.queueOutbox; null refuses every outbox enqueue. */
|
|
229
|
-
function queueOutbox(manifest) {
|
|
230
|
-
const declared = manifest?.ruleParams?.db?.queueOutbox;
|
|
231
|
-
return declared ? { table: declared.table, columns: new Set(declared.columns) } : null;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
/** The queue constant a call of the app's declared outbox enqueue function enqueues (hfs.json sides.be.queueOutbox), or null. */
|
|
235
|
-
function declaredEnqueue(state, call) {
|
|
236
|
-
const declared = state.declaredOutbox;
|
|
237
|
-
if (!declared || qualified(callName(call)) !== declared.enqueue) return null;
|
|
238
|
-
const index = outboxEnqueueQueueIndex(declared, state.functions.get(declared.enqueue), pureCall);
|
|
239
|
-
const queue = index < 0 ? null : sqlLiteral(argument(call, index, [declared.parameter]));
|
|
240
|
-
return queue !== null && state.constants.has(queue) && pureArguments(call) ? queue : null;
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
/** The queue constant one outbox row enqueues: `insert into <outbox> (queue, ...) values ('<queue constant>', ...)` with pure values. */
|
|
244
|
-
function outboxEnqueue(state, insert) {
|
|
245
|
-
if (!state.outbox || insert.returningClause || insert.returningList || insert.onConflictClause || insert.withClause) return null;
|
|
246
|
-
if (`${insert.relation?.schemaname || 'public'}.${insert.relation?.relname}` !== state.outbox.table) return null;
|
|
247
|
-
const columns = (insert.cols ?? []).map((item) => item.ResTarget?.name);
|
|
248
|
-
if (new Set(columns).size !== columns.length || columns.some((column) => !state.outbox.columns.has(column))) return null;
|
|
249
|
-
const rows = insert.selectStmt?.SelectStmt?.valuesLists ?? [];
|
|
250
|
-
const values = rows.length === 1 ? rows[0].List?.items ?? [] : [];
|
|
251
|
-
const at = columns.indexOf('queue');
|
|
252
|
-
if (at < 0 || values.length !== columns.length) return null;
|
|
253
|
-
const queue = sqlLiteral(values[at]);
|
|
254
|
-
return queue !== null && state.constants.has(queue) && pureExpressions(values) ? queue : null;
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
/** The queue one statement enqueues to (an outbox row of a queue constant; a pgmq.send only when `legacy`), or null. */
|
|
258
|
-
function singleEnqueue(state, tree, legacy) {
|
|
259
|
-
const insert = tree?.stmts?.length === 1 ? tree.stmts[0].stmt?.InsertStmt : undefined;
|
|
260
|
-
if (insert) return outboxEnqueue(state, insert);
|
|
261
|
-
const call = singleCall(tree);
|
|
262
|
-
if (!call) return null;
|
|
263
|
-
return declaredEnqueue(state, call) ?? (legacy ? enqueue(state, call) : null);
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
/** PostgreSQL's catalog-owned scalar UUID constructor adds an envelope identity without doing work in the tick. */
|
|
267
|
-
function uuidConstructor(call) {
|
|
268
|
-
return callName(call) === 'pg_catalog.gen_random_uuid' && (call.args?.length ?? 0) === 0
|
|
269
|
-
&& (call.agg_order?.length ?? 0) === 0 && !call.agg_filter && !call.over && !call.agg_star
|
|
270
|
-
&& !call.agg_distinct && !call.agg_within_group && !call.func_variadic;
|
|
271
|
-
}
|
|
272
|
-
|
|
273
149
|
function functionBody(fn) {
|
|
274
150
|
const option = (name) => (fn.options ?? []).find((item) => item.DefElem?.defname === name)?.DefElem?.arg;
|
|
275
151
|
return { body: strings(option('as')?.List?.items).join(''), language: option('language')?.String?.sval ?? 'sql' };
|
|
@@ -289,107 +165,45 @@ function inspectFunctionDeclarations(state, fn, file) {
|
|
|
289
165
|
}
|
|
290
166
|
}
|
|
291
167
|
|
|
292
|
-
function
|
|
293
|
-
if (
|
|
294
|
-
if (node.
|
|
295
|
-
return [{ stmts: [{ stmt: node }] }];
|
|
168
|
+
function migrationStatement(state, node, file) {
|
|
169
|
+
if (node.CreateFunctionStmt) inspectFunctionDeclarations(state, node.CreateFunctionStmt, file);
|
|
170
|
+
if (node.DoStmt) inspectDo(state, node.DoStmt, file);
|
|
296
171
|
}
|
|
297
172
|
|
|
298
|
-
function
|
|
299
|
-
const
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
for (const
|
|
305
|
-
|
|
306
|
-
if (
|
|
173
|
+
function migrationFacts(state, files) {
|
|
174
|
+
for (const file of files.filter((item) => item.endsWith('.sql') && !item.startsWith('db/migrations/archive/')).sort((left, right) => left.localeCompare(right))) {
|
|
175
|
+
const text = readText(state.repoRoot, file);
|
|
176
|
+
if (text === null) continue;
|
|
177
|
+
const tree = sqlTree(text);
|
|
178
|
+
const active = isActiveMigration(file);
|
|
179
|
+
for (const entry of tree?.stmts ?? []) {
|
|
180
|
+
inspectCalls(state, { stmts: [entry] }, file, active);
|
|
181
|
+
if (active) migrationStatement(state, entry.stmt, file);
|
|
307
182
|
}
|
|
308
|
-
});
|
|
309
|
-
return safe ? trees : null;
|
|
310
|
-
}
|
|
311
|
-
|
|
312
|
-
function tickTrees(fn) {
|
|
313
|
-
const { body, language } = functionBody(fn);
|
|
314
|
-
if (language === 'plpgsql') return plpgsqlTickTrees(fn, body);
|
|
315
|
-
if (language !== 'sql') return null;
|
|
316
|
-
if (fn.sql_body) return standardSqlBody(fn.sql_body);
|
|
317
|
-
return sqlTree(body)?.stmts?.map((entry) => ({ stmts: [entry] })) ?? null;
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
function tickQueues(state, fn, legacy) {
|
|
321
|
-
const trees = tickTrees(fn.node);
|
|
322
|
-
if (!trees) return null;
|
|
323
|
-
const queues = trees.map((tree) => singleEnqueue(state, tree, legacy));
|
|
324
|
-
return queues.length && queues.every(Boolean) ? queues : null;
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
/** What a cron command enqueues; `legacy` judges a job's original body, which may still pgmq.send to a queue that existed then. */
|
|
328
|
-
function scheduleTarget(state, command, legacy) {
|
|
329
|
-
const tree = sqlTree(command);
|
|
330
|
-
const target = singleCall(tree);
|
|
331
|
-
if (!target) return { tick: null, queues: singleEnqueue(state, tree, legacy) };
|
|
332
|
-
if (!callName(target).endsWith('_tick')) return { tick: null, queues: declaredEnqueue(state, target) ?? (legacy ? enqueue(state, target) : null) };
|
|
333
|
-
const tick = qualified(callName(target));
|
|
334
|
-
const fn = state.functions.get(tick);
|
|
335
|
-
const queues = fn && pureArguments(target) && tickQueues(state, fn, legacy);
|
|
336
|
-
return { tick, queues };
|
|
337
|
-
}
|
|
338
|
-
|
|
339
|
-
function tickRemoval(state, tick, name) {
|
|
340
|
-
if (!tick) return;
|
|
341
|
-
for (const drop of state.drops.filter((item) => item.name === tick)) {
|
|
342
|
-
if (!state.unschedules.some((item) => item.name === name && item.file === drop.file)) problem(state, DB_CRON_OWNED, drop.file, 'removing a scheduled tick requires cron.unschedule of its job name in the same migration');
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
function scheduleCheck(state, schedule, schedulers, ticks) {
|
|
347
|
-
const { call, file } = schedule;
|
|
348
|
-
const name = sqlLiteral(argument(call, 0, ['job_name']));
|
|
349
|
-
const command = sqlLiteral(argument(call, 2, ['command']));
|
|
350
|
-
if (name === null || command === null || sqlLiteral(argument(call, 1, ['schedule'])) === null) {
|
|
351
|
-
problem(state, DB_CRON_OWNED, file, 'cron.schedule requires literal job name, cadence and command for migration ownership');
|
|
352
|
-
return;
|
|
353
183
|
}
|
|
354
|
-
if (schedule.active.has(name)) problem(state, DB_CRON_OWNED, file, 'a cron job name is scheduled once while it is active: cron.unschedule it before a later migration schedules it again');
|
|
355
|
-
const historical = { ...state, created: schedule.created, functions: schedule.functions };
|
|
356
|
-
const { tick, queues } = scheduleTarget(historical, command, true);
|
|
357
|
-
if (!queues) problem(state, DB_CRON_OWNED, file, 'a cron command inserts one row of a queue constant into the queue outbox, directly or through one existing *_tick function that only does that; business DML, dynamic SQL and network calls are refused');
|
|
358
|
-
if (state.liveSchedules.get(name) === schedule && !scheduleTarget(state, command, false).queues) problem(state, DB_CRON_OWNED, file, 'the live cron job must insert one outbox row of a queue constant (ruling:R212/case-10): it targets a missing queue or tick, still enqueues with pgmq.send, or its current tick no longer only does that');
|
|
359
|
-
tickOwner(state, tick, name, schedule, ticks);
|
|
360
|
-
const owners = [name, ...(tick ? [tick, tick.split('.').at(-1)] : []), ...[queues ?? []].flat()];
|
|
361
|
-
if (owners.some((owner) => schedulers.has(owner))) problem(state, DB_CRON_OWNED, file, 'the tick is also declared in an application scheduler list; give it one owner');
|
|
362
|
-
tickRemoval(state, tick, name);
|
|
363
184
|
}
|
|
364
185
|
|
|
365
|
-
|
|
366
|
-
function tickOwner(state, tick, name, schedule, ticks) {
|
|
367
|
-
if (!tick) return;
|
|
368
|
-
const owners = ticks.get(tick) ?? [];
|
|
369
|
-
if (owners.some((owner) => schedule.active.has(owner))) problem(state, DB_CRON_OWNED, schedule.file, 'a tick function has more than one cron owner');
|
|
370
|
-
ticks.set(tick, [...owners, name]);
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
function queueState({ repoRoot, files = [], all = files, repo = {}, manifest }) {
|
|
186
|
+
function queueState({ repoRoot, files = [], all = files, repo = {} }) {
|
|
374
187
|
const paths = [...new Set(all)];
|
|
375
188
|
const declaration = readJson(repoRoot, 'hfs.json') ?? repo;
|
|
376
189
|
if (repo.edition !== 'lite' && declaration.edition !== 'lite' && !paths.some((file) => file.startsWith('db/'))) return null;
|
|
377
|
-
const state = { repoRoot, appRole: declaration.db?.appRole ?? 'app_be', findings: [],
|
|
378
|
-
|
|
190
|
+
const state = { repoRoot, appRole: declaration.db?.appRole ?? 'app_be', findings: [], constants: new Map(), topics: new Map(), created: new Map(), activeCron: new Map(), grants: [], queueAcls: new Map() };
|
|
191
|
+
queueSourceFacts(state, paths, inspectCalls);
|
|
379
192
|
migrationFacts(state, paths);
|
|
380
193
|
return state;
|
|
381
194
|
}
|
|
382
195
|
|
|
383
196
|
function validateQueues(state, inScope) {
|
|
384
197
|
for (const [queue, file] of state.created) {
|
|
385
|
-
if (
|
|
198
|
+
if (queue.endsWith(DEAD_SUFFIX) && state.topics.has(queue.slice(0, -DEAD_SUFFIX.length))) continue;
|
|
199
|
+
if (state.constants.has(queue)) problem(state, DB_PGMQ_DECLARED, file, `pgmq queue ${queue} is the queue constant of ${state.constants.get(queue)}, and jobs run on BullMQ (ruling:lite-pgmq-bus-bullmq-jobs): drop it with pgmq.drop_queue in a later migration`);
|
|
386
200
|
else if (!state.topics.has(queue)) problem(state, DB_PGMQ_DECLARED, file, `pgmq queue ${queue} has no event-bus owner: name it in a literal *_TOPIC_QUEUE constant under platform/event-bus, or drop it with pgmq.drop_queue in a later migration`);
|
|
387
201
|
}
|
|
388
|
-
for (const [topic, file] of state.topics)
|
|
202
|
+
for (const [topic, file] of state.topics) {
|
|
203
|
+
if (!state.created.has(topic)) problem(state, DB_PGMQ_DECLARED, file, `event-bus topic queue ${topic} has no pgmq.create in an active migration`);
|
|
204
|
+
else if (!state.created.has(`${topic}${DEAD_SUFFIX}`)) problem(state, DB_PGMQ_DECLARED, file, `event-bus topic queue ${topic} has no dead-letter queue: pgmq.create('${topic}${DEAD_SUFFIX}') in an active migration`);
|
|
205
|
+
}
|
|
389
206
|
for (const { node, file, created, executed } of state.grants) grantCheck({ ...state, created: executed ? created : state.created }, node, file);
|
|
390
|
-
const ticks = new Map();
|
|
391
|
-
const schedulers = queueSchedulers(state.sources);
|
|
392
|
-
for (const schedule of state.schedules) scheduleCheck(state, schedule, schedulers, ticks);
|
|
393
207
|
return state.findings.filter((finding) => inScope(finding.path));
|
|
394
208
|
}
|
|
395
209
|
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import { found } from './read.mjs';
|
|
4
4
|
import { parseSql } from '../sql/pg-parse.mjs';
|
|
5
5
|
import { plpgsqlFindings } from './database-plpgsql.mjs';
|
|
6
|
+
import { transactionControlFindings } from './database-transaction.mjs';
|
|
6
7
|
import { DB_MIGRATION_SHAPE, DB_POLICY_SHAPE, DB_RLS_REQUIRED, DB_STORAGE_POLICY } from './database-constants.mjs';
|
|
7
8
|
const POLICY_ACTIONS = ['select', 'insert', 'update', 'delete', 'all'];
|
|
8
9
|
const WRITE_COMMANDS = new Set(['all', 'insert', 'update', 'delete']);
|
|
@@ -491,6 +492,7 @@ export async function sqlAnalysis({ file, text, exposed, forceRls }) {
|
|
|
491
492
|
}
|
|
492
493
|
const facts = factsOf(statementList(parsed.stmts, lineAt));
|
|
493
494
|
return { facts, findings: [
|
|
495
|
+
...transactionControlFindings(file, text, parsed.stmts, lineAt),
|
|
494
496
|
...rlsFindings(file, facts, exposed, forceRls),
|
|
495
497
|
...await plpgsqlFindings(file, facts, exposed),
|
|
496
498
|
...localPolicyFindings(file, facts),
|