@memberjunction/server 6.1.0-edge.1 → 6.1.0-edge.3

package/src/integration/RSUProgressBridge.ts

@@ -0,0 +1,248 @@
+/**
+ * @fileoverview Bridges RSU pipeline lifecycle events onto the integration progress artifact
+ * stream, so a Runtime Schema Update is observable the same way a sync or a connector build is.
+ *
+ * The gap this closes: `IntegrationRunKind` has always included `'RSU'` and
+ * `RUN_KIND_TO_TOPIC` has always mapped it to an `'RSU'` subscription channel — but nothing in
+ * production ever constructed an emitter with that kind, so the channel carried no traffic and
+ * `IntegrationTailRunEvents` had no RSU runs to tail. The only live signal was polling
+ * `RuntimeSchemaUpdateStatus`, which reports the CURRENT step and nothing else: no history, no
+ * durable record, and nothing at all to read after a mid-run API restart — which the RSU pipeline
+ * does to itself, by design, as one of its own steps.
+ *
+ * Layering: `@memberjunction/schema-engine` must not depend on the progress artifacts package, so
+ * it publishes plain {@link RSUObserverEvent}s and this module — one layer up, where both are
+ * already dependencies — owns the emitter lifecycle.
+ */
+import { IntegrationProgressEmitter } from '@memberjunction/integration-progress-artifacts';
+import type { EmitterOptions } from '@memberjunction/integration-progress-artifacts';
+import { RuntimeSchemaManager } from '@memberjunction/schema-engine';
+import type { RSUObserverEvent } from '@memberjunction/schema-engine';
+import { LogError } from '@memberjunction/core';
+import { BaseSingleton } from '@memberjunction/global';
+
+/** The emitter stage name used for run-level (non-step) events. */
+const RUN_STAGE = 'RSUPipeline';
+
+/**
+ * How often an in-flight step reports that it is still alive.
+ *
+ * RSU's expensive steps — CodeGen, TypeScript compile, npm install — run for minutes with no
+ * intermediate observer event, so without this the stream goes silent for the entire step and a
+ * watcher cannot tell "compiling" from "hung". 30s matches the interval the agent notification
+ * conventions ask for on long phases.
+ */
+const HEARTBEAT_INTERVAL_MS = 30_000;
+
+/**
+ * Owns one emitter per RSU pipeline run and translates observer events into progress events.
+ *
+ * A {@link BaseSingleton} because the state here is inherently process-wide AND has to be readable
+ * from elsewhere. `RuntimeSchemaManager` is itself a singleton and serializes its runs behind the
+ * pipeline lock, so at most one RSU run is ever in flight and one current-emitter slot is sufficient
+ * — but the reason for the singleton is {@link CurrentRunID}: a resolver that has just triggered an
+ * RSU needs to hand its client a run to tail, and it can only do that if it can reach this instance.
+ * Constructing the bridge and keeping the only reference inside the observer closure made that
+ * accessor unreachable from any caller.
+ *
+ * NOT a `BaseEngine`: there is no metadata to load and nothing to `Config()`. The distinction is
+ * "single shared instance" (this) versus "cached metadata with a load lifecycle" (that).
+ */
+export class RSUProgressBridge extends BaseSingleton<RSUProgressBridge> {
+    private emitter: IntegrationProgressEmitter | null = null;
+    private currentRunID: string | null = null;
+    /** Liveness timer for the step currently in flight — see {@link HEARTBEAT_INTERVAL_MS}. */
+    private heartbeat: ReturnType<typeof setInterval> | null = null;
+    private emitterOptions: EmitterOptions = {};
+
+    /** The one bridge for this process. */
+    public static get Instance(): RSUProgressBridge {
+        return super.getInstance<RSUProgressBridge>();
+    }
+
+    /**
+     * Options passed through to each run's emitter. Set once, before the first run.
+     *
+     * The only field that matters in practice is `rootDir` — left unset in production so RSU runs
+     * land in the same `logs/integration-runs/` tree as every other run kind and
+     * `IntegrationTailRunEvents` finds them without configuration. Tests point it at a temp dir.
+     *
+     * A settable property rather than a constructor argument because `BaseSingleton.getInstance`
+     * requires a zero-argument constructor.
+     */
+    public Configure(emitterOptions: EmitterOptions): void {
+        this.emitterOptions = emitterOptions;
+    }
+
+    /**
+     * The run ID of the in-flight RSU run, or null when idle. Lets a caller that triggered an RSU
+     * hand a client something to tail via `IntegrationTailRunEvents` / the `RSU` subscription
+     * channel.
+     */
+    public get CurrentRunID(): string | null {
+        return this.currentRunID;
+    }
+
+    /**
+     * Abandons any in-flight run state and stops its heartbeat, without emitting a terminal event.
+     *
+     * Needed because singleton state outlives a single caller: a test (or a server tearing down)
+     * must be able to return the bridge to idle without the next run inheriting a stale emitter or
+     * leaking an interval that keeps the process alive.
+     */
+    public Reset(): void {
+        this.stopHeartbeat();
+        this.emitter = null;
+        this.currentRunID = null;
+    }
+
+    /** The observer to hand to {@link RuntimeSchemaManager.PipelineObserver}. */
+    public readonly Observe = (event: RSUObserverEvent): void => {
+        switch (event.Kind) {
+            case 'run.start':
+                return this.onRunStart(event);
+            case 'step.start':
+                return this.onStepStart(event);
+            case 'step.end':
+                return this.onStepEnd(event);
+            case 'run.end':
+                return this.onRunEnd(event);
+        }
+    };
+
+    private onRunStart(event: Extract<RSUObserverEvent, { Kind: 'run.start' }>): void {
+        // A retry re-enters RunPipelineBatch, so it legitimately starts a NEW run. If a previous
+        // emitter is still open (a throw escaped before run.end), close it rather than leak it.
+        if (this.emitter) this.closeCurrent('Superseded by a new RSU run');
+
+        const runID = IntegrationProgressEmitter.newRunID('rsu');
+        this.currentRunID = runID;
+        this.emitter = new IntegrationProgressEmitter(
+            {
+                runID,
+                runKind: 'RSU',
+                triggerType: 'Pipeline',
+                startedAt: new Date().toISOString(),
+                context: {
+                    itemCount: event.ItemCount,
+                    descriptions: event.Descriptions,
+                    affectedTables: event.AffectedTables,
+                    stepTotal: event.StepTotal,
+                },
+            },
+            this.emitterOptions,
+        );
+        this.emitter.runStart(
+            `RSU pipeline started — ${event.ItemCount} migration(s), ${event.StepTotal} expected step(s): ` +
+            event.Descriptions.join('; ')
+        );
+    }
+
+    private onStepStart(event: Extract<RSUObserverEvent, { Kind: 'step.start' }>): void {
+        if (!this.emitter) return;
+        this.emitter.stageStart(event.Name, this.positionLabel(event.StepIndex, event.StepTotal));
+        this.startHeartbeat(event.Name);
+    }
+
+    private onStepEnd(event: Extract<RSUObserverEvent, { Kind: 'step.end' }>): void {
+        this.stopHeartbeat();
+        if (!this.emitter) return;
+        // A failed step is a stage error, not a stage completion — the run may still continue (a
+        // per-item migration failure does not abort the batch), so this is deliberately not fatal.
+        if (event.Status === 'failed') {
+            this.emitter.stageError(event.Name, event.Message, { durationMs: event.DurationMs, code: 'RSU_STEP_FAILED' });
+            return;
+        }
+        // Deliberately no `counts` here. Counts roll up into the run aggregate, and the countable
+        // unit of an RSU run is MIGRATIONS, not steps — reporting "processed: 12" for a 12-step
+        // single-migration run would read as 12 migrations. The run-level quartet is emitted once,
+        // in onRunEnd. Step position lives in the stage name + the step.start message.
+        this.emitter.stageComplete(event.Name);
+    }
+
+    private onRunEnd(event: Extract<RSUObserverEvent, { Kind: 'run.end' }>): void {
+        this.stopHeartbeat();
+        const emitter = this.emitter;
+        if (!emitter) return;
+        this.emitter = null;
+        this.currentRunID = null;
+        const summary = `${event.SuccessCount}/${event.TotalCount} migration(s) succeeded`;
+        // The one authoritative applied quartet for the run, in migrations — this is what feeds
+        // the reader's aggregateCounts.
+        emitter.stageComplete(RUN_STAGE, {
+            processed: event.TotalCount,
+            succeeded: event.SuccessCount,
+            failed: event.FailureCount,
+            totalKnown: event.TotalCount,
+        });
+        // Terminal writes are async (they flush the JSONL + write result.json). The observer
+        // contract is synchronous, so this is intentionally fire-and-forget with a logged catch —
+        // a progress-artifact write must never surface as a pipeline failure.
+        const terminal = event.Success
+            ? emitter.complete(`RSU pipeline complete — ${summary}`)
+            : emitter.fail(
+                `RSU pipeline failed at '${event.ErrorStep ?? 'unknown step'}' — ${summary}: ` +
+                `${event.ErrorMessage ?? 'no error message'}`,
+                'rsu-pipeline-failed'
+            );
+        void terminal.catch(err => LogError(`RSUProgressBridge: terminal write failed — ${err}`));
+    }
+
+    /**
+     * Starts reporting liveness for the step that just opened. Any previous timer is cleared first,
+     * so a missed `step.end` can never leave two timers running.
+     */
+    private startHeartbeat(stage: string): void {
+        this.stopHeartbeat();
+        const startedAt = Date.now();
+        this.heartbeat = setInterval(() => {
+            const elapsedSec = Math.round((Date.now() - startedAt) / 1000);
+            this.emitter?.heartbeat(stage, `${stage} still running — ${elapsedSec}s elapsed`);
+        }, HEARTBEAT_INTERVAL_MS);
+        // Never hold the process open on a progress timer.
+        this.heartbeat.unref?.();
+    }
+
+    /** Stops the liveness timer. Safe to call when none is running. */
+    private stopHeartbeat(): void {
+        if (!this.heartbeat) return;
+        clearInterval(this.heartbeat);
+        this.heartbeat = null;
+    }
+
+    /** Closes an orphaned emitter so a leaked run does not stay in-flight forever. */
+    private closeCurrent(reason: string): void {
+        this.stopHeartbeat();
+        const emitter = this.emitter;
+        this.emitter = null;
+        this.currentRunID = null;
+        void emitter?.fail(reason, 'rsu-run-abandoned')
+            .catch(err => LogError(`RSUProgressBridge: abandon write failed — ${err}`));
+    }
+
+    /**
+     * "step 4 of 12" — omitted entirely when the determinate counter isn't armed.
+     *
+     * Explicit null/undefined checks rather than truthiness: a 0 index is falsy, and silently
+     * dropping the label for the first step of a 0-based counter would be a bug that only appears
+     * if the pipeline's numbering ever changes.
+     */
+    private positionLabel(stepIndex?: number, stepTotal?: number): string | undefined {
+        if (stepIndex == null || stepTotal == null) return undefined;
+        return `step ${stepIndex} of ${stepTotal}`;
+    }
+}
+
+/**
+ * Attaches a fresh {@link RSUProgressBridge} to the RuntimeSchemaManager singleton. Call once at
+ * server startup, after the progress publish hook is registered so the very first RSU event also
+ * reaches live subscribers.
+ *
+ * @returns the bridge, so callers can read {@link RSUProgressBridge.CurrentRunID}.
+ */
+export function RegisterRSUProgressBridge(emitterOptions: EmitterOptions = {}): RSUProgressBridge {
+    const bridge = RSUProgressBridge.Instance;
+    bridge.Configure(emitterOptions);
+    RuntimeSchemaManager.Instance.PipelineObserver = bridge.Observe;
+    return bridge;
+}

package/src/integration/SchemaRefreshLaunch.ts

@@ -0,0 +1,77 @@
+/**
+ * @fileoverview Pure reporting helpers for the connector schema-refresh pipeline launch.
+ *
+ * The refresh is a live vendor introspect and can run for minutes on a large catalog, so
+ * IntegrationCreateConnection / IntegrationUpdateConnection can launch it *detached* and hand the
+ * caller a tailable run ID instead of blocking. That split creates one easy-to-get-wrong reporting
+ * case: a detached summary's counts are placeholder zeros, and describing them as results ("0
+ * created, 0 updated") reads to an operator as "the refresh found nothing" — the opposite of "the
+ * refresh has not finished yet". This module owns that decision so it can be pinned by a test
+ * without standing up a resolver.
+ */
+
+/**
+ * The subset of the pipeline summary that message construction reads. Structural so both the
+ * GraphQL `@ObjectType` and a test fixture satisfy it without either depending on the other.
+ */
+export interface SchemaRefreshSummaryLike {
+    /** Tailable run ID — the only meaningful field while `InProgress` is true. */
+    RunID: string;
+    /** True ⇒ launched detached and still running; every count below is a placeholder zero. */
+    InProgress: boolean;
+    /**
+     * Whether the pipeline itself succeeded. A pipeline that fails at ConnectionTest RETURNS rather
+     * than throwing, with every count at zero — so counts alone read identically to a clean
+     * no-op refresh. Reporting that as "0 created, 0 updated" tells an operator the vendor had
+     * nothing new when in fact the credentials were never accepted.
+     */
+    Succeeded: boolean;
+    /** The pipeline's own failure reason, when `Succeeded` is false. */
+    FailureMessage?: string;
+    ObjectsCreated: number;
+    ObjectsUpdated: number;
+    UnresolvedObjects: string[];
+}
+
+/** The counts clause, or an explicit failure clause when the pipeline did not succeed. */
+function describeFinishedRefresh(summary: SchemaRefreshSummaryLike): string {
+    if (!summary.Succeeded) {
+        return `schema refresh FAILED (${summary.FailureMessage ?? 'no reason reported'}) — see run ${summary.RunID}`;
+    }
+    return `schema refresh: ${summary.ObjectsCreated} created, ` +
+        `${summary.ObjectsUpdated} updated, ${summary.UnresolvedObjects.length} PK-unresolved`;
+}
+
+/**
+ * Builds the human-facing `Message` for IntegrationCreateConnection.
+ *
+ * @param testConnection whether the caller asked for (and got) a live connection test
+ * @param summary the refresh summary, or undefined when no refresh was requested
+ */
+export function BuildCreateConnectionMessage(
+    testConnection: boolean,
+    summary: SchemaRefreshSummaryLike | undefined
+): string {
+    if (!summary) return 'Connection created and test passed';
+    const created = `Connection created${testConnection ? ', test passed' : ''}`;
+    // Never report a detached run's placeholder counts as findings — point at the run stream instead.
+    if (summary.InProgress) return `${created}, schema refresh running — tail run ${summary.RunID}`;
+    return `${created}, ${describeFinishedRefresh(summary)}`;
+}
+
+/**
+ * Builds the human-facing `Message` for IntegrationUpdateConnection's blocking-refresh path.
+ * Shares {@link describeFinishedRefresh} with create so a failed refresh can never be reported as
+ * a clean zero-count run on one path and honestly on the other.
+ */
+export function BuildUpdateConnectionMessage(summary: SchemaRefreshSummaryLike): string {
+    return `Updated, ${describeFinishedRefresh(summary)}`;
+}
+
+/**
+ * Builds the human-facing `Message` for IntegrationUpdateConnection's detached-refresh path.
+ * Separate from {@link BuildCreateConnectionMessage} because update has no connection-test clause.
+ */
+export function BuildDetachedRefreshMessage(runID: string): string {
+    return `Updated, schema refresh running — tail run ${runID}`;
+}

package/src/logging/secretRedactor.ts

@@ -51,6 +51,20 @@ export function redactArg(ctx: RedactionContext): unknown {
     const entity = ctx.provider.Entities.find((e) => e.ClassName === match.groups!.name);
     if (entity) {
       encryptedFieldNames = new Set(entity.EncryptedFields.map((f) => f.Name));
+    } else {
+      // The name matched the CRUD convention but resolves to no entity — so the
+      // metadata half of the redactor contributes nothing and, absent any @NoLog
+      // field, this arg falls through to shortenForLog with its values intact.
+      //
+      // This is the silent case: the boot audit tests only the NAME pattern, so an
+      // input like `CreateConnectionInput` is classified as metadata-bound and never
+      // warned about, while the lookup here quietly fails. Every hand-written
+      // `Create*Input` in the resolvers hits this — codegen inputs embed the entity
+      // ClassName (`CreateMJCredentialInput`), hand-written ones do not.
+      //
+      // Report it once per input type so the operator who enabled variables logging
+      // learns which args are unprotected, rather than discovering it in a log file.
+      WarnUnboundCrudInput(ctx.inputTypeName);
     }
   }
 
@@ -80,3 +94,32 @@ export function redactArg(ctx: RedactionContext): unknown {
 }
 
 const EMPTY_SET: ReadonlySet<string> = new Set<string>();
+
+/**
+ * Input type names already reported by {@link WarnUnboundCrudInput}. Warning once
+ * per type keeps a hot mutation from flooding the log with the same diagnostic.
+ */
+const warnedUnboundInputTypes = new Set<string>();
+
+/**
+ * Reports, once per input type, that an arg named like a CRUD input resolves to no
+ * entity and is therefore logged unredacted unless its fields carry `@NoLog`.
+ *
+ * Exported for tests; not part of the redaction path's contract.
+ */
+export function WarnUnboundCrudInput(inputTypeName: string): void {
+  if (warnedUnboundInputTypes.has(inputTypeName)) {
+    return;
+  }
+  warnedUnboundInputTypes.add(inputTypeName);
+  console.warn(
+    `[MJServer] Variables logging: '${inputTypeName}' matches the Create/Update/Delete input ` +
+      `naming convention but maps to no entity, so encrypted-field redaction cannot apply to it. ` +
+      `Its values are being logged. Mark any sensitive @Field() on this input with @NoLog.`,
+  );
+}
+
+/** Test seam — clears the once-per-type warning state. */
+export function ResetUnboundCrudInputWarnings(): void {
+  warnedUnboundInputTypes.clear();
+}

package/src/middleware/BaseServerMiddleware.ts

@@ -126,6 +126,11 @@ export abstract class BaseServerMiddleware {
     /**
      * Hook that runs after a RunView operation completes. Can modify the result
      * (e.g., filtering or augmenting data) before it is returned to the caller.
+     *
+     * ⚠️ `results.Results` and its rows may be FROZEN shared cache state (LocalCacheManager
+     * freeze-on-write) — writing into a row throws `TypeError`. To modify, build copies and
+     * either reassign `results.Results = results.Results.map(r => ({ ...r, ... }))` or return
+     * a new result; never mutate rows in place.
      */
     PostRunView(params: RunViewParams, results: RunViewResult, contextUser: UserInfo | undefined): RunViewResult | Promise<RunViewResult> {
         return results;

package/src/providerConfigUnits.ts

@@ -0,0 +1,21 @@
+/**
+ * Unit-conversion seam between MJServer's configuration and SQLServerDataProvider.
+ *
+ * MJServer configures `databaseSettings.metadataCacheRefreshInterval` in **milliseconds**
+ * (default 180000 = 3 minutes, overridable via the METADATA_CACHE_REFRESH_INTERVAL env var).
+ * `SQLServerProviderConfigData`'s third constructor parameter is `checkRefreshIntervalSeconds`
+ * — declared in **seconds**.
+ *
+ * Commit 645c5a5e8 fixed a bug where the raw millisecond value was handed straight to the
+ * provider, so the "3 minute" default became a 180000-second (~50 hour) metadata cache
+ * refresh cadence. This function is the single place that conversion happens; both provider
+ * bootstrap call sites in `index.ts` (the main pool and the CodeGen-credentialed pool) go
+ * through it, and `config-units.test.ts` pins the contract so the units can never silently
+ * regress again.
+ *
+ * @param metadataCacheRefreshIntervalMs - the configured refresh interval, in milliseconds
+ * @returns the same interval expressed in seconds, the unit `checkRefreshIntervalSeconds` declares
+ */
+export function MetadataCacheRefreshIntervalSeconds(metadataCacheRefreshIntervalMs: number): number {
+    return metadataCacheRefreshIntervalMs / 1000;
+}

package/src/realtimeWidget/WidgetSessionService.ts

@@ -17,7 +17,7 @@
 import { Metadata, RunView, UserInfo, LogError, LogStatus } from '@memberjunction/core';
 import { MJLruCache, UUIDsEqual } from '@memberjunction/global';
 import type { MJConversationWidgetInstanceEntity, MJConversationEntity } from '@memberjunction/core-entities';
-import { UserCache } from '@memberjunction/sqlserver-dataprovider';
+import { UserCache } from '@memberjunction/generic-database-provider';
 import { MagicLinkKeyManager } from '../auth/magicLink/MagicLinkKeys.js';
 import { generateSessionId } from '../auth/magicLink/magicLinkCore.js';
 import { MagicLinkService } from '../auth/magicLink/MagicLinkService.js';

package/src/realtimeWidget/widgetGuestElevation.ts

@@ -15,7 +15,7 @@
 
 import { RunView, LogError, type UserInfo, type DatabaseProviderBase } from '@memberjunction/core';
 import type { MJConversationWidgetInstanceEntity } from '@memberjunction/core-entities';
-import { UserCache } from '@memberjunction/sqlserver-dataprovider';
+import { UserCache } from '@memberjunction/generic-database-provider';
 import type { UserPayload } from '../types.js';
 
 const WIDGET_ENTITY = 'MJ: Conversation Widget Instances';

package/src/resolvers/ActionResolver.ts

@@ -366,6 +366,19 @@ export class ActionResolver extends ResolverBase {
       // Run the entity action
       const result = await EntityActionEngineServer.Instance.RunEntityAction(params);
 
+      if (!result) {
+        // The binding is scoped (ScopeEntityID/ScopeRecordID) and this record falls outside it, so
+        // the action never ran. That is a legitimate answer, not a failure — dereferencing the null
+        // here turned "this binding does not apply to that record" into a server error for the
+        // caller. Reported as a successful no-op with an explicit message, so a client can tell it
+        // apart from an action that ran and did nothing.
+        return {
+          Success: true,
+          Message: 'The entity action did not run: its binding is scoped to different records.',
+          ResultData: JSON.stringify({ Ran: false, Reason: 'OutOfScope' })
+        };
+      }
+
       // Return the result
       return {
         Success: result.Success,

package/src/resolvers/ComponentRegistryResolver.ts

@@ -1,7 +1,7 @@
 import { Arg, Ctx, Field, InputType, ObjectType, Query, Mutation, Resolver } from 'type-graphql';
 import { UserInfo, LogError, LogStatus } from '@memberjunction/core';
 import { UUIDsEqual, MJLruCache } from '@memberjunction/global';
-import { UserCache } from '@memberjunction/sqlserver-dataprovider';
+import { UserCache } from '@memberjunction/generic-database-provider';
 import { MJComponentRegistryEntity, ComponentMetadataEngine } from '@memberjunction/core-entities';
 import { CredentialEngine } from '@memberjunction/credentials';
 import { ComponentSpec } from '@memberjunction/interactive-component-types';

package/src/resolvers/EntityPermissionResolver.ts

@@ -3,7 +3,7 @@ import { LogError } from '@memberjunction/core';
 import { AppContext } from '../types.js';
 import { ResolverBase } from '../generic/ResolverBase.js';
 import { GetReadOnlyProvider } from '../util.js';
-import { UserCache } from '@memberjunction/sqlserver-dataprovider';
+import { UserCache } from '@memberjunction/generic-database-provider';
 
 @ObjectType()
 class EntityPermissionResult {