@memberjunction/record-set-processor 0.0.1 → 5.43.0

package/README.md

@@ -1,45 +1,60 @@
 # @memberjunction/record-set-processor
 
-## ⚠️ IMPORTANT NOTICE ⚠️
-
-**This package is created solely for the purpose of setting up OIDC (OpenID Connect) trusted publishing with npm.**
-
-This is **NOT** a functional package and contains **NO** code or functionality beyond the OIDC setup configuration.
-
-## Purpose
-
-This package exists to:
-1. Configure OIDC trusted publishing for the package name `@memberjunction/record-set-processor`
-2. Enable secure, token-less publishing from CI/CD workflows
-3. Establish provenance for packages published under this name
-
-## What is OIDC Trusted Publishing?
-
-OIDC trusted publishing allows package maintainers to publish packages directly from their CI/CD workflows without needing to manage npm access tokens. Instead, it uses OpenID Connect to establish trust between the CI/CD provider (like GitHub Actions) and npm.
-
-## Setup Instructions
-
-To properly configure OIDC trusted publishing for this package:
-
-1. Go to [npmjs.com](https://www.npmjs.com/) and navigate to your package settings
-2. Configure the trusted publisher (e.g., GitHub Actions)
-3. Specify the repository and workflow that should be allowed to publish
-4. Use the configured workflow to publish your actual package
-
-## DO NOT USE THIS PACKAGE
-
-This package is a placeholder for OIDC configuration only. It:
-- Contains no executable code
-- Provides no functionality
-- Should not be installed as a dependency
-- Exists only for administrative purposes
-
-## More Information
-
-For more details about npm's trusted publishing feature, see:
-- [npm Trusted Publishing Documentation](https://docs.npmjs.com/generating-provenance-statements)
-- [GitHub Actions OIDC Documentation](https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect)
-
----
-
-**Maintained for OIDC setup purposes only**
+The server-side **Record Set Processor** engine — the single substrate every set-iterating job in
+MemberJunction routes through. It owns the hardened iteration lifecycle so each consumer doesn't
+reinvent it, while delegating *what to iterate*, *what to do*, and *where to persist* to the
+pluggable seams defined in [`@memberjunction/record-set-processor-base`](../base).
+
+## What the engine owns
+
+- **Batching** with a configurable batch size
+- **Bounded concurrency** within a batch (`maxConcurrency`)
+- **Rate limiting** (optional, token-bucket with a sliding window)
+- **Error-rate circuit breaker** — auto-fails a run when the error rate exceeds a threshold
+- **Budget gate** — an `onAfterBatch` hook to pause when a cost/item budget is exhausted
+- **Progress events** after each batch
+- **Pause / cancel handshake** — the tracker re-checks its row's cancellation flag at each checkpoint
+- **Resume from a checkpoint** — offset or keyset cursor, round-tripped through the tracker
+- **Per-record error isolation** — one bad record never aborts its batch
+
+## Usage
+
+```typescript
+import { RecordSetProcessor, FunctionRecordProcessor } from '@memberjunction/record-set-processor';
+import { ViewSource } from '@memberjunction/record-set-processor-base';
+
+const result = await RecordSetProcessor.Instance.Process({
+    source: new ViewSource(activeCustomersViewID),
+    processor: new FunctionRecordProcessor(async (record, ctx) => {
+        // ...do the work for `record` using ctx.provider / ctx.contextUser...
+        return { Status: 'Succeeded', ResultPayload: { summarized: true } };
+    }),
+    contextUser,
+    batchSize: 100,
+    maxConcurrency: 4,
+    recordProcessID,          // optional facade linkage
+    triggeredBy: 'Schedule',
+});
+
+console.log(`${result.Status}: ${result.Success}/${result.Processed} ok, run ${result.ProcessRunID}`);
+```
+
+By default the engine persists to `MJ: Process Runs` / `MJ: Process Run Details` via
+`GenericProcessRunTracker`. Pass `tracker: new NoOpTracker()` for fire-and-forget single-record work,
+or your own `IProcessRunTracker` to persist into domain-specific tables.
+
+## Exports
+
+| Export | Purpose |
+|---|---|
+| `RecordSetProcessor` | the engine singleton — `RecordSetProcessor.Instance.Process(options)` |
+| `GenericProcessRunTracker` | default tracker writing the generic Process Run tables |
+| `NoOpTracker` | a tracker that persists nothing |
+| `FunctionRecordProcessor` | a processor backed by a function |
+| `RateLimiter` | the token-bucket limiter used when `rateLimit` is configured |
+
+Source adapters, the seam interfaces, and all shared types are re-exported from
+`@memberjunction/record-set-processor-base`.
+
+> **Server-only.** This package executes work and persists runs; use it on the server. The base
+> package (types + seams + sources) is client-safe.

package/dist/RateLimiter.d.ts

@@ -0,0 +1,46 @@
+/**
+ * @fileoverview Token-bucket rate limiter with a sliding 60-second window and exponential backoff.
+ * Generic port of the content-autotagging limiter — gates the per-batch processing of a record set
+ * when a rate limit is configured.
+ * @module @memberjunction/record-set-processor
+ */
+/** Configuration for a {@link RateLimiter}. */
+export interface RateLimiterConfig {
+    /** Maximum requests per rolling 60-second window (default 60). */
+    RequestsPerMinute?: number;
+    /** Maximum tokens per rolling 60-second window (default 1,000,000). */
+    TokensPerMinute?: number;
+    /** Label used in diagnostics (default `RateLimiter`). */
+    Name?: string;
+}
+/**
+ * Limits the rate of an operation to a configured requests/tokens-per-minute, blocking callers in
+ * {@link Acquire} until capacity is available. Includes exponential backoff that callers can drive
+ * via {@link ReportRateLimitError} when the downstream service signals throttling.
+ */
+export declare class RateLimiter {
+    private readonly requestsPerMinute;
+    private readonly tokensPerMinute;
+    private readonly name;
+    private requestTimestamps;
+    private tokenLog;
+    private backoffMs;
+    private static readonly WINDOW_MS;
+    private static readonly MAX_BACKOFF_MS;
+    private static readonly POLL_MS;
+    constructor(config?: RateLimiterConfig);
+    /**
+     * Blocks until the configured request (and optional token) budget permits another call, then
+     * records the consumption.
+     * @param tokenCost - Token cost of the upcoming call (0 to skip the token budget).
+     */
+    Acquire(tokenCost?: number): Promise<void>;
+    /** Signals a downstream throttling error; grows the backoff window (1s → 2s → … → 60s cap). */
+    ReportRateLimitError(): void;
+    /** Clears any active backoff. */
+    ResetBackoff(): void;
+    /** Drops timestamps/token entries older than the rolling window. */
+    private prune;
+    private delay;
+}
+//# sourceMappingURL=RateLimiter.d.ts.map

package/dist/RateLimiter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RateLimiter.d.ts","sourceRoot":"","sources":["../src/RateLimiter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,+CAA+C;AAC/C,MAAM,WAAW,iBAAiB;IAC9B,kEAAkE;IAClE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,uEAAuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,yDAAyD;IACzD,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAAgB;IACzC,OAAO,CAAC,QAAQ,CAA+C;IAC/D,OAAO,CAAC,SAAS,CAAK;IAEtB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAU;IAC3C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAU;IAChD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAO;gBAE1B,MAAM,GAAE,iBAAsB;IAM1C;;;;OAIG;IACU,OAAO,CAAC,SAAS,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BlD,+FAA+F;IACxF,oBAAoB,IAAI,IAAI;IAInC,iCAAiC;IAC1B,YAAY,IAAI,IAAI;IAI3B,oEAAoE;IACpE,OAAO,CAAC,KAAK;IAMb,OAAO,CAAC,KAAK;CAGhB"}

package/dist/RateLimiter.js

@@ -0,0 +1,71 @@
+/**
+ * @fileoverview Token-bucket rate limiter with a sliding 60-second window and exponential backoff.
+ * Generic port of the content-autotagging limiter — gates the per-batch processing of a record set
+ * when a rate limit is configured.
+ * @module @memberjunction/record-set-processor
+ */
+/**
+ * Limits the rate of an operation to a configured requests/tokens-per-minute, blocking callers in
+ * {@link Acquire} until capacity is available. Includes exponential backoff that callers can drive
+ * via {@link ReportRateLimitError} when the downstream service signals throttling.
+ */
+export class RateLimiter {
+    static { this.WINDOW_MS = 60_000; }
+    static { this.MAX_BACKOFF_MS = 60_000; }
+    static { this.POLL_MS = 250; }
+    constructor(config = {}) {
+        this.requestTimestamps = [];
+        this.tokenLog = [];
+        this.backoffMs = 0;
+        this.requestsPerMinute = config.RequestsPerMinute ?? 60;
+        this.tokensPerMinute = config.TokensPerMinute ?? 1_000_000;
+        this.name = config.Name ?? 'RateLimiter';
+    }
+    /**
+     * Blocks until the configured request (and optional token) budget permits another call, then
+     * records the consumption.
+     * @param tokenCost - Token cost of the upcoming call (0 to skip the token budget).
+     */
+    async Acquire(tokenCost = 0) {
+        if (this.backoffMs > 0) {
+            await this.delay(this.backoffMs);
+        }
+        // Busy-wait (polling) until under both the request and token windows.
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const now = Date.now();
+            this.prune(now);
+            const underRequests = this.requestTimestamps.length < this.requestsPerMinute;
+            const tokensInWindow = this.tokenLog.reduce((sum, t) => sum + t.tokens, 0);
+            const underTokens = tokenCost <= 0 || tokensInWindow + tokenCost <= this.tokensPerMinute;
+            if (underRequests && underTokens) {
+                this.requestTimestamps.push(now);
+                if (tokenCost > 0) {
+                    this.tokenLog.push({ time: now, tokens: tokenCost });
+                }
+                // Halve backoff on a successful acquire.
+                this.backoffMs = Math.floor(this.backoffMs / 2);
+                return;
+            }
+            await this.delay(RateLimiter.POLL_MS);
+        }
+    }
+    /** Signals a downstream throttling error; grows the backoff window (1s → 2s → … → 60s cap). */
+    ReportRateLimitError() {
+        this.backoffMs = this.backoffMs === 0 ? 1000 : Math.min(this.backoffMs * 2, RateLimiter.MAX_BACKOFF_MS);
+    }
+    /** Clears any active backoff. */
+    ResetBackoff() {
+        this.backoffMs = 0;
+    }
+    /** Drops timestamps/token entries older than the rolling window. */
+    prune(now) {
+        const cutoff = now - RateLimiter.WINDOW_MS;
+        this.requestTimestamps = this.requestTimestamps.filter((t) => t > cutoff);
+        this.tokenLog = this.tokenLog.filter((t) => t.time > cutoff);
+    }
+    delay(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+}
+//# sourceMappingURL=RateLimiter.js.map

package/dist/RateLimiter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RateLimiter.js","sourceRoot":"","sources":["../src/RateLimiter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAYH;;;;GAIG;AACH,MAAM,OAAO,WAAW;aAQI,cAAS,GAAG,MAAM,AAAT,CAAU;aACnB,mBAAc,GAAG,MAAM,AAAT,CAAU;aACxB,YAAO,GAAG,GAAG,AAAN,CAAO;IAEtC,YAAY,SAA4B,EAAE;QARlC,sBAAiB,GAAa,EAAE,CAAC;QACjC,aAAQ,GAA4C,EAAE,CAAC;QACvD,cAAS,GAAG,CAAC,CAAC;QAOlB,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,SAAS,CAAC;QAC3D,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,aAAa,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC;QAC9B,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACrC,CAAC;QAED,sEAAsE;QACtE,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAEhB,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC;YAC7E,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC3E,MAAM,WAAW,GAAG,SAAS,IAAI,CAAC,IAAI,cAAc,GAAG,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC;YAEzF,IAAI,aAAa,IAAI,WAAW,EAAE,CAAC;gBAC/B,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBACzD,CAAC;gBACD,yCAAyC;gBACzC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;gBAChD,OAAO;YACX,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;IACL,CAAC;IAED,+FAA+F;IACxF,oBAAoB;QACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,WAAW,CAAC,cAAc,CAAC,CAAC;IAC5G,CAAC;IAED,iCAAiC;IAC1B,YAAY;QACf,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;IACvB,CAAC;IAED,oEAAoE;IAC5D,KAAK,CAAC,GAAW;QACrB,MAAM,MAAM,GAAG,GAAG,GAAG,WAAW,CAAC,SAAS,CAAC;QAC3C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;QAC1E,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC;IACjE,CAAC;IAEO,KAAK,CAAC,EAAU;QACpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC"}

package/dist/RecordProcessExecutor.d.ts

@@ -0,0 +1,53 @@
+/**
+ * @fileoverview Turns a `MJ: Record Processes` definition into a concrete RecordSetProcessor run:
+ * builds the source from its Scope, the processor from its Work (+ output-mapping write-back), and
+ * executes. Shared by the on-demand (RunNow) operation, the scheduled-job driver, and on-change.
+ * @module @memberjunction/record-set-processor
+ */
+import { IMetadataProvider, UserInfo } from '@memberjunction/core';
+import { MJRecordProcessEntity } from '@memberjunction/core-entities';
+import { IRecordProcessor, IRecordSetSource, ProcessRunResult, ProgressInfo, TriggeredByValue } from '@memberjunction/record-set-processor-base';
+import { type RecordProcessScopeOverride } from '@memberjunction/core-entities';
+/** Options for executing a Record Process. */
+export interface RunRecordProcessOptions {
+    /** The acting user. */
+    contextUser: UserInfo;
+    /** The owning provider (defaults to the active provider). */
+    provider?: IMetadataProvider;
+    /** What triggered the run (default `OnDemand`). */
+    triggeredBy?: TriggeredByValue;
+    /** Process a single record (the on-change / on-demand single-record case) instead of the scope. */
+    singleRecordID?: string;
+    /**
+     * Runtime scope override (selected rows / a view / a list / a filter), used instead of the stored
+     * Scope. This is how a grid/list UI runs a process against exactly what the user is looking at.
+     */
+    scope?: RecordProcessScopeOverride;
+    /**
+     * Compute-only: for work types that support it (currently `FieldRules`), produce the per-record diff
+     * WITHOUT writing. Powers the preview step of the bulk-update UX.
+     */
+    dryRun?: boolean;
+    /** FK to the owning `ScheduledJobRun` when launched by the scheduler (links the Process Run back). */
+    scheduledJobRunID?: string;
+    /** Progress callback. */
+    onProgress?: (progress: ProgressInfo) => void;
+}
+/** Builds and runs a RecordSetProcessor pass from a Record Process definition. */
+export declare class RecordProcessExecutor {
+    /** Loads the Record Process by ID and runs it. */
+    RunByID(recordProcessID: string, options: RunRecordProcessOptions): Promise<ProcessRunResult>;
+    /** Runs an already-loaded Record Process. */
+    Run(rp: MJRecordProcessEntity, options: RunRecordProcessOptions): Promise<ProcessRunResult>;
+    /** Builds the record-set source from a single-record override, a runtime scope override, or the process's stored Scope. */
+    buildSource(rp: MJRecordProcessEntity, provider: IMetadataProvider, singleRecordID?: string, scope?: RecordProcessScopeOverride): IRecordSetSource;
+    /** Resolves a runtime scope override (UI invocation: selection / view / list / filter) to a source. */
+    private buildSourceFromScope;
+    /**
+     * Builds the processor from the process's Work. `FieldRules` reads its rule set from `Configuration`
+     * and writes itself (honoring `dryRun`); the other work types build a base processor wrapped with
+     * output-mapping write-back when configured.
+     */
+    buildProcessor(rp: MJRecordProcessEntity, dryRun?: boolean): IRecordProcessor;
+}
+//# sourceMappingURL=RecordProcessExecutor.d.ts.map

package/dist/RecordProcessExecutor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RecordProcessExecutor.d.ts","sourceRoot":"","sources":["../src/RecordProcessExecutor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,iBAAiB,EAAY,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAE7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAGH,gBAAgB,EAChB,gBAAgB,EAEhB,gBAAgB,EAChB,YAAY,EACZ,gBAAgB,EAEnB,MAAM,2CAA2C,CAAC;AAEnD,OAAO,EAAE,KAAK,0BAA0B,EAAE,MAAM,+BAA+B,CAAC;AAShF,8CAA8C;AAC9C,MAAM,WAAW,uBAAuB;IACpC,uBAAuB;IACvB,WAAW,EAAE,QAAQ,CAAC;IACtB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,mDAAmD;IACnD,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,mGAAmG;IACnG,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,KAAK,CAAC,EAAE,0BAA0B,CAAC;IACnC;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,sGAAsG;IACtG,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,yBAAyB;IACzB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CACjD;AAED,kFAAkF;AAClF,qBAAa,qBAAqB;IAC9B,kDAAkD;IACrC,OAAO,CAAC,eAAe,EAAE,MAAM,EAAE,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAU1G,6CAA6C;IAChC,GAAG,CAAC,EAAE,EAAE,qBAAqB,EAAE,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAmBxG,2HAA2H;IACpH,WAAW,CAAC,EAAE,EAAE,qBAAqB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,cAAc,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,0BAA0B,GAAG,gBAAgB;IAgCzJ,uGAAuG;IACvG,OAAO,CAAC,oBAAoB;IAkB5B;;;;OAIG;IACI,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,gBAAgB;CAqCvF"}

package/dist/RecordProcessExecutor.js

@@ -0,0 +1,141 @@
+/**
+ * @fileoverview Turns a `MJ: Record Processes` definition into a concrete RecordSetProcessor run:
+ * builds the source from its Scope, the processor from its Work (+ output-mapping write-back), and
+ * executes. Shared by the on-demand (RunNow) operation, the scheduled-job driver, and on-change.
+ * @module @memberjunction/record-set-processor
+ */
+import { Metadata } from '@memberjunction/core';
+import { SafeJSONParse } from '@memberjunction/global';
+import { ArraySource, FilterSource, ListSource, ViewSource, } from '@memberjunction/record-set-processor-base';
+import { RecordSetProcessor } from './RecordSetProcessor.js';
+import { ActionRecordProcessor } from './processors/ActionRecordProcessor.js';
+import { AgentRecordProcessor } from './processors/AgentRecordProcessor.js';
+import { InferProcessor } from './processors/InferProcessor.js';
+import { FieldRulesProcessor } from './processors/FieldRulesProcessor.js';
+import { WriteBackProcessor } from './processors/WriteBackProcessor.js';
+/** Builds and runs a RecordSetProcessor pass from a Record Process definition. */
+export class RecordProcessExecutor {
+    /** Loads the Record Process by ID and runs it. */
+    async RunByID(recordProcessID, options) {
+        const provider = options.provider ?? Metadata.Provider;
+        const rp = await provider.GetEntityObject('MJ: Record Processes', options.contextUser);
+        const loaded = await rp.Load(recordProcessID);
+        if (!loaded) {
+            throw new Error(`Record Process '${recordProcessID}' not found`);
+        }
+        return this.Run(rp, options);
+    }
+    /** Runs an already-loaded Record Process. */
+    async Run(rp, options) {
+        const provider = options.provider ?? Metadata.Provider;
+        return RecordSetProcessor.Instance.Process({
+            source: this.buildSource(rp, provider, options.singleRecordID, options.scope),
+            processor: this.buildProcessor(rp, options.dryRun),
+            contextUser: options.contextUser,
+            provider,
+            dryRun: options.dryRun,
+            recordProcessID: rp.ID,
+            scheduledJobRunID: options.scheduledJobRunID,
+            entityID: rp.EntityID,
+            triggeredBy: options.triggeredBy ?? 'OnDemand',
+            batchSize: rp.BatchSize ?? undefined,
+            maxConcurrency: rp.MaxConcurrency ?? undefined,
+            onProgress: options.onProgress,
+            configuration: { recordProcessName: rp.Name, workType: rp.WorkType, scopeType: rp.ScopeType },
+        });
+    }
+    /** Builds the record-set source from a single-record override, a runtime scope override, or the process's stored Scope. */
+    buildSource(rp, provider, singleRecordID, scope) {
+        if (singleRecordID) {
+            return new ArraySource([{ EntityID: rp.EntityID, RecordID: singleRecordID }], rp.EntityID, 'SingleRecord');
+        }
+        if (scope) {
+            return this.buildSourceFromScope(rp, provider, scope);
+        }
+        switch (rp.ScopeType) {
+            case 'View':
+                if (!rp.ScopeViewID) {
+                    throw new Error(`Record Process '${rp.Name}': ScopeType=View requires ScopeViewID`);
+                }
+                return new ViewSource(rp.ScopeViewID);
+            case 'List':
+                if (!rp.ScopeListID) {
+                    throw new Error(`Record Process '${rp.Name}': ScopeType=List requires ScopeListID`);
+                }
+                return new ListSource(rp.ScopeListID);
+            case 'Filter': {
+                const entity = provider.EntityByID(rp.EntityID);
+                if (!entity) {
+                    throw new Error(`Record Process '${rp.Name}': entity '${rp.EntityID}' not found in metadata`);
+                }
+                return new FilterSource(entity.Name, rp.ScopeFilter ?? undefined);
+            }
+            case 'SingleRecord':
+                throw new Error(`Record Process '${rp.Name}': ScopeType=SingleRecord requires a record ID (pass singleRecordID)`);
+            default:
+                throw new Error(`Record Process '${rp.Name}': unsupported ScopeType '${rp.ScopeType}'`);
+        }
+    }
+    /** Resolves a runtime scope override (UI invocation: selection / view / list / filter) to a source. */
+    buildSourceFromScope(rp, provider, scope) {
+        switch (scope.Kind) {
+            case 'records':
+                return new ArraySource(scope.RecordIDs.map((id) => ({ EntityID: rp.EntityID, RecordID: id })), rp.EntityID, 'Array');
+            case 'view':
+                return new ViewSource(scope.ViewID);
+            case 'list':
+                return new ListSource(scope.ListID);
+            case 'filter': {
+                const entity = provider.EntityByID(rp.EntityID);
+                if (!entity) {
+                    throw new Error(`Record Process '${rp.Name}': entity '${rp.EntityID}' not found in metadata`);
+                }
+                return new FilterSource(entity.Name, scope.Filter);
+            }
+        }
+    }
+    /**
+     * Builds the processor from the process's Work. `FieldRules` reads its rule set from `Configuration`
+     * and writes itself (honoring `dryRun`); the other work types build a base processor wrapped with
+     * output-mapping write-back when configured.
+     */
+    buildProcessor(rp, dryRun) {
+        if (rp.WorkType === 'FieldRules') {
+            const ruleSet = rp.Configuration ? SafeJSONParse(rp.Configuration) : undefined;
+            if (!ruleSet || !Array.isArray(ruleSet.Rules)) {
+                throw new Error(`Record Process '${rp.Name}': WorkType=FieldRules requires a FieldRuleSet (with a Rules array) in Configuration`);
+            }
+            // FieldRules owns its own write / dry-run preview — no output-mapping write-back wrapper.
+            return new FieldRulesProcessor({ RuleSet: ruleSet, DryRun: dryRun });
+        }
+        const inputMapping = rp.InputMapping ? SafeJSONParse(rp.InputMapping) : undefined;
+        let base;
+        if (rp.WorkType === 'Action') {
+            if (!rp.ActionID) {
+                throw new Error(`Record Process '${rp.Name}': WorkType=Action requires ActionID`);
+            }
+            base = new ActionRecordProcessor(rp.ActionID, inputMapping);
+        }
+        else if (rp.WorkType === 'Agent') {
+            if (!rp.AgentID) {
+                throw new Error(`Record Process '${rp.Name}': WorkType=Agent requires AgentID`);
+            }
+            base = new AgentRecordProcessor(rp.AgentID, inputMapping);
+        }
+        else if (rp.WorkType === 'Infer') {
+            if (!rp.PromptID) {
+                throw new Error(`Record Process '${rp.Name}': WorkType=Infer requires PromptID`);
+            }
+            base = new InferProcessor(rp.PromptID, inputMapping);
+        }
+        else {
+            throw new Error(`Record Process '${rp.Name}': unsupported WorkType '${rp.WorkType}'`);
+        }
+        const outputMapping = rp.OutputMapping ? SafeJSONParse(rp.OutputMapping) : undefined;
+        if (outputMapping && (outputMapping.fields || outputMapping.childRecord)) {
+            return new WriteBackProcessor(base, outputMapping);
+        }
+        return base;
+    }
+}
+//# sourceMappingURL=RecordProcessExecutor.js.map

package/dist/RecordProcessExecutor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RecordProcessExecutor.js","sourceRoot":"","sources":["../src/RecordProcessExecutor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAqB,QAAQ,EAAY,MAAM,sBAAsB,CAAC;AAC7E,OAAO,EAAE,aAAa,EAAqB,MAAM,wBAAwB,CAAC;AAE1E,OAAO,EACH,WAAW,EACX,YAAY,EAGZ,UAAU,EAIV,UAAU,GACb,MAAM,2CAA2C,CAAC;AAGnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AA6BrE,kFAAkF;AAClF,MAAM,OAAO,qBAAqB;IAC9B,kDAAkD;IAC3C,KAAK,CAAC,OAAO,CAAC,eAAuB,EAAE,OAAgC;QAC1E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC;QACvD,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAwB,sBAAsB,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9G,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,mBAAmB,eAAe,aAAa,CAAC,CAAC;QACrE,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACjC,CAAC;IAED,6CAA6C;IACtC,KAAK,CAAC,GAAG,CAAC,EAAyB,EAAE,OAAgC;QACxE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC;QACvD,OAAO,kBAAkB,CAAC,QAAQ,CAAC,OAAO,CAAC;YACvC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,KAAK,CAAC;YAC7E,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC;YAClD,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,QAAQ;YACR,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,eAAe,EAAE,EAAE,CAAC,EAAE;YACtB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,UAAU;YAC9C,SAAS,EAAE,EAAE,CAAC,SAAS,IAAI,SAAS;YACpC,cAAc,EAAE,EAAE,CAAC,cAAc,IAAI,SAAS;YAC9C,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,aAAa,EAAE,EAAE,iBAAiB,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE;SAChG,CAAC,CAAC;IACP,CAAC;IAED,2HAA2H;IACpH,WAAW,CAAC,EAAyB,EAAE,QAA2B,EAAE,cAAuB,EAAE,KAAkC;QAClI,IAAI,cAAc,EAAE,CAAC;YACjB,OAAO,IAAI,WAAW,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QAC/G,CAAC;QACD,IAAI,KAAK,EAAE,CAAC;YACR,OAAO,IAAI,CAAC,oBAAoB,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC1D,CAAC;QACD,QAAQ,EAAE,CAAC,SAAS,EAAE,CAAC;YACnB,KAAK,MAAM;gBACP,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,wCAAwC,CAAC,CAAC;gBACxF,CAAC;gBACD,OAAO,IAAI,UAAU,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;YAC1C,KAAK,MAAM;gBACP,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,wCAAwC,CAAC,CAAC;gBACxF,CAAC;gBACD,OAAO,IAAI,UAAU,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;YAC1C,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACZ,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC,QAAQ,yBAAyB,CAAC,CAAC;gBAClG,CAAC;gBACD,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,IAAI,SAAS,CAAC,CAAC;YACtE,CAAC;YACD,KAAK,cAAc;gBACf,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,sEAAsE,CAAC,CAAC;YACtH;gBACI,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,6BAA6B,EAAE,CAAC,SAAS,GAAG,CAAC,CAAC;QAChG,CAAC;IACL,CAAC;IAED,uGAAuG;IAC/F,oBAAoB,CAAC,EAAyB,EAAE,QAA2B,EAAE,KAAiC;QAClH,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACjB,KAAK,SAAS;gBACV,OAAO,IAAI,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACzH,KAAK,MAAM;gBACP,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACxC,KAAK,MAAM;gBACP,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACxC,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACZ,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC,QAAQ,yBAAyB,CAAC,CAAC;gBAClG,CAAC;gBACD,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACvD,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;OAIG;IACI,cAAc,CAAC,EAAyB,EAAE,MAAgB;QAC7D,IAAI,EAAE,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAe,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7F,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5C,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,sFAAsF,CAAC,CAAC;YACtI,CAAC;YACD,0FAA0F;YAC1F,OAAO,IAAI,mBAAmB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAClF,IAAI,IAAsB,CAAC;QAC3B,IAAI,EAAE,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,sCAAsC,CAAC,CAAC;YACtF,CAAC;YACD,IAAI,GAAG,IAAI,qBAAqB,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAChE,CAAC;aAAM,IAAI,EAAE,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACjC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,oCAAoC,CAAC,CAAC;YACpF,CAAC;YACD,IAAI,GAAG,IAAI,oBAAoB,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QAC9D,CAAC;aAAM,IAAI,EAAE,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACjC,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,qCAAqC,CAAC,CAAC;YACrF,CAAC;YACD,IAAI,GAAG,IAAI,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACzD,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,4BAA4B,EAAE,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC1F,CAAC;QAED,MAAM,aAAa,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAsB,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1G,IAAI,aAAa,IAAI,CAAC,aAAa,CAAC,MAAM,IAAI,aAAa,CAAC,WAAW,CAAC,EAAE,CAAC;YACvE,OAAO,IAAI,kBAAkB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ"}

package/dist/RecordSetProcessor.d.ts

@@ -0,0 +1,36 @@
+/**
+ * @fileoverview The Record Set Processor engine — the single substrate every set-iterating job
+ * routes through. It owns batching, bounded concurrency, optional rate limiting, an error-rate
+ * circuit breaker, a budget-gate hook, progress events, the pause/cancel handshake, resume from a
+ * checkpoint, and per-record error isolation — while delegating *what to iterate* (source), *what
+ * to do* (processor), and *where to persist* (tracker) to pluggable seams.
+ * @module @memberjunction/record-set-processor
+ */
+import { BaseSingleton } from '@memberjunction/global';
+import { ProcessRunResult, RecordSetProcessOptions } from '@memberjunction/record-set-processor-base';
+/**
+ * The server-side engine that processes a record set through the source/processor/tracker pipeline.
+ * Use the {@link Instance} singleton: `await RecordSetProcessor.Instance.Process({ ... })`.
+ */
+export declare class RecordSetProcessor extends BaseSingleton<RecordSetProcessor> {
+    /** The process-wide singleton instance. */
+    static get Instance(): RecordSetProcessor;
+    /**
+     * Processes a record set end to end and returns the run summary.
+     * Logical failures never throw — inspect the returned {@link ProcessRunResult}.
+     *
+     * @param options - The source, processor, tracker, and execution knobs for this run.
+     * @returns The run summary plus the persisted run ID (when a persisting tracker was used).
+     */
+    Process(options: RecordSetProcessOptions): Promise<ProcessRunResult>;
+    /** The core batch loop. Returns the terminal status. */
+    private runBatchLoop;
+    /** Processes one batch with bounded concurrency, isolating per-record failures. */
+    private processBatch;
+    /** Runs `worker` over `items` with at most `limit` in flight at once. */
+    private runWithConcurrency;
+    /** Invokes the budget-gate hook, treating a thrown hook as non-fatal (continue). */
+    private safeAfterBatch;
+    private delay;
+}
+//# sourceMappingURL=RecordSetProcessor.d.ts.map

package/dist/RecordSetProcessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RecordSetProcessor.d.ts","sourceRoot":"","sources":["../src/RecordSetProcessor.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAIH,gBAAgB,EAKhB,uBAAuB,EAG1B,MAAM,2CAA2C,CAAC;AASnD;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,aAAa,CAAC,kBAAkB,CAAC;IACrE,2CAA2C;IAC3C,WAAkB,QAAQ,IAAI,kBAAkB,CAE/C;IAED;;;;;;OAMG;IACU,OAAO,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAmEjF,wDAAwD;YAC1C,YAAY;IAoE1B,mFAAmF;YACrE,YAAY;IA6B1B,yEAAyE;YAC3D,kBAAkB;IAiBhC,oFAAoF;YACtE,cAAc;IAa5B,OAAO,CAAC,KAAK;CAGhB"}