@memberjunction/record-set-processor 0.0.1 → 5.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +58 -43
  2. package/dist/RateLimiter.d.ts +46 -0
  3. package/dist/RateLimiter.d.ts.map +1 -0
  4. package/dist/RateLimiter.js +71 -0
  5. package/dist/RateLimiter.js.map +1 -0
  6. package/dist/RecordProcessExecutor.d.ts +36 -0
  7. package/dist/RecordProcessExecutor.d.ts.map +1 -0
  8. package/dist/RecordProcessExecutor.js +106 -0
  9. package/dist/RecordProcessExecutor.js.map +1 -0
  10. package/dist/RecordSetProcessor.d.ts +36 -0
  11. package/dist/RecordSetProcessor.d.ts.map +1 -0
  12. package/dist/RecordSetProcessor.js +215 -0
  13. package/dist/RecordSetProcessor.js.map +1 -0
  14. package/dist/index.d.ts +20 -0
  15. package/dist/index.d.ts.map +1 -0
  16. package/dist/index.js +20 -0
  17. package/dist/index.js.map +1 -0
  18. package/dist/operations/RecordProcessControlOperations.d.ts +61 -0
  19. package/dist/operations/RecordProcessControlOperations.d.ts.map +1 -0
  20. package/dist/operations/RecordProcessControlOperations.js +117 -0
  21. package/dist/operations/RecordProcessControlOperations.js.map +1 -0
  22. package/dist/operations/RecordProcessGetRunStatusOperation.d.ts +34 -0
  23. package/dist/operations/RecordProcessGetRunStatusOperation.d.ts.map +1 -0
  24. package/dist/operations/RecordProcessGetRunStatusOperation.js +53 -0
  25. package/dist/operations/RecordProcessGetRunStatusOperation.js.map +1 -0
  26. package/dist/processors/ActionRecordProcessor.d.ts +27 -0
  27. package/dist/processors/ActionRecordProcessor.d.ts.map +1 -0
  28. package/dist/processors/ActionRecordProcessor.js +61 -0
  29. package/dist/processors/ActionRecordProcessor.js.map +1 -0
  30. package/dist/processors/AgentRecordProcessor.d.ts +23 -0
  31. package/dist/processors/AgentRecordProcessor.d.ts.map +1 -0
  32. package/dist/processors/AgentRecordProcessor.js +51 -0
  33. package/dist/processors/AgentRecordProcessor.js.map +1 -0
  34. package/dist/processors/FunctionRecordProcessor.d.ts +16 -0
  35. package/dist/processors/FunctionRecordProcessor.d.ts.map +1 -0
  36. package/dist/processors/FunctionRecordProcessor.js +16 -0
  37. package/dist/processors/FunctionRecordProcessor.js.map +1 -0
  38. package/dist/processors/InferProcessor.d.ts +24 -0
  39. package/dist/processors/InferProcessor.d.ts.map +1 -0
  40. package/dist/processors/InferProcessor.js +60 -0
  41. package/dist/processors/InferProcessor.js.map +1 -0
  42. package/dist/processors/WriteBackProcessor.d.ts +20 -0
  43. package/dist/processors/WriteBackProcessor.d.ts.map +1 -0
  44. package/dist/processors/WriteBackProcessor.js +41 -0
  45. package/dist/processors/WriteBackProcessor.js.map +1 -0
  46. package/dist/trackers/GenericProcessRunTracker.d.ts +17 -0
  47. package/dist/trackers/GenericProcessRunTracker.d.ts.map +1 -0
  48. package/dist/trackers/GenericProcessRunTracker.js +120 -0
  49. package/dist/trackers/GenericProcessRunTracker.js.map +1 -0
  50. package/dist/trackers/NoOpTracker.d.ts +15 -0
  51. package/dist/trackers/NoOpTracker.d.ts.map +1 -0
  52. package/dist/trackers/NoOpTracker.js +24 -0
  53. package/dist/trackers/NoOpTracker.js.map +1 -0
  54. package/dist/writeBack.d.ts +42 -0
  55. package/dist/writeBack.d.ts.map +1 -0
  56. package/dist/writeBack.js +59 -0
  57. package/dist/writeBack.js.map +1 -0
  58. package/package.json +37 -7
package/README.md CHANGED
@@ -1,45 +1,60 @@
1
1
  # @memberjunction/record-set-processor
2
2
 
3
- ## ⚠️ IMPORTANT NOTICE ⚠️
4
-
5
- **This package is created solely for the purpose of setting up OIDC (OpenID Connect) trusted publishing with npm.**
6
-
7
- This is **NOT** a functional package and contains **NO** code or functionality beyond the OIDC setup configuration.
8
-
9
- ## Purpose
10
-
11
- This package exists to:
12
- 1. Configure OIDC trusted publishing for the package name `@memberjunction/record-set-processor`
13
- 2. Enable secure, token-less publishing from CI/CD workflows
14
- 3. Establish provenance for packages published under this name
15
-
16
- ## What is OIDC Trusted Publishing?
17
-
18
- OIDC trusted publishing allows package maintainers to publish packages directly from their CI/CD workflows without needing to manage npm access tokens. Instead, it uses OpenID Connect to establish trust between the CI/CD provider (like GitHub Actions) and npm.
19
-
20
- ## Setup Instructions
21
-
22
- To properly configure OIDC trusted publishing for this package:
23
-
24
- 1. Go to [npmjs.com](https://www.npmjs.com/) and navigate to your package settings
25
- 2. Configure the trusted publisher (e.g., GitHub Actions)
26
- 3. Specify the repository and workflow that should be allowed to publish
27
- 4. Use the configured workflow to publish your actual package
28
-
29
- ## DO NOT USE THIS PACKAGE
30
-
31
- This package is a placeholder for OIDC configuration only. It:
32
- - Contains no executable code
33
- - Provides no functionality
34
- - Should not be installed as a dependency
35
- - Exists only for administrative purposes
36
-
37
- ## More Information
38
-
39
- For more details about npm's trusted publishing feature, see:
40
- - [npm Trusted Publishing Documentation](https://docs.npmjs.com/generating-provenance-statements)
41
- - [GitHub Actions OIDC Documentation](https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect)
42
-
43
- ---
44
-
45
- **Maintained for OIDC setup purposes only**
3
+ The server-side **Record Set Processor** engine — the single substrate every set-iterating job in
4
+ MemberJunction routes through. It owns the hardened iteration lifecycle so each consumer doesn't
5
+ reinvent it, while delegating *what to iterate*, *what to do*, and *where to persist* to the
6
+ pluggable seams defined in [`@memberjunction/record-set-processor-base`](../base).
7
+
8
+ ## What the engine owns
9
+
10
+ - **Batching** with a configurable batch size
11
+ - **Bounded concurrency** within a batch (`maxConcurrency`)
12
+ - **Rate limiting** (optional, token-bucket with a sliding window)
13
+ - **Error-rate circuit breaker** — auto-fails a run when the error rate exceeds a threshold
14
+ - **Budget gate** an `onAfterBatch` hook to pause when a cost/item budget is exhausted
15
+ - **Progress events** after each batch
16
+ - **Pause / cancel handshake** — the tracker re-checks its row's cancellation flag at each checkpoint
17
+ - **Resume from a checkpoint** — offset or keyset cursor, round-tripped through the tracker
18
+ - **Per-record error isolation** one bad record never aborts its batch
19
+
20
+ ## Usage
21
+
22
+ ```typescript
23
+ import { RecordSetProcessor, FunctionRecordProcessor } from '@memberjunction/record-set-processor';
24
+ import { ViewSource } from '@memberjunction/record-set-processor-base';
25
+
26
+ const result = await RecordSetProcessor.Instance.Process({
27
+ source: new ViewSource(activeCustomersViewID),
28
+ processor: new FunctionRecordProcessor(async (record, ctx) => {
29
+ // ...do the work for `record` using ctx.provider / ctx.contextUser...
30
+ return { Status: 'Succeeded', ResultPayload: { summarized: true } };
31
+ }),
32
+ contextUser,
33
+ batchSize: 100,
34
+ maxConcurrency: 4,
35
+ recordProcessID, // optional facade linkage
36
+ triggeredBy: 'Schedule',
37
+ });
38
+
39
+ console.log(`${result.Status}: ${result.Success}/${result.Processed} ok, run ${result.ProcessRunID}`);
40
+ ```
41
+
42
+ By default the engine persists to `MJ: Process Runs` / `MJ: Process Run Details` via
43
+ `GenericProcessRunTracker`. Pass `tracker: new NoOpTracker()` for fire-and-forget single-record work,
44
+ or your own `IProcessRunTracker` to persist into domain-specific tables.
45
+
46
+ ## Exports
47
+
48
+ | Export | Purpose |
49
+ |---|---|
50
+ | `RecordSetProcessor` | the engine singleton — `RecordSetProcessor.Instance.Process(options)` |
51
+ | `GenericProcessRunTracker` | default tracker writing the generic Process Run tables |
52
+ | `NoOpTracker` | a tracker that persists nothing |
53
+ | `FunctionRecordProcessor` | a processor backed by a function |
54
+ | `RateLimiter` | the token-bucket limiter used when `rateLimit` is configured |
55
+
56
+ Source adapters, the seam interfaces, and all shared types are re-exported from
57
+ `@memberjunction/record-set-processor-base`.
58
+
59
+ > **Server-only.** This package executes work and persists runs; use it on the server. The base
60
+ > package (types + seams + sources) is client-safe.
@@ -0,0 +1,46 @@
1
+ /**
2
+ * @fileoverview Token-bucket rate limiter with a sliding 60-second window and exponential backoff.
3
+ * Generic port of the content-autotagging limiter — gates the per-batch processing of a record set
4
+ * when a rate limit is configured.
5
+ * @module @memberjunction/record-set-processor
6
+ */
7
+ /** Configuration for a {@link RateLimiter}. */
8
+ export interface RateLimiterConfig {
9
+ /** Maximum requests per rolling 60-second window (default 60). */
10
+ RequestsPerMinute?: number;
11
+ /** Maximum tokens per rolling 60-second window (default 1,000,000). */
12
+ TokensPerMinute?: number;
13
+ /** Label used in diagnostics (default `RateLimiter`). */
14
+ Name?: string;
15
+ }
16
+ /**
17
+ * Limits the rate of an operation to a configured requests/tokens-per-minute, blocking callers in
18
+ * {@link Acquire} until capacity is available. Includes exponential backoff that callers can drive
19
+ * via {@link ReportRateLimitError} when the downstream service signals throttling.
20
+ */
21
+ export declare class RateLimiter {
22
+ private readonly requestsPerMinute;
23
+ private readonly tokensPerMinute;
24
+ private readonly name;
25
+ private requestTimestamps;
26
+ private tokenLog;
27
+ private backoffMs;
28
+ private static readonly WINDOW_MS;
29
+ private static readonly MAX_BACKOFF_MS;
30
+ private static readonly POLL_MS;
31
+ constructor(config?: RateLimiterConfig);
32
+ /**
33
+ * Blocks until the configured request (and optional token) budget permits another call, then
34
+ * records the consumption.
35
+ * @param tokenCost - Token cost of the upcoming call (0 to skip the token budget).
36
+ */
37
+ Acquire(tokenCost?: number): Promise<void>;
38
+ /** Signals a downstream throttling error; grows the backoff window (1s → 2s → … → 60s cap). */
39
+ ReportRateLimitError(): void;
40
+ /** Clears any active backoff. */
41
+ ResetBackoff(): void;
42
+ /** Drops timestamps/token entries older than the rolling window. */
43
+ private prune;
44
+ private delay;
45
+ }
46
+ //# sourceMappingURL=RateLimiter.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"RateLimiter.d.ts","sourceRoot":"","sources":["../src/RateLimiter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,+CAA+C;AAC/C,MAAM,WAAW,iBAAiB;IAC9B,kEAAkE;IAClE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,uEAAuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,yDAAyD;IACzD,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAAgB;IACzC,OAAO,CAAC,QAAQ,CAA+C;IAC/D,OAAO,CAAC,SAAS,CAAK;IAEtB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAU;IAC3C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAU;IAChD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAO;gBAE1B,MAAM,GAAE,iBAAsB;IAM1C;;;;OAIG;IACU,OAAO,CAAC,SAAS,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BlD,+FAA+F;IACxF,oBAAoB,IAAI,IAAI;IAInC,iCAAiC;IAC1B,YAAY,IAAI,IAAI;IAI3B,oEAAoE;IACpE,OAAO,CAAC,KAAK;IAMb,OAAO,CAAC,KAAK;CAGhB"}
@@ -0,0 +1,71 @@
1
+ /**
2
+ * @fileoverview Token-bucket rate limiter with a sliding 60-second window and exponential backoff.
3
+ * Generic port of the content-autotagging limiter — gates the per-batch processing of a record set
4
+ * when a rate limit is configured.
5
+ * @module @memberjunction/record-set-processor
6
+ */
7
+ /**
8
+ * Limits the rate of an operation to a configured requests/tokens-per-minute, blocking callers in
9
+ * {@link Acquire} until capacity is available. Includes exponential backoff that callers can drive
10
+ * via {@link ReportRateLimitError} when the downstream service signals throttling.
11
+ */
12
+ export class RateLimiter {
13
+ static { this.WINDOW_MS = 60_000; }
14
+ static { this.MAX_BACKOFF_MS = 60_000; }
15
+ static { this.POLL_MS = 250; }
16
+ constructor(config = {}) {
17
+ this.requestTimestamps = [];
18
+ this.tokenLog = [];
19
+ this.backoffMs = 0;
20
+ this.requestsPerMinute = config.RequestsPerMinute ?? 60;
21
+ this.tokensPerMinute = config.TokensPerMinute ?? 1_000_000;
22
+ this.name = config.Name ?? 'RateLimiter';
23
+ }
24
+ /**
25
+ * Blocks until the configured request (and optional token) budget permits another call, then
26
+ * records the consumption.
27
+ * @param tokenCost - Token cost of the upcoming call (0 to skip the token budget).
28
+ */
29
+ async Acquire(tokenCost = 0) {
30
+ if (this.backoffMs > 0) {
31
+ await this.delay(this.backoffMs);
32
+ }
33
+ // Busy-wait (polling) until under both the request and token windows.
34
+ // eslint-disable-next-line no-constant-condition
35
+ while (true) {
36
+ const now = Date.now();
37
+ this.prune(now);
38
+ const underRequests = this.requestTimestamps.length < this.requestsPerMinute;
39
+ const tokensInWindow = this.tokenLog.reduce((sum, t) => sum + t.tokens, 0);
40
+ const underTokens = tokenCost <= 0 || tokensInWindow + tokenCost <= this.tokensPerMinute;
41
+ if (underRequests && underTokens) {
42
+ this.requestTimestamps.push(now);
43
+ if (tokenCost > 0) {
44
+ this.tokenLog.push({ time: now, tokens: tokenCost });
45
+ }
46
+ // Halve backoff on a successful acquire.
47
+ this.backoffMs = Math.floor(this.backoffMs / 2);
48
+ return;
49
+ }
50
+ await this.delay(RateLimiter.POLL_MS);
51
+ }
52
+ }
53
+ /** Signals a downstream throttling error; grows the backoff window (1s → 2s → … → 60s cap). */
54
+ ReportRateLimitError() {
55
+ this.backoffMs = this.backoffMs === 0 ? 1000 : Math.min(this.backoffMs * 2, RateLimiter.MAX_BACKOFF_MS);
56
+ }
57
+ /** Clears any active backoff. */
58
+ ResetBackoff() {
59
+ this.backoffMs = 0;
60
+ }
61
+ /** Drops timestamps/token entries older than the rolling window. */
62
+ prune(now) {
63
+ const cutoff = now - RateLimiter.WINDOW_MS;
64
+ this.requestTimestamps = this.requestTimestamps.filter((t) => t > cutoff);
65
+ this.tokenLog = this.tokenLog.filter((t) => t.time > cutoff);
66
+ }
67
+ delay(ms) {
68
+ return new Promise((resolve) => setTimeout(resolve, ms));
69
+ }
70
+ }
71
+ //# sourceMappingURL=RateLimiter.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"RateLimiter.js","sourceRoot":"","sources":["../src/RateLimiter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAYH;;;;GAIG;AACH,MAAM,OAAO,WAAW;aAQI,cAAS,GAAG,MAAM,AAAT,CAAU;aACnB,mBAAc,GAAG,MAAM,AAAT,CAAU;aACxB,YAAO,GAAG,GAAG,AAAN,CAAO;IAEtC,YAAY,SAA4B,EAAE;QARlC,sBAAiB,GAAa,EAAE,CAAC;QACjC,aAAQ,GAA4C,EAAE,CAAC;QACvD,cAAS,GAAG,CAAC,CAAC;QAOlB,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,IAAI,SAAS,CAAC;QAC3D,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,aAAa,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC;QAC9B,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACrC,CAAC;QAED,sEAAsE;QACtE,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAEhB,MAAM,aAAa,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC;YAC7E,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC3E,MAAM,WAAW,GAAG,SAAS,IAAI,CAAC,IAAI,cAAc,GAAG,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC;YAEzF,IAAI,aAAa,IAAI,WAAW,EAAE,CAAC;gBAC/B,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBACzD,CAAC;gBACD,yCAAyC;gBACzC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;gBAChD,OAAO;YACX,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;IACL,CAAC;IAED,+FAA+F;IACxF,oBAAoB;QACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,WAAW,CAAC,cAAc,CAAC,CAAC;IAC5G,CAAC;IAED,iCAAiC;IAC1B,YAAY;QACf,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;IACvB,CAAC;IAED,oEAAoE;IAC5D,KAAK,CAAC,GAAW;QACrB,MAAM,MAAM,GAAG,GAAG,GAAG,WAAW,CAAC,SAAS,CAAC;QAC3C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;QAC1E,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC;IACjE,CAAC;IAEO,KAAK,CAAC,EAAU;QACpB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC"}
@@ -0,0 +1,36 @@
1
+ /**
2
+ * @fileoverview Turns a `MJ: Record Processes` definition into a concrete RecordSetProcessor run:
3
+ * builds the source from its Scope, the processor from its Work (+ output-mapping write-back), and
4
+ * executes. Shared by the on-demand (RunNow) operation, the scheduled-job driver, and on-change.
5
+ * @module @memberjunction/record-set-processor
6
+ */
7
+ import { IMetadataProvider, UserInfo } from '@memberjunction/core';
8
+ import { MJRecordProcessEntity } from '@memberjunction/core-entities';
9
+ import { IRecordProcessor, IRecordSetSource, ProcessRunResult, ProgressInfo, TriggeredByValue } from '@memberjunction/record-set-processor-base';
10
+ /** Options for executing a Record Process. */
11
+ export interface RunRecordProcessOptions {
12
+ /** The acting user. */
13
+ contextUser: UserInfo;
14
+ /** The owning provider (defaults to the active provider). */
15
+ provider?: IMetadataProvider;
16
+ /** What triggered the run (default `OnDemand`). */
17
+ triggeredBy?: TriggeredByValue;
18
+ /** Process a single record (the on-change / on-demand single-record case) instead of the scope. */
19
+ singleRecordID?: string;
20
+ /** FK to the owning `ScheduledJobRun` when launched by the scheduler (links the Process Run back). */
21
+ scheduledJobRunID?: string;
22
+ /** Progress callback. */
23
+ onProgress?: (progress: ProgressInfo) => void;
24
+ }
25
+ /** Builds and runs a RecordSetProcessor pass from a Record Process definition. */
26
+ export declare class RecordProcessExecutor {
27
+ /** Loads the Record Process by ID and runs it. */
28
+ RunByID(recordProcessID: string, options: RunRecordProcessOptions): Promise<ProcessRunResult>;
29
+ /** Runs an already-loaded Record Process. */
30
+ Run(rp: MJRecordProcessEntity, options: RunRecordProcessOptions): Promise<ProcessRunResult>;
31
+ /** Builds the record-set source from the process's Scope (or a single-record override). */
32
+ buildSource(rp: MJRecordProcessEntity, provider: IMetadataProvider, singleRecordID?: string): IRecordSetSource;
33
+ /** Builds the processor from the process's Work, wrapping with output-mapping write-back when configured. */
34
+ buildProcessor(rp: MJRecordProcessEntity): IRecordProcessor;
35
+ }
36
+ //# sourceMappingURL=RecordProcessExecutor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"RecordProcessExecutor.d.ts","sourceRoot":"","sources":["../src/RecordProcessExecutor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,iBAAiB,EAAY,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAE7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAGH,gBAAgB,EAChB,gBAAgB,EAEhB,gBAAgB,EAChB,YAAY,EACZ,gBAAgB,EAEnB,MAAM,2CAA2C,CAAC;AAQnD,8CAA8C;AAC9C,MAAM,WAAW,uBAAuB;IACpC,uBAAuB;IACvB,WAAW,EAAE,QAAQ,CAAC;IACtB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,mDAAmD;IACnD,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,mGAAmG;IACnG,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,sGAAsG;IACtG,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,yBAAyB;IACzB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CACjD;AAED,kFAAkF;AAClF,qBAAa,qBAAqB;IAC9B,kDAAkD;IACrC,OAAO,CAAC,eAAe,EAAE,MAAM,EAAE,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAU1G,6CAA6C;IAChC,GAAG,CAAC,EAAE,EAAE,qBAAqB,EAAE,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkBxG,2FAA2F;IACpF,WAAW,CAAC,EAAE,EAAE,qBAAqB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,gBAAgB;IA6BrH,6GAA6G;IACtG,cAAc,CAAC,EAAE,EAAE,qBAAqB,GAAG,gBAAgB;CA4BrE"}
@@ -0,0 +1,106 @@
1
+ /**
2
+ * @fileoverview Turns a `MJ: Record Processes` definition into a concrete RecordSetProcessor run:
3
+ * builds the source from its Scope, the processor from its Work (+ output-mapping write-back), and
4
+ * executes. Shared by the on-demand (RunNow) operation, the scheduled-job driver, and on-change.
5
+ * @module @memberjunction/record-set-processor
6
+ */
7
+ import { Metadata } from '@memberjunction/core';
8
+ import { SafeJSONParse } from '@memberjunction/global';
9
+ import { ArraySource, FilterSource, ListSource, ViewSource, } from '@memberjunction/record-set-processor-base';
10
+ import { RecordSetProcessor } from './RecordSetProcessor.js';
11
+ import { ActionRecordProcessor } from './processors/ActionRecordProcessor.js';
12
+ import { AgentRecordProcessor } from './processors/AgentRecordProcessor.js';
13
+ import { InferProcessor } from './processors/InferProcessor.js';
14
+ import { WriteBackProcessor } from './processors/WriteBackProcessor.js';
15
+ /** Builds and runs a RecordSetProcessor pass from a Record Process definition. */
16
+ export class RecordProcessExecutor {
17
+ /** Loads the Record Process by ID and runs it. */
18
+ async RunByID(recordProcessID, options) {
19
+ const provider = options.provider ?? Metadata.Provider;
20
+ const rp = await provider.GetEntityObject('MJ: Record Processes', options.contextUser);
21
+ const loaded = await rp.Load(recordProcessID);
22
+ if (!loaded) {
23
+ throw new Error(`Record Process '${recordProcessID}' not found`);
24
+ }
25
+ return this.Run(rp, options);
26
+ }
27
+ /** Runs an already-loaded Record Process. */
28
+ async Run(rp, options) {
29
+ const provider = options.provider ?? Metadata.Provider;
30
+ return RecordSetProcessor.Instance.Process({
31
+ source: this.buildSource(rp, provider, options.singleRecordID),
32
+ processor: this.buildProcessor(rp),
33
+ contextUser: options.contextUser,
34
+ provider,
35
+ recordProcessID: rp.ID,
36
+ scheduledJobRunID: options.scheduledJobRunID,
37
+ entityID: rp.EntityID,
38
+ triggeredBy: options.triggeredBy ?? 'OnDemand',
39
+ batchSize: rp.BatchSize ?? undefined,
40
+ maxConcurrency: rp.MaxConcurrency ?? undefined,
41
+ onProgress: options.onProgress,
42
+ configuration: { recordProcessName: rp.Name, workType: rp.WorkType, scopeType: rp.ScopeType },
43
+ });
44
+ }
45
+ /** Builds the record-set source from the process's Scope (or a single-record override). */
46
+ buildSource(rp, provider, singleRecordID) {
47
+ if (singleRecordID) {
48
+ return new ArraySource([{ EntityID: rp.EntityID, RecordID: singleRecordID }], rp.EntityID, 'SingleRecord');
49
+ }
50
+ switch (rp.ScopeType) {
51
+ case 'View':
52
+ if (!rp.ScopeViewID) {
53
+ throw new Error(`Record Process '${rp.Name}': ScopeType=View requires ScopeViewID`);
54
+ }
55
+ return new ViewSource(rp.ScopeViewID);
56
+ case 'List':
57
+ if (!rp.ScopeListID) {
58
+ throw new Error(`Record Process '${rp.Name}': ScopeType=List requires ScopeListID`);
59
+ }
60
+ return new ListSource(rp.ScopeListID);
61
+ case 'Filter': {
62
+ const entity = provider.EntityByID(rp.EntityID);
63
+ if (!entity) {
64
+ throw new Error(`Record Process '${rp.Name}': entity '${rp.EntityID}' not found in metadata`);
65
+ }
66
+ return new FilterSource(entity.Name, rp.ScopeFilter ?? undefined);
67
+ }
68
+ case 'SingleRecord':
69
+ throw new Error(`Record Process '${rp.Name}': ScopeType=SingleRecord requires a record ID (pass singleRecordID)`);
70
+ default:
71
+ throw new Error(`Record Process '${rp.Name}': unsupported ScopeType '${rp.ScopeType}'`);
72
+ }
73
+ }
74
+ /** Builds the processor from the process's Work, wrapping with output-mapping write-back when configured. */
75
+ buildProcessor(rp) {
76
+ const inputMapping = rp.InputMapping ? SafeJSONParse(rp.InputMapping) : undefined;
77
+ let base;
78
+ if (rp.WorkType === 'Action') {
79
+ if (!rp.ActionID) {
80
+ throw new Error(`Record Process '${rp.Name}': WorkType=Action requires ActionID`);
81
+ }
82
+ base = new ActionRecordProcessor(rp.ActionID, inputMapping);
83
+ }
84
+ else if (rp.WorkType === 'Agent') {
85
+ if (!rp.AgentID) {
86
+ throw new Error(`Record Process '${rp.Name}': WorkType=Agent requires AgentID`);
87
+ }
88
+ base = new AgentRecordProcessor(rp.AgentID, inputMapping);
89
+ }
90
+ else if (rp.WorkType === 'Infer') {
91
+ if (!rp.PromptID) {
92
+ throw new Error(`Record Process '${rp.Name}': WorkType=Infer requires PromptID`);
93
+ }
94
+ base = new InferProcessor(rp.PromptID, inputMapping);
95
+ }
96
+ else {
97
+ throw new Error(`Record Process '${rp.Name}': unsupported WorkType '${rp.WorkType}'`);
98
+ }
99
+ const outputMapping = rp.OutputMapping ? SafeJSONParse(rp.OutputMapping) : undefined;
100
+ if (outputMapping && (outputMapping.fields || outputMapping.childRecord)) {
101
+ return new WriteBackProcessor(base, outputMapping);
102
+ }
103
+ return base;
104
+ }
105
+ }
106
+ //# sourceMappingURL=RecordProcessExecutor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"RecordProcessExecutor.js","sourceRoot":"","sources":["../src/RecordProcessExecutor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAqB,QAAQ,EAAY,MAAM,sBAAsB,CAAC;AAC7E,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EACH,WAAW,EACX,YAAY,EAGZ,UAAU,EAIV,UAAU,GACb,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAmBrE,kFAAkF;AAClF,MAAM,OAAO,qBAAqB;IAC9B,kDAAkD;IAC3C,KAAK,CAAC,OAAO,CAAC,eAAuB,EAAE,OAAgC;QAC1E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC;QACvD,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAwB,sBAAsB,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9G,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,mBAAmB,eAAe,aAAa,CAAC,CAAC;QACrE,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACjC,CAAC;IAED,6CAA6C;IACtC,KAAK,CAAC,GAAG,CAAC,EAAyB,EAAE,OAAgC;QACxE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC;QACvD,OAAO,kBAAkB,CAAC,QAAQ,CAAC,OAAO,CAAC;YACvC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,cAAc,CAAC;YAC9D,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,QAAQ;YACR,eAAe,EAAE,EAAE,CAAC,EAAE;YACtB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,UAAU;YAC9C,SAAS,EAAE,EAAE,CAAC,SAAS,IAAI,SAAS;YACpC,cAAc,EAAE,EAAE,CAAC,cAAc,IAAI,SAAS;YAC9C,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,aAAa,EAAE,EAAE,iBAAiB,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,CAAC,SAAS,EAAE;SAChG,CAAC,CAAC;IACP,CAAC;IAED,2FAA2F;IACpF,WAAW,CAAC,EAAyB,EAAE,QAA2B,EAAE,cAAuB;QAC9F,IAAI,cAAc,EAAE,CAAC;YACjB,OAAO,IAAI,WAAW,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QAC/G,CAAC;QACD,QAAQ,EAAE,CAAC,SAAS,EAAE,CAAC;YACnB,KAAK,MAAM;gBACP,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,wCAAwC,CAAC,CAAC;gBACxF,CAAC;gBACD,OAAO,IAAI,UAAU,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;YAC1C,KAAK,MAAM;gBACP,IAAI,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC;oBAClB,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,wCAAwC,CAAC,CAAC;gBACxF,CAAC;gBACD,OAAO,IAAI,UAAU,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;YAC1C,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACZ,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC,QAAQ,yBAAyB,CAAC,CAAC;gBAClG,CAAC;gBACD,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,IAAI,SAAS,CAAC,CAAC;YACtE,CAAC;YACD,KAAK,cAAc;gBACf,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,sEAAsE,CAAC,CAAC;YACtH;gBACI,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,6BAA6B,EAAE,CAAC,SAAS,GAAG,CAAC,CAAC;QAChG,CAAC;IACL,CAAC;IAED,6GAA6G;IACtG,cAAc,CAAC,EAAyB;QAC3C,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,aAAa,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAClF,IAAI,IAAsB,CAAC;QAC3B,IAAI,EAAE,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,sCAAsC,CAAC,CAAC;YACtF,CAAC;YACD,IAAI,GAAG,IAAI,qBAAqB,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAChE,CAAC;aAAM,IAAI,EAAE,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACjC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,oCAAoC,CAAC,CAAC;YACpF,CAAC;YACD,IAAI,GAAG,IAAI,oBAAoB,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QAC9D,CAAC;aAAM,IAAI,EAAE,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACjC,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,qCAAqC,CAAC,CAAC;YACrF,CAAC;YACD,IAAI,GAAG,IAAI,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QACzD,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC,IAAI,4BAA4B,EAAE,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC1F,CAAC;QAED,MAAM,aAAa,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAsB,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1G,IAAI,aAAa,IAAI,CAAC,aAAa,CAAC,MAAM,IAAI,aAAa,CAAC,WAAW,CAAC,EAAE,CAAC;YACvE,OAAO,IAAI,kBAAkB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ"}
@@ -0,0 +1,36 @@
1
+ /**
2
+ * @fileoverview The Record Set Processor engine — the single substrate every set-iterating job
3
+ * routes through. It owns batching, bounded concurrency, optional rate limiting, an error-rate
4
+ * circuit breaker, a budget-gate hook, progress events, the pause/cancel handshake, resume from a
5
+ * checkpoint, and per-record error isolation — while delegating *what to iterate* (source), *what
6
+ * to do* (processor), and *where to persist* (tracker) to pluggable seams.
7
+ * @module @memberjunction/record-set-processor
8
+ */
9
+ import { BaseSingleton } from '@memberjunction/global';
10
+ import { ProcessRunResult, RecordSetProcessOptions } from '@memberjunction/record-set-processor-base';
11
+ /**
12
+ * The server-side engine that processes a record set through the source/processor/tracker pipeline.
13
+ * Use the {@link Instance} singleton: `await RecordSetProcessor.Instance.Process({ ... })`.
14
+ */
15
+ export declare class RecordSetProcessor extends BaseSingleton<RecordSetProcessor> {
16
+ /** The process-wide singleton instance. */
17
+ static get Instance(): RecordSetProcessor;
18
+ /**
19
+ * Processes a record set end to end and returns the run summary.
20
+ * Logical failures never throw — inspect the returned {@link ProcessRunResult}.
21
+ *
22
+ * @param options - The source, processor, tracker, and execution knobs for this run.
23
+ * @returns The run summary plus the persisted run ID (when a persisting tracker was used).
24
+ */
25
+ Process(options: RecordSetProcessOptions): Promise<ProcessRunResult>;
26
+ /** The core batch loop. Returns the terminal status. */
27
+ private runBatchLoop;
28
+ /** Processes one batch with bounded concurrency, isolating per-record failures. */
29
+ private processBatch;
30
+ /** Runs `worker` over `items` with at most `limit` in flight at once. */
31
+ private runWithConcurrency;
32
+ /** Invokes the budget-gate hook, treating a thrown hook as non-fatal (continue). */
33
+ private safeAfterBatch;
34
+ private delay;
35
+ }
36
+ //# sourceMappingURL=RecordSetProcessor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"RecordSetProcessor.d.ts","sourceRoot":"","sources":["../src/RecordSetProcessor.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAIH,gBAAgB,EAKhB,uBAAuB,EAG1B,MAAM,2CAA2C,CAAC;AASnD;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,aAAa,CAAC,kBAAkB,CAAC;IACrE,2CAA2C;IAC3C,WAAkB,QAAQ,IAAI,kBAAkB,CAE/C;IAED;;;;;;OAMG;IACU,OAAO,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkEjF,wDAAwD;YAC1C,YAAY;IAoE1B,mFAAmF;YACrE,YAAY;IA6B1B,yEAAyE;YAC3D,kBAAkB;IAiBhC,oFAAoF;YACtE,cAAc;IAa5B,OAAO,CAAC,KAAK;CAGhB"}
@@ -0,0 +1,215 @@
1
+ /**
2
+ * @fileoverview The Record Set Processor engine — the single substrate every set-iterating job
3
+ * routes through. It owns batching, bounded concurrency, optional rate limiting, an error-rate
4
+ * circuit breaker, a budget-gate hook, progress events, the pause/cancel handshake, resume from a
5
+ * checkpoint, and per-record error isolation — while delegating *what to iterate* (source), *what
6
+ * to do* (processor), and *where to persist* (tracker) to pluggable seams.
7
+ * @module @memberjunction/record-set-processor
8
+ */
9
+ import { BaseSingleton } from '@memberjunction/global';
10
+ import { LogError, Metadata } from '@memberjunction/core';
11
+ import { RateLimiter } from './RateLimiter.js';
12
+ import { GenericProcessRunTracker } from './trackers/GenericProcessRunTracker.js';
13
+ /** Default batch size when none is supplied. */
14
+ const DEFAULT_BATCH_SIZE = 100;
15
+ /** Default error-rate circuit-breaker threshold (percent). */
16
+ const DEFAULT_ERROR_THRESHOLD = 20;
17
+ /**
18
+ * The server-side engine that processes a record set through the source/processor/tracker pipeline.
19
+ * Use the {@link Instance} singleton: `await RecordSetProcessor.Instance.Process({ ... })`.
20
+ */
21
+ export class RecordSetProcessor extends BaseSingleton {
22
+ /** The process-wide singleton instance. */
23
+ static get Instance() {
24
+ return super.getInstance();
25
+ }
26
+ /**
27
+ * Processes a record set end to end and returns the run summary.
28
+ * Logical failures never throw — inspect the returned {@link ProcessRunResult}.
29
+ *
30
+ * @param options - The source, processor, tracker, and execution knobs for this run.
31
+ * @returns The run summary plus the persisted run ID (when a persisting tracker was used).
32
+ */
33
+ async Process(options) {
34
+ const { source, processor, contextUser } = options;
35
+ const provider = options.provider ?? Metadata.Provider;
36
+ const tracker = options.tracker ?? new GenericProcessRunTracker();
37
+ const batchSize = options.batchSize ?? DEFAULT_BATCH_SIZE;
38
+ const maxConcurrency = Math.max(1, options.maxConcurrency ?? 1);
39
+ const errorThreshold = options.errorThresholdPercent ?? DEFAULT_ERROR_THRESHOLD;
40
+ const delayMs = options.delayBetweenBatchesMs ?? 0;
41
+ const shouldResume = options.resume ?? true;
42
+ const rateLimiter = options.rateLimit
43
+ ? new RateLimiter({ RequestsPerMinute: options.rateLimit.requestsPerMinute, TokensPerMinute: options.rateLimit.tokensPerMinute })
44
+ : undefined;
45
+ const descriptor = source.Describe();
46
+ const meta = {
47
+ RecordProcessID: options.recordProcessID,
48
+ ScheduledJobRunID: options.scheduledJobRunID,
49
+ EntityID: options.entityID ?? descriptor.EntityID,
50
+ TriggeredBy: options.triggeredBy ?? 'Manual',
51
+ SourceType: descriptor.SourceType,
52
+ SourceID: descriptor.SourceID,
53
+ SourceFilter: descriptor.SourceFilter,
54
+ BatchSize: batchSize,
55
+ Configuration: options.configuration,
56
+ };
57
+ const handle = await tracker.BeginRun(meta, contextUser, provider);
58
+ const counts = { Processed: 0, Success: 0, Error: 0, Skipped: 0 };
59
+ let cursor = shouldResume ? await tracker.LoadResumeCursor(handle, contextUser, provider) : undefined;
60
+ let total = null;
61
+ let status = 'Running';
62
+ let errorMessage;
63
+ const startTime = new Date();
64
+ try {
65
+ status = await this.runBatchLoop({
66
+ options, source, processor, tracker, handle, provider,
67
+ batchSize, maxConcurrency, errorThreshold, delayMs, rateLimiter,
68
+ counts,
69
+ getCursor: () => cursor,
70
+ setCursor: (c) => { cursor = c; },
71
+ getTotal: () => total,
72
+ setTotal: (t) => { total = t; },
73
+ setErrorMessage: (m) => { errorMessage = m; },
74
+ });
75
+ }
76
+ catch (e) {
77
+ status = 'Failed';
78
+ errorMessage = e instanceof Error ? e.message : String(e);
79
+ LogError(`RecordSetProcessor: run failed: ${errorMessage}`);
80
+ }
81
+ const summary = {
82
+ Status: status,
83
+ Total: total,
84
+ Processed: counts.Processed,
85
+ Success: counts.Success,
86
+ Error: counts.Error,
87
+ Skipped: counts.Skipped,
88
+ StartTime: startTime,
89
+ EndTime: new Date(),
90
+ ErrorMessage: errorMessage,
91
+ };
92
+ await tracker.CompleteRun(handle, summary, contextUser, provider);
93
+ return { ...summary, ProcessRunID: handle.ProcessRunID };
94
+ }
95
+ /** The core batch loop. Returns the terminal status. */
96
+ async runBatchLoop(ctx) {
97
+ const { options, source, batchSize, counts } = ctx;
98
+ const { contextUser } = options;
99
+ // eslint-disable-next-line no-constant-condition
100
+ while (true) {
101
+ const batch = await source.NextBatch(ctx.getCursor(), batchSize, contextUser, ctx.provider);
102
+ if (batch.TotalRowCount != null && ctx.getTotal() == null) {
103
+ ctx.setTotal(batch.TotalRowCount);
104
+ }
105
+ if (batch.Records.length === 0) {
106
+ return 'Completed';
107
+ }
108
+ // Honor a hard record cap by trimming the batch.
109
+ let records = batch.Records;
110
+ if (options.maxRecords != null) {
111
+ const remaining = options.maxRecords - counts.Processed;
112
+ if (remaining <= 0) {
113
+ return 'Completed';
114
+ }
115
+ if (records.length > remaining) {
116
+ records = records.slice(0, remaining);
117
+ }
118
+ }
119
+ if (ctx.rateLimiter) {
120
+ await ctx.rateLimiter.Acquire();
121
+ }
122
+ await this.processBatch(records, ctx);
123
+ options.onProgress?.({ ...counts, Total: ctx.getTotal() });
124
+ // Checkpoint + pause/cancel handshake.
125
+ ctx.setCursor(batch.NextCursor);
126
+ const shouldContinue = await ctx.tracker.Checkpoint(ctx.handle, batch.NextCursor, counts, contextUser, ctx.provider);
127
+ if (!shouldContinue) {
128
+ return 'Paused';
129
+ }
130
+ // Budget gate.
131
+ if (options.onAfterBatch) {
132
+ const verdict = await this.safeAfterBatch(options.onAfterBatch, records, counts.Processed);
133
+ if (verdict && !verdict.continue) {
134
+ ctx.setErrorMessage(verdict.reason ? `Auto-paused: ${verdict.reason}` : 'Auto-paused: budget exceeded');
135
+ return 'Paused';
136
+ }
137
+ }
138
+ // Error-rate circuit breaker.
139
+ if (counts.Processed > 0 && counts.Error > 0) {
140
+ const errorRate = (counts.Error / counts.Processed) * 100;
141
+ if (errorRate > ctx.errorThreshold) {
142
+ ctx.setErrorMessage(`Circuit breaker: error rate ${errorRate.toFixed(1)}% exceeded ${ctx.errorThreshold}%`);
143
+ return 'Failed';
144
+ }
145
+ }
146
+ if (batch.Exhausted) {
147
+ return 'Completed';
148
+ }
149
+ if (ctx.delayMs > 0) {
150
+ await this.delay(ctx.delayMs);
151
+ }
152
+ }
153
+ }
154
+ /** Processes one batch with bounded concurrency, isolating per-record failures. */
155
+ async processBatch(records, ctx) {
156
+ const { options, counts } = ctx;
157
+ const recordContext = { contextUser: options.contextUser, provider: ctx.provider, processRunID: ctx.handle.ProcessRunID };
158
+ const worker = async (record) => {
159
+ const started = Date.now();
160
+ let result;
161
+ try {
162
+ result = await options.processor.ProcessRecord(record, recordContext);
163
+ }
164
+ catch (e) {
165
+ result = { Status: 'Failed', ErrorMessage: e instanceof Error ? e.message : String(e) };
166
+ }
167
+ if (result.DurationMs == null) {
168
+ result.DurationMs = Date.now() - started;
169
+ }
170
+ counts.Processed++;
171
+ if (result.Status === 'Succeeded') {
172
+ counts.Success++;
173
+ }
174
+ else if (result.Status === 'Skipped') {
175
+ counts.Skipped++;
176
+ }
177
+ else {
178
+ counts.Error++;
179
+ }
180
+ await ctx.tracker.RecordResult(ctx.handle, record, result, options.contextUser, ctx.provider);
181
+ };
182
+ await this.runWithConcurrency(records, ctx.maxConcurrency, worker);
183
+ }
184
+ /** Runs `worker` over `items` with at most `limit` in flight at once. */
185
+ async runWithConcurrency(items, limit, worker) {
186
+ if (limit <= 1) {
187
+ for (const item of items) {
188
+ await worker(item);
189
+ }
190
+ return;
191
+ }
192
+ let index = 0;
193
+ const runNext = async () => {
194
+ while (index < items.length) {
195
+ const current = index++;
196
+ await worker(items[current]);
197
+ }
198
+ };
199
+ await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => runNext()));
200
+ }
201
+ /** Invokes the budget-gate hook, treating a thrown hook as non-fatal (continue). */
202
+ async safeAfterBatch(hook, records, processed) {
203
+ try {
204
+ return await hook(records, processed);
205
+ }
206
+ catch (e) {
207
+ LogError(`RecordSetProcessor: onAfterBatch hook threw (continuing): ${e instanceof Error ? e.message : String(e)}`);
208
+ return null;
209
+ }
210
+ }
211
+ delay(ms) {
212
+ return new Promise((resolve) => setTimeout(resolve, ms));
213
+ }
214
+ }
215
+ //# sourceMappingURL=RecordSetProcessor.js.map