@memberjunction/record-set-processor 0.0.1 → 5.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -43
- package/dist/RateLimiter.d.ts +46 -0
- package/dist/RateLimiter.d.ts.map +1 -0
- package/dist/RateLimiter.js +71 -0
- package/dist/RateLimiter.js.map +1 -0
- package/dist/RecordProcessExecutor.d.ts +53 -0
- package/dist/RecordProcessExecutor.d.ts.map +1 -0
- package/dist/RecordProcessExecutor.js +141 -0
- package/dist/RecordProcessExecutor.js.map +1 -0
- package/dist/RecordSetProcessor.d.ts +36 -0
- package/dist/RecordSetProcessor.d.ts.map +1 -0
- package/dist/RecordSetProcessor.js +216 -0
- package/dist/RecordSetProcessor.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +21 -0
- package/dist/index.js.map +1 -0
- package/dist/operations/RecordProcessControlOperations.d.ts +33 -0
- package/dist/operations/RecordProcessControlOperations.d.ts.map +1 -0
- package/dist/operations/RecordProcessControlOperations.js +117 -0
- package/dist/operations/RecordProcessControlOperations.js.map +1 -0
- package/dist/operations/RecordProcessGetRunStatusOperation.d.ts +19 -0
- package/dist/operations/RecordProcessGetRunStatusOperation.d.ts.map +1 -0
- package/dist/operations/RecordProcessGetRunStatusOperation.js +51 -0
- package/dist/operations/RecordProcessGetRunStatusOperation.js.map +1 -0
- package/dist/processors/ActionRecordProcessor.d.ts +27 -0
- package/dist/processors/ActionRecordProcessor.d.ts.map +1 -0
- package/dist/processors/ActionRecordProcessor.js +61 -0
- package/dist/processors/ActionRecordProcessor.js.map +1 -0
- package/dist/processors/AgentRecordProcessor.d.ts +23 -0
- package/dist/processors/AgentRecordProcessor.d.ts.map +1 -0
- package/dist/processors/AgentRecordProcessor.js +51 -0
- package/dist/processors/AgentRecordProcessor.js.map +1 -0
- package/dist/processors/FieldRulesProcessor.d.ts +44 -0
- package/dist/processors/FieldRulesProcessor.d.ts.map +1 -0
- package/dist/processors/FieldRulesProcessor.js +130 -0
- package/dist/processors/FieldRulesProcessor.js.map +1 -0
- package/dist/processors/FunctionRecordProcessor.d.ts +16 -0
- package/dist/processors/FunctionRecordProcessor.d.ts.map +1 -0
- package/dist/processors/FunctionRecordProcessor.js +16 -0
- package/dist/processors/FunctionRecordProcessor.js.map +1 -0
- package/dist/processors/InferProcessor.d.ts +24 -0
- package/dist/processors/InferProcessor.d.ts.map +1 -0
- package/dist/processors/InferProcessor.js +60 -0
- package/dist/processors/InferProcessor.js.map +1 -0
- package/dist/processors/WriteBackProcessor.d.ts +20 -0
- package/dist/processors/WriteBackProcessor.d.ts.map +1 -0
- package/dist/processors/WriteBackProcessor.js +41 -0
- package/dist/processors/WriteBackProcessor.js.map +1 -0
- package/dist/trackers/GenericProcessRunTracker.d.ts +24 -0
- package/dist/trackers/GenericProcessRunTracker.d.ts.map +1 -0
- package/dist/trackers/GenericProcessRunTracker.js +133 -0
- package/dist/trackers/GenericProcessRunTracker.js.map +1 -0
- package/dist/trackers/NoOpTracker.d.ts +15 -0
- package/dist/trackers/NoOpTracker.d.ts.map +1 -0
- package/dist/trackers/NoOpTracker.js +24 -0
- package/dist/trackers/NoOpTracker.js.map +1 -0
- package/dist/writeBack.d.ts +42 -0
- package/dist/writeBack.d.ts.map +1 -0
- package/dist/writeBack.js +59 -0
- package/dist/writeBack.js.map +1 -0
- package/package.json +39 -7
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview The default persistence tracker — writes `MJ: Process Runs` (the run header) and
|
|
3
|
+
* `MJ: Process Run Details` (one row per processed record), and implements the pause/cancel
|
|
4
|
+
* handshake by re-reading the run row's `CancellationRequested` flag at each checkpoint.
|
|
5
|
+
* @module @memberjunction/record-set-processor
|
|
6
|
+
*/
|
|
7
|
+
import { BaseEntitySaveQueue, LogError, Metadata } from '@memberjunction/core';
|
|
8
|
+
/** The default tracker, persisting to the generic `MJ: Process Runs` / `MJ: Process Run Details` tables. */
|
|
9
|
+
export class GenericProcessRunTracker {
|
|
10
|
+
constructor() {
|
|
11
|
+
/**
|
|
12
|
+
* Fire-and-forget queue for per-record detail INSERTs. Each detail is its own entity instance (a
|
|
13
|
+
* distinct key), so detail writes run concurrently without blocking the per-record loop; `CompleteRun`
|
|
14
|
+
* flushes them before finalizing the run. (Reliable/insert-before-fire mode is a future enhancement —
|
|
15
|
+
* it needs a per-record start hook on `IProcessRunTracker`.)
|
|
16
|
+
*/
|
|
17
|
+
this.detailQueue = new BaseEntitySaveQueue();
|
|
18
|
+
}
|
|
19
|
+
async BeginRun(meta, contextUser, provider) {
|
|
20
|
+
const md = provider ?? Metadata.Provider;
|
|
21
|
+
const run = await md.GetEntityObject('MJ: Process Runs', contextUser);
|
|
22
|
+
run.NewRecord();
|
|
23
|
+
run.RecordProcessID = meta.RecordProcessID ?? null;
|
|
24
|
+
run.ScheduledJobRunID = meta.ScheduledJobRunID ?? null;
|
|
25
|
+
run.EntityID = meta.EntityID ?? null;
|
|
26
|
+
run.TriggeredBy = meta.TriggeredBy;
|
|
27
|
+
run.SourceType = meta.SourceType;
|
|
28
|
+
run.SourceID = meta.SourceID ?? null;
|
|
29
|
+
run.SourceFilter = meta.SourceFilter ?? null;
|
|
30
|
+
run.Status = 'Running';
|
|
31
|
+
run.StartTime = new Date();
|
|
32
|
+
run.TotalItemCount = meta.TotalItemCount ?? null;
|
|
33
|
+
run.ProcessedItems = 0;
|
|
34
|
+
run.SuccessCount = 0;
|
|
35
|
+
run.ErrorCount = 0;
|
|
36
|
+
run.SkippedCount = 0;
|
|
37
|
+
run.BatchSize = meta.BatchSize ?? null;
|
|
38
|
+
run.CancellationRequested = false;
|
|
39
|
+
run.DryRun = meta.DryRun ?? false;
|
|
40
|
+
run.StartedByUserID = contextUser?.ID ?? null;
|
|
41
|
+
if (meta.Configuration !== undefined) {
|
|
42
|
+
run.Configuration = JSON.stringify(meta.Configuration);
|
|
43
|
+
}
|
|
44
|
+
const saved = await run.Save();
|
|
45
|
+
if (!saved) {
|
|
46
|
+
throw new Error(`GenericProcessRunTracker: failed to create Process Run: ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
47
|
+
}
|
|
48
|
+
return { ProcessRunID: run.ID, run };
|
|
49
|
+
}
|
|
50
|
+
async RecordResult(handle, record, result, contextUser, provider) {
|
|
51
|
+
const md = provider ?? Metadata.Provider;
|
|
52
|
+
const run = handle.run;
|
|
53
|
+
const detail = await md.GetEntityObject('MJ: Process Run Details', contextUser);
|
|
54
|
+
detail.NewRecord();
|
|
55
|
+
detail.ProcessRunID = run.ID;
|
|
56
|
+
detail.EntityID = record.EntityID;
|
|
57
|
+
detail.RecordID = record.RecordID;
|
|
58
|
+
detail.Status = result.Status;
|
|
59
|
+
detail.CompletedAt = new Date();
|
|
60
|
+
detail.StartedAt = result.DurationMs != null ? new Date(Date.now() - result.DurationMs) : null;
|
|
61
|
+
detail.DurationMs = result.DurationMs ?? null;
|
|
62
|
+
detail.AttemptCount = result.AttemptCount ?? 1;
|
|
63
|
+
detail.ResultPayload = result.ResultPayload !== undefined ? JSON.stringify(result.ResultPayload) : null;
|
|
64
|
+
detail.ErrorMessage = result.ErrorMessage ?? null;
|
|
65
|
+
detail.ActionExecutionLogID = result.ActionExecutionLogID ?? null;
|
|
66
|
+
detail.AIAgentRunID = result.AIAgentRunID ?? null;
|
|
67
|
+
// Fire-and-forget — the detail write must not block the per-record loop, and a failed write must
|
|
68
|
+
// not abort the run (the queue logs failures; CompleteRun flushes + reports the count).
|
|
69
|
+
this.detailQueue.Insert(detail);
|
|
70
|
+
}
|
|
71
|
+
async Checkpoint(handle, cursor, counts, _contextUser, _provider) {
|
|
72
|
+
const run = handle.run;
|
|
73
|
+
run.ProcessedItems = counts.Processed;
|
|
74
|
+
run.SuccessCount = counts.Success;
|
|
75
|
+
run.ErrorCount = counts.Error;
|
|
76
|
+
run.SkippedCount = counts.Skipped;
|
|
77
|
+
if (cursor.Offset != null) {
|
|
78
|
+
run.LastProcessedOffset = cursor.Offset;
|
|
79
|
+
}
|
|
80
|
+
if (cursor.Key != null) {
|
|
81
|
+
run.LastProcessedKey = cursor.Key;
|
|
82
|
+
}
|
|
83
|
+
const saved = await run.Save();
|
|
84
|
+
if (!saved) {
|
|
85
|
+
LogError(`GenericProcessRunTracker: checkpoint save failed for run '${run.ID}': ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
86
|
+
return true; // a transient checkpoint failure should not halt processing
|
|
87
|
+
}
|
|
88
|
+
// Re-read to pick up an externally-requested pause/cancel.
|
|
89
|
+
await run.Load(run.ID);
|
|
90
|
+
if (run.CancellationRequested) {
|
|
91
|
+
run.Status = 'Paused';
|
|
92
|
+
run.EndTime = new Date();
|
|
93
|
+
await run.Save();
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
return true;
|
|
97
|
+
}
|
|
98
|
+
async CompleteRun(handle, summary, _contextUser, _provider) {
|
|
99
|
+
// Ensure every fire-and-forget detail write has landed before we finalize the run header.
|
|
100
|
+
const flush = await this.detailQueue.Flush();
|
|
101
|
+
if (flush.failures > 0) {
|
|
102
|
+
LogError(`GenericProcessRunTracker: ${flush.failures} Process Run Detail write(s) failed for run '${handle.run.ID}'`);
|
|
103
|
+
}
|
|
104
|
+
const run = handle.run;
|
|
105
|
+
run.Status = summary.Status;
|
|
106
|
+
run.EndTime = summary.EndTime ?? new Date();
|
|
107
|
+
run.ProcessedItems = summary.Processed;
|
|
108
|
+
run.SuccessCount = summary.Success;
|
|
109
|
+
run.ErrorCount = summary.Error;
|
|
110
|
+
run.SkippedCount = summary.Skipped;
|
|
111
|
+
if (summary.Total != null) {
|
|
112
|
+
run.TotalItemCount = summary.Total;
|
|
113
|
+
}
|
|
114
|
+
if (summary.ErrorMessage) {
|
|
115
|
+
run.ErrorMessage = summary.ErrorMessage;
|
|
116
|
+
}
|
|
117
|
+
const saved = await run.Save();
|
|
118
|
+
if (!saved) {
|
|
119
|
+
LogError(`GenericProcessRunTracker: failed to complete run '${run.ID}': ${run.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
async LoadResumeCursor(handle, _contextUser, _provider) {
|
|
123
|
+
const run = handle.run;
|
|
124
|
+
if (run.LastProcessedKey != null) {
|
|
125
|
+
return { Key: run.LastProcessedKey };
|
|
126
|
+
}
|
|
127
|
+
if (run.LastProcessedOffset != null && run.LastProcessedOffset > 0) {
|
|
128
|
+
return { Offset: run.LastProcessedOffset };
|
|
129
|
+
}
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
//# sourceMappingURL=GenericProcessRunTracker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"GenericProcessRunTracker.js","sourceRoot":"","sources":["../../src/trackers/GenericProcessRunTracker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,mBAAmB,EAAqB,QAAQ,EAAE,QAAQ,EAAY,MAAM,sBAAsB,CAAC;AAkB5G,4GAA4G;AAC5G,MAAM,OAAO,wBAAwB;IAArC;QACI;;;;;WAKG;QACc,gBAAW,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAwH7D,CAAC;IAtHU,KAAK,CAAC,QAAQ,CAAC,IAAoB,EAAE,WAAqB,EAAE,QAA4B;QAC3F,MAAM,EAAE,GAAG,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC;QACzC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,eAAe,CAAqB,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAC1F,GAAG,CAAC,SAAS,EAAE,CAAC;QAChB,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC;QACnD,GAAG,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC;QACvD,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;QACrC,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACnC,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACjC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;QACrC,GAAG,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;QAC7C,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,GAAG,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAC3B,GAAG,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC;QACjD,GAAG,CAAC,cAAc,GAAG,CAAC,CAAC;QACvB,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC;QACrB,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC;QACnB,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC;QACrB,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC;QACvC,GAAG,CAAC,qBAAqB,GAAG,KAAK,CAAC;QAClC,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC;QAClC,GAAG,CAAC,eAAe,GAAG,WAAW,EAAE,EAAE,IAAI,IAAI,CAAC;QAC9C,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YACnC,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,2DAA2D,GAAG,CAAC,YAAY,EAAE,eAAe,IAAI,eAAe,EAAE,CAAC,CAAC;QACvI,CAAC;QACD,OAAO,EAAE,YAAY,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAsB,CAAC;IAC7D,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,MAAiB,EAAE,MAAiB,EAAE,MAAoB,EAAE,WAAqB,EAAE,QAA4B;QACrI,MAAM,EAAE,GAAG,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC;QACzC,MAAM,GAAG,GAAI,MAA2B,CAAC,GAAG,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,eAAe,CAA2B,yBAAyB,EAAE,WAAW,CAAC,CAAC;QAC1G,MAAM,CAAC,SAAS,EAAE,CAAC;QACnB,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAClC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAClC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC9B,MAAM,CAAC,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;QAChC,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/F,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC;QAC9C,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC;QAC/C,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxG,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC;QAClD,MAAM,CAAC,oBAAoB,GAAG,MAAM,CAAC,oBAAoB,IAAI,IAAI,CAAC;QAClE,MAAM,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,IAAI,CAAC;QAClD,iGAAiG;QACjG,wFAAwF;QACxF,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEM,KAAK,CAAC,UAAU,CAAC,MAAiB,EAAE,MAAqB,EAAE,MAAiB,EAAE,YAAsB,EAAE,SAA6B;QACtI,MAAM,GAAG,GAAI,MAA2B,CAAC,GAAG,CAAC;QAC7C,GAAG,CAAC,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC;QACtC,GAAG,CAAC,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC;QAClC,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC;QAC9B,GAAG,CAAC,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC;QAClC,IAAI,MAAM,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;YACxB,GAAG,CAAC,mBAAmB,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5C,CAAC;QACD,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,GAAG,CAAC,gBAAgB,GAAG,MAAM,CAAC,GAAG,CAAC;QACtC,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,QAAQ,CAAC,6DAA6D,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,YAAY,EAAE,eAAe,IAAI,eAAe,EAAE,CAAC,CAAC;YAC1I,OAAO,IAAI,CAAC,CAAC,4DAA4D;QAC7E,CAAC;QACD,2DAA2D;QAC3D,MAAM,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,GAAG,CAAC,qBAAqB,EAAE,CAAC;YAC5B,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC;YACtB,GAAG,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,KAAK,CAAC,WAAW,CAAC,MAAiB,EAAE,OAA0B,EAAE,YAAsB,EAAE,SAA6B;QACzH,0FAA0F;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAC7C,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YACrB,QAAQ,CAAC,6BAA6B,KAAK,CAAC,QAAQ,gDAAiD,MAA2B,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;QAChJ,CAAC;QAED,MAAM,GAAG,GAAI,MAA2B,CAAC,GAAG,CAAC;QAC7C,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC5B,GAAG,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,IAAI,EAAE,CAAC;QAC5C,GAAG,CAAC,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC;QACvC,GAAG,CAAC,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC;QACnC,GAAG,CAAC,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC;QAC/B,GAAG,CAAC,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC;QACnC,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;YACxB,GAAG,CAAC,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC;QACvC,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACvB,GAAG,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QAC5C,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,QAAQ,CAAC,qDAAqD,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,YAAY,EAAE,eAAe,IAAI,eAAe,EAAE,CAAC,CAAC;QACtI,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAAC,MAAiB,EAAE,YAAsB,EAAE,SAA6B;QAClG,MAAM,GAAG,GAAI,MAA2B,CAAC,GAAG,CAAC;QAC7C,IAAI,GAAG,CAAC,gBAAgB,IAAI,IAAI,EAAE,CAAC;YAC/B,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACzC,CAAC;QACD,IAAI,GAAG,CAAC,mBAAmB,IAAI,IAAI,IAAI,GAAG,CAAC,mBAAmB,GAAG,CAAC,EAAE,CAAC;YACjE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,mBAAmB,EAAE,CAAC;QAC/C,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview A tracker that persists nothing — for fire-and-forget single-record work where a
|
|
3
|
+
* full run record would be overkill.
|
|
4
|
+
* @module @memberjunction/record-set-processor
|
|
5
|
+
*/
|
|
6
|
+
import { IProcessRunTracker, ProcessCursor, RunHandle } from '@memberjunction/record-set-processor-base';
|
|
7
|
+
/** An {@link IProcessRunTracker} that discards all run/detail tracking. */
|
|
8
|
+
export declare class NoOpTracker implements IProcessRunTracker {
|
|
9
|
+
BeginRun(): Promise<RunHandle>;
|
|
10
|
+
RecordResult(): Promise<void>;
|
|
11
|
+
Checkpoint(): Promise<boolean>;
|
|
12
|
+
CompleteRun(): Promise<void>;
|
|
13
|
+
LoadResumeCursor(): Promise<ProcessCursor | undefined>;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=NoOpTracker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"NoOpTracker.d.ts","sourceRoot":"","sources":["../../src/trackers/NoOpTracker.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACH,kBAAkB,EAClB,aAAa,EACb,SAAS,EACZ,MAAM,2CAA2C,CAAC;AAEnD,2EAA2E;AAC3E,qBAAa,WAAY,YAAW,kBAAkB;IACrC,QAAQ,IAAI,OAAO,CAAC,SAAS,CAAC;IAG9B,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAG7B,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAG9B,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAG5B,gBAAgB,IAAI,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;CAGtE"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview A tracker that persists nothing — for fire-and-forget single-record work where a
|
|
3
|
+
* full run record would be overkill.
|
|
4
|
+
* @module @memberjunction/record-set-processor
|
|
5
|
+
*/
|
|
6
|
+
/** An {@link IProcessRunTracker} that discards all run/detail tracking. */
|
|
7
|
+
export class NoOpTracker {
|
|
8
|
+
async BeginRun() {
|
|
9
|
+
return {};
|
|
10
|
+
}
|
|
11
|
+
async RecordResult() {
|
|
12
|
+
// intentionally empty
|
|
13
|
+
}
|
|
14
|
+
async Checkpoint() {
|
|
15
|
+
return true;
|
|
16
|
+
}
|
|
17
|
+
async CompleteRun() {
|
|
18
|
+
// intentionally empty
|
|
19
|
+
}
|
|
20
|
+
async LoadResumeCursor() {
|
|
21
|
+
return undefined;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=NoOpTracker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"NoOpTracker.js","sourceRoot":"","sources":["../../src/trackers/NoOpTracker.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAQH,2EAA2E;AAC3E,MAAM,OAAO,WAAW;IACb,KAAK,CAAC,QAAQ;QACjB,OAAO,EAAE,CAAC;IACd,CAAC;IACM,KAAK,CAAC,YAAY;QACrB,sBAAsB;IAC1B,CAAC;IACM,KAAK,CAAC,UAAU;QACnB,OAAO,IAAI,CAAC;IAChB,CAAC;IACM,KAAK,CAAC,WAAW;QACpB,sBAAsB;IAC1B,CAAC;IACM,KAAK,CAAC,gBAAgB;QACzB,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Output-mapping write-back — applies a work's structured result back onto the data
|
|
3
|
+
* model per an `OutputMapping` config: update fields on the processed record and/or create a child
|
|
4
|
+
* record. Reads from the result use the generic value-mapping resolver (`$` = the result root).
|
|
5
|
+
* @module @memberjunction/record-set-processor
|
|
6
|
+
*/
|
|
7
|
+
import { IMetadataProvider, UserInfo } from '@memberjunction/core';
|
|
8
|
+
import { RecordRef } from '@memberjunction/record-set-processor-base';
|
|
9
|
+
/** Declarative description of how a work's structured output is written back. */
|
|
10
|
+
export interface OutputMappingConfig {
|
|
11
|
+
/** Map of `EntityFieldName -> resultRef` (e.g. `{ "Satisfaction": "$.satisfaction" }`) applied to the processed record. */
|
|
12
|
+
fields?: Record<string, string>;
|
|
13
|
+
/** Optional child record to create from the result. */
|
|
14
|
+
childRecord?: {
|
|
15
|
+
/** Target child entity name. */
|
|
16
|
+
entity: string;
|
|
17
|
+
/** FK field on the child set to the processed record's primary key. */
|
|
18
|
+
parentField: string;
|
|
19
|
+
/** Map of `ChildFieldName -> resultRef`. */
|
|
20
|
+
map: Record<string, string>;
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
/** Outcome of {@link applyOutputMapping}. */
|
|
24
|
+
export interface WriteBackResult {
|
|
25
|
+
/** True if the processed record's fields were updated. */
|
|
26
|
+
updatedRecord: boolean;
|
|
27
|
+
/** ID of the created child record, when a child mapping was applied. */
|
|
28
|
+
createdChildID?: string;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Applies an {@link OutputMappingConfig} against a work result. Single-primary-key entities only
|
|
32
|
+
* (composite keys throw a clear error). Field/child values are resolved from the result via the
|
|
33
|
+
* shared resolver, so `"$.path"` reads `result.path`.
|
|
34
|
+
*/
|
|
35
|
+
export declare function applyOutputMapping(opts: {
|
|
36
|
+
outputMapping: OutputMappingConfig;
|
|
37
|
+
result: unknown;
|
|
38
|
+
record: RecordRef;
|
|
39
|
+
contextUser: UserInfo;
|
|
40
|
+
provider?: IMetadataProvider;
|
|
41
|
+
}): Promise<WriteBackResult>;
|
|
42
|
+
//# sourceMappingURL=writeBack.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"writeBack.d.ts","sourceRoot":"","sources":["../src/writeBack.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAA4B,iBAAiB,EAAY,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAEvG,OAAO,EAAE,SAAS,EAAE,MAAM,2CAA2C,CAAC;AAEtE,iFAAiF;AACjF,MAAM,WAAW,mBAAmB;IAChC,2HAA2H;IAC3H,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,uDAAuD;IACvD,WAAW,CAAC,EAAE;QACV,gCAAgC;QAChC,MAAM,EAAE,MAAM,CAAC;QACf,uEAAuE;QACvE,WAAW,EAAE,MAAM,CAAC;QACpB,4CAA4C;QAC5C,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC/B,CAAC;CACL;AAED,6CAA6C;AAC7C,MAAM,WAAW,eAAe;IAC5B,0DAA0D;IAC1D,aAAa,EAAE,OAAO,CAAC;IACvB,wEAAwE;IACxE,cAAc,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC3C,aAAa,EAAE,mBAAmB,CAAC;IACnC,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,WAAW,EAAE,QAAQ,CAAC;IACtB,QAAQ,CAAC,EAAE,iBAAiB,CAAC;CAChC,GAAG,OAAO,CAAC,eAAe,CAAC,CA+C3B"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Output-mapping write-back — applies a work's structured result back onto the data
|
|
3
|
+
* model per an `OutputMapping` config: update fields on the processed record and/or create a child
|
|
4
|
+
* record. Reads from the result use the generic value-mapping resolver (`$` = the result root).
|
|
5
|
+
* @module @memberjunction/record-set-processor
|
|
6
|
+
*/
|
|
7
|
+
import { CompositeKey, Metadata } from '@memberjunction/core';
|
|
8
|
+
import { resolveMappingRef } from '@memberjunction/global';
|
|
9
|
+
/**
|
|
10
|
+
* Applies an {@link OutputMappingConfig} against a work result. Single-primary-key entities only
|
|
11
|
+
* (composite keys throw a clear error). Field/child values are resolved from the result via the
|
|
12
|
+
* shared resolver, so `"$.path"` reads `result.path`.
|
|
13
|
+
*/
|
|
14
|
+
export async function applyOutputMapping(opts) {
|
|
15
|
+
const { outputMapping, result, record, contextUser } = opts;
|
|
16
|
+
const provider = opts.provider ?? Metadata.Provider;
|
|
17
|
+
const sources = { $: result, record: record.Record ?? {} };
|
|
18
|
+
const out = { updatedRecord: false };
|
|
19
|
+
// 1) Update fields on the processed record.
|
|
20
|
+
if (outputMapping.fields && Object.keys(outputMapping.fields).length > 0) {
|
|
21
|
+
const entity = provider.EntityByID(record.EntityID);
|
|
22
|
+
if (!entity) {
|
|
23
|
+
throw new Error(`applyOutputMapping: entity '${record.EntityID}' not found in metadata`);
|
|
24
|
+
}
|
|
25
|
+
if (entity.PrimaryKeys.length !== 1) {
|
|
26
|
+
throw new Error(`applyOutputMapping: write-back currently supports single-primary-key entities only ('${entity.Name}')`);
|
|
27
|
+
}
|
|
28
|
+
const obj = await provider.GetEntityObject(entity.Name, contextUser);
|
|
29
|
+
const loaded = await obj.InnerLoad(CompositeKey.FromKeyValuePair(entity.FirstPrimaryKey.Name, record.RecordID));
|
|
30
|
+
if (!loaded) {
|
|
31
|
+
throw new Error(`applyOutputMapping: record '${record.RecordID}' of '${entity.Name}' not found`);
|
|
32
|
+
}
|
|
33
|
+
for (const [field, ref] of Object.entries(outputMapping.fields)) {
|
|
34
|
+
// Dynamic, config-driven field names — the legitimate use of Set() (no compile-time property).
|
|
35
|
+
obj.Set(field, resolveMappingRef(ref, sources));
|
|
36
|
+
}
|
|
37
|
+
const saved = await obj.Save();
|
|
38
|
+
if (!saved) {
|
|
39
|
+
throw new Error(`applyOutputMapping: failed updating '${entity.Name}' record '${record.RecordID}': ${obj.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
40
|
+
}
|
|
41
|
+
out.updatedRecord = true;
|
|
42
|
+
}
|
|
43
|
+
// 2) Create a child record.
|
|
44
|
+
if (outputMapping.childRecord) {
|
|
45
|
+
const child = await provider.GetEntityObject(outputMapping.childRecord.entity, contextUser);
|
|
46
|
+
child.NewRecord();
|
|
47
|
+
child.Set(outputMapping.childRecord.parentField, record.RecordID);
|
|
48
|
+
for (const [field, ref] of Object.entries(outputMapping.childRecord.map)) {
|
|
49
|
+
child.Set(field, resolveMappingRef(ref, sources));
|
|
50
|
+
}
|
|
51
|
+
const saved = await child.Save();
|
|
52
|
+
if (!saved) {
|
|
53
|
+
throw new Error(`applyOutputMapping: failed creating '${outputMapping.childRecord.entity}' child: ${child.LatestResult?.CompleteMessage ?? 'unknown error'}`);
|
|
54
|
+
}
|
|
55
|
+
out.createdChildID = child.FirstPrimaryKey?.Value != null ? String(child.FirstPrimaryKey.Value) : undefined;
|
|
56
|
+
}
|
|
57
|
+
return out;
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=writeBack.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"writeBack.js","sourceRoot":"","sources":["../src/writeBack.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAc,YAAY,EAAqB,QAAQ,EAAY,MAAM,sBAAsB,CAAC;AACvG,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AA0B3D;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAMxC;IACG,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;IAC5D,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC;IACpD,MAAM,OAAO,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;IAC3D,MAAM,GAAG,GAAoB,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;IAEtD,4CAA4C;IAC5C,IAAI,aAAa,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvE,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,CAAC,QAAQ,yBAAyB,CAAC,CAAC;QAC7F,CAAC;QACD,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,wFAAwF,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;QAC7H,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAa,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACjF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAChH,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,CAAC,QAAQ,SAAS,MAAM,CAAC,IAAI,aAAa,CAAC,CAAC;QACrG,CAAC;QACD,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9D,+FAA+F;YAC/F,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,wCAAwC,MAAM,CAAC,IAAI,aAAa,MAAM,CAAC,QAAQ,MAAM,GAAG,CAAC,YAAY,EAAE,eAAe,IAAI,eAAe,EAAE,CAAC,CAAC;QACjK,CAAC;QACD,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC;IAC7B,CAAC;IAED,4BAA4B;IAC5B,IAAI,aAAa,CAAC,WAAW,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAa,aAAa,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACxG,KAAK,CAAC,SAAS,EAAE,CAAC;QAClB,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,WAAW,CAAC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QAClE,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACvE,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,iBAAiB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,wCAAwC,aAAa,CAAC,WAAW,CAAC,MAAM,YAAY,KAAK,CAAC,YAAY,EAAE,eAAe,IAAI,eAAe,EAAE,CAAC,CAAC;QAClK,CAAC;QACD,GAAG,CAAC,cAAc,GAAG,KAAK,CAAC,eAAe,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAChH,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,10 +1,42 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@memberjunction/record-set-processor",
|
|
3
|
-
"
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "5.43.0",
|
|
5
|
+
"description": "MemberJunction: Record Set Processor - server engine that batches, rate-limits, resumes, and audits processing of a record set through a pluggable source/processor/tracker pipeline.",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"files": [
|
|
9
|
+
"/dist"
|
|
10
|
+
],
|
|
11
|
+
"scripts": {
|
|
12
|
+
"build": "tsc && tsc-alias -f",
|
|
13
|
+
"watch": "tsc --watch",
|
|
14
|
+
"test": "vitest run",
|
|
15
|
+
"test:watch": "vitest"
|
|
16
|
+
},
|
|
17
|
+
"author": "MemberJunction.com",
|
|
18
|
+
"license": "ISC",
|
|
19
|
+
"dependencies": {
|
|
20
|
+
"@memberjunction/core": "5.43.0",
|
|
21
|
+
"@memberjunction/global": "5.43.0",
|
|
22
|
+
"@memberjunction/core-entities": "5.43.0",
|
|
23
|
+
"@memberjunction/field-rules-transforms": "5.43.0",
|
|
24
|
+
"@memberjunction/record-set-processor-base": "5.43.0",
|
|
25
|
+
"@memberjunction/actions": "5.43.0",
|
|
26
|
+
"@memberjunction/actions-base": "5.43.0",
|
|
27
|
+
"@memberjunction/ai-agents": "5.43.0",
|
|
28
|
+
"@memberjunction/aiengine": "5.43.0",
|
|
29
|
+
"@memberjunction/ai-prompts": "5.43.0",
|
|
30
|
+
"@memberjunction/ai-core-plus": "5.43.0",
|
|
31
|
+
"@memberjunction/templates": "5.43.0"
|
|
32
|
+
},
|
|
33
|
+
"devDependencies": {
|
|
34
|
+
"@types/node": "24.10.11",
|
|
35
|
+
"typescript": "^5.9.3",
|
|
36
|
+
"vitest": "^3.1.1"
|
|
37
|
+
},
|
|
38
|
+
"repository": {
|
|
39
|
+
"type": "git",
|
|
40
|
+
"url": "https://github.com/MemberJunction/MJ"
|
|
41
|
+
}
|
|
10
42
|
}
|