@reaatech/media-pipeline-mcp-persistence 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +247 -0
- package/dist/index.cjs +423 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +255 -0
- package/dist/index.d.ts +255 -0
- package/dist/index.js +399 -0
- package/dist/index.js.map +1 -0
- package/package.json +57 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Media Pipeline MCP Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
# @reaatech/media-pipeline-mcp-persistence
|
|
2
|
+
|
|
3
|
+
[](https://www.npmjs.com/package/@reaatech/media-pipeline-mcp-persistence)
|
|
4
|
+
[](https://github.com/reaatech/media-pipeline-mcp/blob/main/LICENSE)
|
|
5
|
+
[](https://github.com/reaatech/media-pipeline-mcp/actions/workflows/ci.yml)
|
|
6
|
+
|
|
7
|
+
> **Status:** Pre-1.0 — APIs may change in minor versions. Pin to a specific version in production.
|
|
8
|
+
|
|
9
|
+
Pipeline state store abstractions with in-memory and Redis implementations. Provides optimistic locking, run lifecycle management, event log persistence, and tenant-scoped queries for multi-tenant pipeline orchestration.
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
npm install @reaatech/media-pipeline-mcp-persistence
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pnpm add @reaatech/media-pipeline-mcp-persistence
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
The Redis backend requires the optional `ioredis` peer dependency:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pnpm add ioredis
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Feature Overview
|
|
28
|
+
|
|
29
|
+
- In-memory store for development and single-process deployments
|
|
30
|
+
- Redis-backed store with TTL-managed keys, per-tenant zset indexes, and external job ID mappings
|
|
31
|
+
- Optimistic locking with expected-version conflict detection (`update(runId, patch, expectedVersion)`)
|
|
32
|
+
- Exclusive write lock via `withLock()` — in-memory mutex for single-process, `SET NX EX` for Redis
|
|
33
|
+
- Canonical event taxonomy with 12 event types tracking the full run lifecycle
|
|
34
|
+
- Run filtering by status, tenant, idempotency key, and time window
|
|
35
|
+
- Idempotency key deduplication preventing duplicate run creation
|
|
36
|
+
- External job ID lookup for webhook-based provider integrations
|
|
37
|
+
|
|
38
|
+
## Quick Start
|
|
39
|
+
|
|
40
|
+
```typescript
|
|
41
|
+
import { InMemoryPipelineStateStore } from '@reaatech/media-pipeline-mcp-persistence';
|
|
42
|
+
|
|
43
|
+
const store = new InMemoryPipelineStateStore();
|
|
44
|
+
|
|
45
|
+
// Create a pipeline run
|
|
46
|
+
await store.create({
|
|
47
|
+
runId: 'run-42',
|
|
48
|
+
pipelineId: 'product-photo',
|
|
49
|
+
status: 'pending',
|
|
50
|
+
tenantId: 'acme',
|
|
51
|
+
pipelineDefHash: 'sha256-abc123',
|
|
52
|
+
currentStepIndex: 0,
|
|
53
|
+
steps: [
|
|
54
|
+
{
|
|
55
|
+
stepId: 'gen',
|
|
56
|
+
operation: 'image.generate',
|
|
57
|
+
inputs: { prompt: 'A product photo of a sneaker' },
|
|
58
|
+
status: 'pending',
|
|
59
|
+
attempts: 0,
|
|
60
|
+
maxRetries: 2,
|
|
61
|
+
artifactIds: [],
|
|
62
|
+
costUsd: 0,
|
|
63
|
+
},
|
|
64
|
+
],
|
|
65
|
+
events: [],
|
|
66
|
+
externalJobIds: {},
|
|
67
|
+
version: 1,
|
|
68
|
+
createdAt: new Date().toISOString(),
|
|
69
|
+
updatedAt: new Date().toISOString(),
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
// Append events as the run progresses
|
|
73
|
+
await store.appendEvent('run-42', {
|
|
74
|
+
kind: 'run-started',
|
|
75
|
+
runId: 'run-42',
|
|
76
|
+
at: Date.now(),
|
|
77
|
+
});
|
|
78
|
+
await store.appendEvent('run-42', {
|
|
79
|
+
kind: 'step-started',
|
|
80
|
+
runId: 'run-42',
|
|
81
|
+
stepId: 'gen',
|
|
82
|
+
at: Date.now(),
|
|
83
|
+
attempt: 1,
|
|
84
|
+
});
|
|
85
|
+
await store.appendEvent('run-42', {
|
|
86
|
+
kind: 'step-completed',
|
|
87
|
+
runId: 'run-42',
|
|
88
|
+
stepId: 'gen',
|
|
89
|
+
at: Date.now(),
|
|
90
|
+
artifactIds: ['artifact-99'],
|
|
91
|
+
costUsd: 0.014,
|
|
92
|
+
});
|
|
93
|
+
await store.appendEvent('run-42', {
|
|
94
|
+
kind: 'run-completed',
|
|
95
|
+
runId: 'run-42',
|
|
96
|
+
at: Date.now(),
|
|
97
|
+
totalCostUsd: 0.014,
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
// Update with optimistic locking
|
|
101
|
+
await store.update(
|
|
102
|
+
'run-42',
|
|
103
|
+
{ status: 'completed', completedAt: new Date().toISOString() },
|
|
104
|
+
1, // expectedVersion — fails if another writer updated first
|
|
105
|
+
);
|
|
106
|
+
|
|
107
|
+
// Replay events (for reconnect/resume)
|
|
108
|
+
const events = await store.listEvents('run-42', /* sinceSeq */ 0);
|
|
109
|
+
|
|
110
|
+
// Query runs by tenant
|
|
111
|
+
const runs = await store.listRuns({
|
|
112
|
+
tenantId: 'acme',
|
|
113
|
+
status: 'completed',
|
|
114
|
+
since: '2026-01-01T00:00:00Z',
|
|
115
|
+
limit: 10,
|
|
116
|
+
offset: 0,
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
// Cancel a run
|
|
120
|
+
await store.cancel('run-42', 'User requested cancellation');
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
### Redis
|
|
124
|
+
|
|
125
|
+
```typescript
|
|
126
|
+
import { Redis } from 'ioredis';
|
|
127
|
+
import { RedisPipelineStateStore } from '@reaatech/media-pipeline-mcp-persistence';
|
|
128
|
+
|
|
129
|
+
const redis = new Redis();
|
|
130
|
+
const store = new RedisPipelineStateStore({
|
|
131
|
+
client: redis,
|
|
132
|
+
prefix: 'mp',
|
|
133
|
+
runTtlSeconds: 30 * 86400, // 30 days
|
|
134
|
+
lockTtlSeconds: 60, // 60s lock TTL
|
|
135
|
+
idempotencyTtlSeconds: 86400, // 24h
|
|
136
|
+
externalJobTtlSeconds: 7 * 86400, // 7 days
|
|
137
|
+
lockAcquireTimeoutMs: 5000, // 5s max wait for lock
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
// Locked mutation
|
|
141
|
+
const result = await store.withLock('run-42', async (run) => {
|
|
142
|
+
// Safe to mutate — exclusive lock held
|
|
143
|
+
return performCriticalUpdate(run);
|
|
144
|
+
}, /* timeoutMs */ 10_000);
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
## API Reference
|
|
148
|
+
|
|
149
|
+
### Types
|
|
150
|
+
|
|
151
|
+
| Type | Description |
|
|
152
|
+
|------|-------------|
|
|
153
|
+
| `PipelineRunStatus` | `'pending' \| 'running' \| 'suspended' \| 'completed' \| 'failed' \| 'cancelled'` |
|
|
154
|
+
| `StepStatus` | `'pending' \| 'running' \| 'completed' \| 'failed' \| 'gated' \| 'cached' \| 'cancelled'` |
|
|
155
|
+
| `StepState` | Per-step state: operation, status, attempts, artifact IDs, cost, error, timestamps, cache key |
|
|
156
|
+
| `PipelineRun` | Full run record: status, steps, events, version, timestamps, idempotency key, external job IDs |
|
|
157
|
+
| `PipelineEvent` | Discriminated union of 12 event types (`run-created`, `run-started`, `step-started`, `step-progress`, `step-cached`, `step-completed`, `step-failed`, `step-gated`, `run-suspended`, `run-resumed`, `run-completed`, `run-failed`) |
|
|
158
|
+
| `RunFilter` | Query filter: status (single or array), tenantId, idempotencyKey, since/until time window, limit/offset |
|
|
159
|
+
| `PipelineStateStore` | Interface: `create()`, `get()`, `update()`, `cancel()`, `appendEvent()`, `listEvents()`, `listRuns()`, `findByExternalJobId()`, `withLock()` |
|
|
160
|
+
|
|
161
|
+
### Classes
|
|
162
|
+
|
|
163
|
+
| Class | Description |
|
|
164
|
+
|-------|-------------|
|
|
165
|
+
| `InMemoryPipelineStateStore` | In-process store with Map-based storage and async-mutex locking. Single-process only. |
|
|
166
|
+
| `RedisPipelineStateStore` | Redis-backed store with TTL-managed keys, tenant zset indexes, and `SET NX EX` distributed locking. |
|
|
167
|
+
|
|
168
|
+
**`RedisPipelineStateStoreConfig`:**
|
|
169
|
+
|
|
170
|
+
| Option | Type | Default | Description |
|
|
171
|
+
|--------|------|---------|-------------|
|
|
172
|
+
| `client` | `RedisClientLike` | _(required)_ | ioredis-compatible client |
|
|
173
|
+
| `prefix` | `string` | `'mp'` | Redis key namespace |
|
|
174
|
+
| `runTtlSeconds` | `number` | `2_592_000` (30d) | TTL for run, events, and tenant index keys |
|
|
175
|
+
| `lockTtlSeconds` | `number` | `60` | Lock TTL (SET NX EX) |
|
|
176
|
+
| `idempotencyTtlSeconds` | `number` | `86_400` (24h) | Idempotency entry TTL |
|
|
177
|
+
| `externalJobTtlSeconds` | `number` | `604_800` (7d) | External job mapping TTL |
|
|
178
|
+
| `lockAcquireTimeoutMs` | `number` | `5_000` | Max time to wait for lock acquisition |
|
|
179
|
+
| `lockPollIntervalMs` | `number` | `100` | Poll interval during lock acquisition |
|
|
180
|
+
|
|
181
|
+
### Event Taxonomy
|
|
182
|
+
|
|
183
|
+
| Event | Description |
|
|
184
|
+
|-------|-------------|
|
|
185
|
+
| `run-created` | Run initialized with pipeline definition hash |
|
|
186
|
+
| `run-started` | Execution has begun |
|
|
187
|
+
| `step-started` | Step execution started (includes attempt counter) |
|
|
188
|
+
| `step-progress` | In-flight progress with percentage, ETA, accrued cost |
|
|
189
|
+
| `step-cached` | Step result retrieved from cache |
|
|
190
|
+
| `step-completed` | Step finished successfully with artifact IDs and cost |
|
|
191
|
+
| `step-failed` | Step failed with error code and retryability flag |
|
|
192
|
+
| `step-gated` | Step blocked by quality gate with gate type and verdict |
|
|
193
|
+
| `run-suspended` | Execution suspended (webhook, budget, or gate) with resume token |
|
|
194
|
+
| `run-resumed` | Execution resumed from a specific step |
|
|
195
|
+
| `run-completed` | Pipeline finished successfully with total cost |
|
|
196
|
+
| `run-failed` | Pipeline terminated with error code and terminal reason |
|
|
197
|
+
|
|
198
|
+
## Usage Patterns
|
|
199
|
+
|
|
200
|
+
### Optimistic Locking
|
|
201
|
+
|
|
202
|
+
```typescript
|
|
203
|
+
async function atomicStepComplete(runId: string, stepId: string, currentVersion: number) {
|
|
204
|
+
await store.update(runId, {
|
|
205
|
+
steps: updatedSteps,
|
|
206
|
+
currentStepIndex: nextIndex,
|
|
207
|
+
}, currentVersion); // Throws RunInProgressError if version mismatch
|
|
208
|
+
}
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
### Idempotent Run Creation
|
|
212
|
+
|
|
213
|
+
```typescript
|
|
214
|
+
async function createIfAbsent(run: PipelineRun) {
|
|
215
|
+
const existing = await store.listRuns({ idempotencyKey: run.idempotencyKey });
|
|
216
|
+
if (existing.length > 0) return existing[0];
|
|
217
|
+
await store.create(run);
|
|
218
|
+
return run;
|
|
219
|
+
}
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
### Webhook Callback Resolution
|
|
223
|
+
|
|
224
|
+
```typescript
|
|
225
|
+
// When a provider webhook reports job completion
|
|
226
|
+
async function onJobComplete(provider: string, jobId: string, result: unknown) {
|
|
227
|
+
const run = await store.findByExternalJobId(provider, jobId);
|
|
228
|
+
if (!run) return; // Stale / unknown callback
|
|
229
|
+
await store.appendEvent(run.runId, {
|
|
230
|
+
kind: 'step-completed',
|
|
231
|
+
runId: run.runId,
|
|
232
|
+
stepId: currentStep(run).stepId,
|
|
233
|
+
at: Date.now(),
|
|
234
|
+
artifactIds: [result.id],
|
|
235
|
+
costUsd: result.cost,
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
## Related Packages
|
|
241
|
+
|
|
242
|
+
- [@reaatech/media-pipeline-mcp-core](https://www.npmjs.com/package/@reaatech/media-pipeline-mcp-core) — `RunNotFoundError`, `RunInProgressError`, `StateStoreUnavailableError`
|
|
243
|
+
- [@reaatech/media-pipeline-mcp](https://www.npmjs.com/package/@reaatech/media-pipeline-mcp) — Full MCP server that consumes this store for pipeline orchestration
|
|
244
|
+
|
|
245
|
+
## License
|
|
246
|
+
|
|
247
|
+
[MIT](https://github.com/reaatech/media-pipeline-mcp/blob/main/LICENSE)
|
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// src/index.ts
|
|
21
|
+
var index_exports = {};
|
|
22
|
+
__export(index_exports, {
|
|
23
|
+
InMemoryPipelineStateStore: () => InMemoryPipelineStateStore,
|
|
24
|
+
RedisPipelineStateStore: () => RedisPipelineStateStore
|
|
25
|
+
});
|
|
26
|
+
module.exports = __toCommonJS(index_exports);
|
|
27
|
+
|
|
28
|
+
// src/in-memory-store.ts
|
|
29
|
+
var import_media_pipeline_mcp_core = require("@reaatech/media-pipeline-mcp-core");
|
|
30
|
+
var DEFAULT_LOCK_TIMEOUT_MS = 5e3;
|
|
31
|
+
var LOCK_POLL_INTERVAL_MS = 25;
|
|
32
|
+
function cloneRun(run) {
|
|
33
|
+
return {
|
|
34
|
+
...run,
|
|
35
|
+
steps: run.steps.map((s) => ({ ...s })),
|
|
36
|
+
events: run.events.map((e) => ({ ...e }))
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function matchesFilter(run, filter) {
|
|
40
|
+
if (!filter) return true;
|
|
41
|
+
if (filter.status) {
|
|
42
|
+
const statuses = Array.isArray(filter.status) ? filter.status : [filter.status];
|
|
43
|
+
if (!statuses.includes(run.status)) return false;
|
|
44
|
+
}
|
|
45
|
+
if (filter.tenantId && run.tenantId !== filter.tenantId) return false;
|
|
46
|
+
if (filter.idempotencyKey && run.idempotencyKey !== filter.idempotencyKey) return false;
|
|
47
|
+
if (filter.since && run.createdAt < filter.since) return false;
|
|
48
|
+
if (filter.until && run.createdAt > filter.until) return false;
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
var InMemoryPipelineStateStore = class {
|
|
52
|
+
runs = /* @__PURE__ */ new Map();
|
|
53
|
+
locks = /* @__PURE__ */ new Map();
|
|
54
|
+
async create(run) {
|
|
55
|
+
if (this.runs.has(run.runId)) {
|
|
56
|
+
throw new import_media_pipeline_mcp_core.RunInProgressError();
|
|
57
|
+
}
|
|
58
|
+
const stored = cloneRun(run);
|
|
59
|
+
this.runs.set(run.runId, stored);
|
|
60
|
+
}
|
|
61
|
+
async get(runId) {
|
|
62
|
+
const run = this.runs.get(runId);
|
|
63
|
+
if (!run) return null;
|
|
64
|
+
return cloneRun(run);
|
|
65
|
+
}
|
|
66
|
+
async update(runId, patch, expectedVersion) {
|
|
67
|
+
const existing = this.runs.get(runId);
|
|
68
|
+
if (!existing) {
|
|
69
|
+
throw new import_media_pipeline_mcp_core.RunNotFoundError();
|
|
70
|
+
}
|
|
71
|
+
if (expectedVersion !== void 0 && existing.version !== expectedVersion) {
|
|
72
|
+
throw new import_media_pipeline_mcp_core.RunInProgressError();
|
|
73
|
+
}
|
|
74
|
+
const stored = cloneRun(existing);
|
|
75
|
+
Object.assign(stored, patch);
|
|
76
|
+
stored.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
77
|
+
stored.version = (stored.version ?? 0) + 1;
|
|
78
|
+
this.runs.set(runId, stored);
|
|
79
|
+
}
|
|
80
|
+
async cancel(runId, reason) {
|
|
81
|
+
const run = this.runs.get(runId);
|
|
82
|
+
if (!run) {
|
|
83
|
+
throw new import_media_pipeline_mcp_core.RunNotFoundError();
|
|
84
|
+
}
|
|
85
|
+
run.status = "cancelled";
|
|
86
|
+
run.error = reason;
|
|
87
|
+
run.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
88
|
+
run.version = (run.version ?? 0) + 1;
|
|
89
|
+
}
|
|
90
|
+
async appendEvent(runId, event) {
|
|
91
|
+
const run = this.runs.get(runId);
|
|
92
|
+
if (!run) {
|
|
93
|
+
throw new import_media_pipeline_mcp_core.RunNotFoundError();
|
|
94
|
+
}
|
|
95
|
+
run.events.push(event);
|
|
96
|
+
run.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
97
|
+
}
|
|
98
|
+
async listEvents(runId, sinceSeq) {
|
|
99
|
+
const run = this.runs.get(runId);
|
|
100
|
+
if (!run) {
|
|
101
|
+
throw new import_media_pipeline_mcp_core.RunNotFoundError();
|
|
102
|
+
}
|
|
103
|
+
const events = run.events.map((e) => ({ ...e }));
|
|
104
|
+
if (sinceSeq === void 0 || sinceSeq < 0) return events;
|
|
105
|
+
return events.slice(sinceSeq);
|
|
106
|
+
}
|
|
107
|
+
async listRuns(filter) {
|
|
108
|
+
let result = Array.from(this.runs.values()).filter((r) => matchesFilter(r, filter));
|
|
109
|
+
result.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
110
|
+
if (filter?.offset) {
|
|
111
|
+
result = result.slice(filter.offset);
|
|
112
|
+
}
|
|
113
|
+
if (filter?.limit) {
|
|
114
|
+
result = result.slice(0, filter.limit);
|
|
115
|
+
}
|
|
116
|
+
return result.map(cloneRun);
|
|
117
|
+
}
|
|
118
|
+
async findByExternalJobId(provider, jobId) {
|
|
119
|
+
const key = `${provider}:${jobId}`;
|
|
120
|
+
for (const run of this.runs.values()) {
|
|
121
|
+
if (run.externalJobId === key || run.externalJobIds?.[provider] === jobId) {
|
|
122
|
+
return cloneRun(run);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* In-memory mutex via a Promise chain. Wraps the lock acquisition in a timeout race
|
|
129
|
+
* — if the prior holder hasn't released within `timeoutMs`, throws `RunInProgressError`.
|
|
130
|
+
*
|
|
131
|
+
* NOTE: this is single-process only. Cross-process locking (Redis SET NX EX, Postgres
|
|
132
|
+
* advisory locks) belongs in those backends' implementations.
|
|
133
|
+
*/
|
|
134
|
+
async withLock(runId, fn, timeoutMs = DEFAULT_LOCK_TIMEOUT_MS) {
|
|
135
|
+
const prev = this.locks.get(runId) ?? Promise.resolve();
|
|
136
|
+
const deadline = Date.now() + timeoutMs;
|
|
137
|
+
while (this.locks.has(runId) && Date.now() < deadline) {
|
|
138
|
+
try {
|
|
139
|
+
await Promise.race([
|
|
140
|
+
prev,
|
|
141
|
+
new Promise(
|
|
142
|
+
(resolve) => setTimeout(resolve, Math.min(LOCK_POLL_INTERVAL_MS, deadline - Date.now()))
|
|
143
|
+
)
|
|
144
|
+
]);
|
|
145
|
+
} catch {
|
|
146
|
+
break;
|
|
147
|
+
}
|
|
148
|
+
if (!this.locks.has(runId)) break;
|
|
149
|
+
}
|
|
150
|
+
if (this.locks.has(runId) && Date.now() >= deadline) {
|
|
151
|
+
throw new import_media_pipeline_mcp_core.RunInProgressError();
|
|
152
|
+
}
|
|
153
|
+
const next = (async () => {
|
|
154
|
+
const run = await this.get(runId);
|
|
155
|
+
if (!run) throw new import_media_pipeline_mcp_core.RunNotFoundError();
|
|
156
|
+
return fn(run);
|
|
157
|
+
})();
|
|
158
|
+
this.locks.set(
|
|
159
|
+
runId,
|
|
160
|
+
next.catch(() => {
|
|
161
|
+
}).finally(() => {
|
|
162
|
+
if (this.locks.get(runId) === void 0) return;
|
|
163
|
+
})
|
|
164
|
+
);
|
|
165
|
+
try {
|
|
166
|
+
return await next;
|
|
167
|
+
} finally {
|
|
168
|
+
const current = this.locks.get(runId);
|
|
169
|
+
if (current === next.catch(() => {
|
|
170
|
+
})) {
|
|
171
|
+
this.locks.delete(runId);
|
|
172
|
+
} else {
|
|
173
|
+
this.locks.delete(runId);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
// src/redis-store.ts
|
|
180
|
+
var import_media_pipeline_mcp_core2 = require("@reaatech/media-pipeline-mcp-core");
|
|
181
|
+
var DAY = 86400;
|
|
182
|
+
var RedisPipelineStateStore = class {
|
|
183
|
+
client;
|
|
184
|
+
prefix;
|
|
185
|
+
runTtlSeconds;
|
|
186
|
+
lockTtlSeconds;
|
|
187
|
+
idempotencyTtlSeconds;
|
|
188
|
+
externalJobTtlSeconds;
|
|
189
|
+
lockAcquireTimeoutMs;
|
|
190
|
+
lockPollIntervalMs;
|
|
191
|
+
constructor(config) {
|
|
192
|
+
this.client = config.client;
|
|
193
|
+
this.prefix = config.prefix ?? "mp";
|
|
194
|
+
this.runTtlSeconds = config.runTtlSeconds ?? 30 * DAY;
|
|
195
|
+
this.lockTtlSeconds = config.lockTtlSeconds ?? 60;
|
|
196
|
+
this.idempotencyTtlSeconds = config.idempotencyTtlSeconds ?? DAY;
|
|
197
|
+
this.externalJobTtlSeconds = config.externalJobTtlSeconds ?? 7 * DAY;
|
|
198
|
+
this.lockAcquireTimeoutMs = config.lockAcquireTimeoutMs ?? 5e3;
|
|
199
|
+
this.lockPollIntervalMs = config.lockPollIntervalMs ?? 100;
|
|
200
|
+
}
|
|
201
|
+
runKey(runId) {
|
|
202
|
+
return `${this.prefix}:run:${runId}`;
|
|
203
|
+
}
|
|
204
|
+
eventsKey(runId) {
|
|
205
|
+
return `${this.prefix}:run:${runId}:events`;
|
|
206
|
+
}
|
|
207
|
+
lockKey(runId) {
|
|
208
|
+
return `${this.prefix}:run:${runId}:lock`;
|
|
209
|
+
}
|
|
210
|
+
idempotencyKey(key) {
|
|
211
|
+
return `${this.prefix}:idem:${key}`;
|
|
212
|
+
}
|
|
213
|
+
jobKey(provider, jobId) {
|
|
214
|
+
return `${this.prefix}:job:${provider}:${jobId}`;
|
|
215
|
+
}
|
|
216
|
+
tenantKey(tenantId) {
|
|
217
|
+
return `${this.prefix}:tenant:${tenantId}:runs`;
|
|
218
|
+
}
|
|
219
|
+
async wrap(op) {
|
|
220
|
+
try {
|
|
221
|
+
return await op();
|
|
222
|
+
} catch (err) {
|
|
223
|
+
const error = err;
|
|
224
|
+
if (error?.code === "ECONNREFUSED" || error?.code === "ECONNRESET" || error?.code === "ETIMEDOUT" || /redis|connection/i.test(error?.message ?? "")) {
|
|
225
|
+
throw new import_media_pipeline_mcp_core2.StateStoreUnavailableError();
|
|
226
|
+
}
|
|
227
|
+
throw err;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
async create(run) {
|
|
231
|
+
return this.wrap(async () => {
|
|
232
|
+
const key = this.runKey(run.runId);
|
|
233
|
+
const existing = await this.client.get(key);
|
|
234
|
+
if (existing) {
|
|
235
|
+
throw new import_media_pipeline_mcp_core2.RunInProgressError();
|
|
236
|
+
}
|
|
237
|
+
await this.client.set(key, JSON.stringify(run), "EX", this.runTtlSeconds);
|
|
238
|
+
if (run.tenantId) {
|
|
239
|
+
const ts = Date.parse(run.createdAt) || Date.now();
|
|
240
|
+
await this.client.zadd(this.tenantKey(run.tenantId), ts, run.runId);
|
|
241
|
+
await this.client.expire(this.tenantKey(run.tenantId), this.runTtlSeconds);
|
|
242
|
+
}
|
|
243
|
+
if (run.idempotencyKey) {
|
|
244
|
+
await this.client.set(
|
|
245
|
+
this.idempotencyKey(run.idempotencyKey),
|
|
246
|
+
run.runId,
|
|
247
|
+
"EX",
|
|
248
|
+
this.idempotencyTtlSeconds
|
|
249
|
+
);
|
|
250
|
+
}
|
|
251
|
+
for (const [provider, jobId] of Object.entries(run.externalJobIds ?? {})) {
|
|
252
|
+
await this.client.set(
|
|
253
|
+
this.jobKey(provider, jobId),
|
|
254
|
+
run.runId,
|
|
255
|
+
"EX",
|
|
256
|
+
this.externalJobTtlSeconds
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
async get(runId) {
|
|
262
|
+
return this.wrap(async () => {
|
|
263
|
+
const raw = await this.client.get(this.runKey(runId));
|
|
264
|
+
if (!raw) return null;
|
|
265
|
+
const run = JSON.parse(raw);
|
|
266
|
+
const evs = await this.client.lrange(this.eventsKey(runId), 0, -1);
|
|
267
|
+
run.events = evs.map((s) => JSON.parse(s));
|
|
268
|
+
return run;
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
async update(runId, patch, expectedVersion) {
|
|
272
|
+
return this.wrap(async () => {
|
|
273
|
+
const raw = await this.client.get(this.runKey(runId));
|
|
274
|
+
if (!raw) {
|
|
275
|
+
throw new import_media_pipeline_mcp_core2.RunNotFoundError();
|
|
276
|
+
}
|
|
277
|
+
const existing = JSON.parse(raw);
|
|
278
|
+
if (expectedVersion !== void 0 && existing.version !== expectedVersion) {
|
|
279
|
+
throw new import_media_pipeline_mcp_core2.RunInProgressError();
|
|
280
|
+
}
|
|
281
|
+
const previousJobs = existing.externalJobIds ?? {};
|
|
282
|
+
const merged = {
|
|
283
|
+
...existing,
|
|
284
|
+
...patch,
|
|
285
|
+
// Don't let patch reset events; those live in a separate list.
|
|
286
|
+
events: existing.events,
|
|
287
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
288
|
+
version: (existing.version ?? 0) + 1
|
|
289
|
+
};
|
|
290
|
+
await this.client.set(
|
|
291
|
+
this.runKey(runId),
|
|
292
|
+
JSON.stringify({ ...merged, events: [] }),
|
|
293
|
+
"EX",
|
|
294
|
+
this.runTtlSeconds
|
|
295
|
+
);
|
|
296
|
+
const newJobs = merged.externalJobIds ?? {};
|
|
297
|
+
for (const [provider, jobId] of Object.entries(newJobs)) {
|
|
298
|
+
if (previousJobs[provider] !== jobId) {
|
|
299
|
+
await this.client.set(
|
|
300
|
+
this.jobKey(provider, jobId),
|
|
301
|
+
runId,
|
|
302
|
+
"EX",
|
|
303
|
+
this.externalJobTtlSeconds
|
|
304
|
+
);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
async cancel(runId, reason) {
|
|
310
|
+
return this.wrap(async () => {
|
|
311
|
+
const raw = await this.client.get(this.runKey(runId));
|
|
312
|
+
if (!raw) {
|
|
313
|
+
throw new import_media_pipeline_mcp_core2.RunNotFoundError();
|
|
314
|
+
}
|
|
315
|
+
const existing = JSON.parse(raw);
|
|
316
|
+
existing.status = "cancelled";
|
|
317
|
+
existing.error = reason;
|
|
318
|
+
existing.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
319
|
+
existing.version = (existing.version ?? 0) + 1;
|
|
320
|
+
await this.client.set(
|
|
321
|
+
this.runKey(runId),
|
|
322
|
+
JSON.stringify({ ...existing, events: [] }),
|
|
323
|
+
"EX",
|
|
324
|
+
this.runTtlSeconds
|
|
325
|
+
);
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
async appendEvent(runId, event) {
|
|
329
|
+
return this.wrap(async () => {
|
|
330
|
+
const exists = await this.client.get(this.runKey(runId));
|
|
331
|
+
if (!exists) throw new import_media_pipeline_mcp_core2.RunNotFoundError();
|
|
332
|
+
await this.client.rpush(this.eventsKey(runId), JSON.stringify(event));
|
|
333
|
+
await this.client.expire(this.eventsKey(runId), this.runTtlSeconds);
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
async listEvents(runId, sinceSeq) {
|
|
337
|
+
return this.wrap(async () => {
|
|
338
|
+
const exists = await this.client.get(this.runKey(runId));
|
|
339
|
+
if (!exists) throw new import_media_pipeline_mcp_core2.RunNotFoundError();
|
|
340
|
+
const start = sinceSeq !== void 0 && sinceSeq >= 0 ? sinceSeq : 0;
|
|
341
|
+
const raw = await this.client.lrange(this.eventsKey(runId), start, -1);
|
|
342
|
+
return raw.map((s) => JSON.parse(s));
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
async listRuns(filter) {
|
|
346
|
+
return this.wrap(async () => {
|
|
347
|
+
if (filter?.tenantId) {
|
|
348
|
+
const min = filter.since ? Date.parse(filter.since) || 0 : 0;
|
|
349
|
+
const max = filter.until ? Date.parse(filter.until) || Number.MAX_SAFE_INTEGER : Number.MAX_SAFE_INTEGER;
|
|
350
|
+
const runIds = await this.client.zrangebyscore(this.tenantKey(filter.tenantId), min, max);
|
|
351
|
+
const runs = [];
|
|
352
|
+
for (const id of runIds) {
|
|
353
|
+
const run = await this.get(id);
|
|
354
|
+
if (run && matchesFilter2(run, filter)) runs.push(run);
|
|
355
|
+
}
|
|
356
|
+
runs.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
357
|
+
return applyPaging(runs, filter);
|
|
358
|
+
}
|
|
359
|
+
if (filter?.idempotencyKey) {
|
|
360
|
+
const runId = await this.client.get(this.idempotencyKey(filter.idempotencyKey));
|
|
361
|
+
if (!runId) return [];
|
|
362
|
+
const run = await this.get(runId);
|
|
363
|
+
return run && matchesFilter2(run, filter) ? [run] : [];
|
|
364
|
+
}
|
|
365
|
+
return [];
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
async findByExternalJobId(provider, jobId) {
|
|
369
|
+
return this.wrap(async () => {
|
|
370
|
+
const runId = await this.client.get(this.jobKey(provider, jobId));
|
|
371
|
+
if (!runId) return null;
|
|
372
|
+
return this.get(runId);
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
async withLock(runId, fn, timeoutMs) {
|
|
376
|
+
const deadline = Date.now() + (timeoutMs ?? this.lockAcquireTimeoutMs);
|
|
377
|
+
const lockKey = this.lockKey(runId);
|
|
378
|
+
let acquired = false;
|
|
379
|
+
while (Date.now() < deadline) {
|
|
380
|
+
const res = await this.wrap(
|
|
381
|
+
() => this.client.set(lockKey, "1", "NX", "EX", this.lockTtlSeconds)
|
|
382
|
+
);
|
|
383
|
+
if (res === "OK" || res === 1) {
|
|
384
|
+
acquired = true;
|
|
385
|
+
break;
|
|
386
|
+
}
|
|
387
|
+
await new Promise((r) => setTimeout(r, this.lockPollIntervalMs));
|
|
388
|
+
}
|
|
389
|
+
if (!acquired) {
|
|
390
|
+
throw new import_media_pipeline_mcp_core2.RunInProgressError();
|
|
391
|
+
}
|
|
392
|
+
try {
|
|
393
|
+
const run = await this.get(runId);
|
|
394
|
+
if (!run) throw new import_media_pipeline_mcp_core2.RunNotFoundError();
|
|
395
|
+
return await fn(run);
|
|
396
|
+
} finally {
|
|
397
|
+
await this.client.del(lockKey).catch(() => void 0);
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
};
|
|
401
|
+
function matchesFilter2(run, filter) {
|
|
402
|
+
if (!filter) return true;
|
|
403
|
+
if (filter.status) {
|
|
404
|
+
const statuses = Array.isArray(filter.status) ? filter.status : [filter.status];
|
|
405
|
+
if (!statuses.includes(run.status)) return false;
|
|
406
|
+
}
|
|
407
|
+
if (filter.idempotencyKey && run.idempotencyKey !== filter.idempotencyKey) return false;
|
|
408
|
+
if (filter.since && run.createdAt < filter.since) return false;
|
|
409
|
+
if (filter.until && run.createdAt > filter.until) return false;
|
|
410
|
+
return true;
|
|
411
|
+
}
|
|
412
|
+
function applyPaging(runs, filter) {
|
|
413
|
+
let result = runs;
|
|
414
|
+
if (filter?.offset) result = result.slice(filter.offset);
|
|
415
|
+
if (filter?.limit) result = result.slice(0, filter.limit);
|
|
416
|
+
return result;
|
|
417
|
+
}
|
|
418
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
419
|
+
0 && (module.exports = {
|
|
420
|
+
InMemoryPipelineStateStore,
|
|
421
|
+
RedisPipelineStateStore
|
|
422
|
+
});
|
|
423
|
+
//# sourceMappingURL=index.cjs.map
|