@fractary/faber-mcp 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +523 -0
- package/dist/backends/local-files.d.ts +89 -0
- package/dist/backends/local-files.d.ts.map +1 -0
- package/dist/backends/local-files.js +538 -0
- package/dist/backends/local-files.js.map +1 -0
- package/dist/backends/s3-archive.d.ts +74 -0
- package/dist/backends/s3-archive.d.ts.map +1 -0
- package/dist/backends/s3-archive.js +341 -0
- package/dist/backends/s3-archive.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +14 -0
- package/dist/index.js.map +1 -0
- package/dist/resources/runs.d.ts +36 -0
- package/dist/resources/runs.d.ts.map +1 -0
- package/dist/resources/runs.js +91 -0
- package/dist/resources/runs.js.map +1 -0
- package/dist/server.d.ts +17 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +130 -0
- package/dist/server.js.map +1 -0
- package/dist/tools/events.d.ts +30 -0
- package/dist/tools/events.d.ts.map +1 -0
- package/dist/tools/events.js +199 -0
- package/dist/tools/events.js.map +1 -0
- package/dist/tools/index.d.ts +9 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +8 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/workflow.d.ts +28 -0
- package/dist/tools/workflow.d.ts.map +1 -0
- package/dist/tools/workflow.js +282 -0
- package/dist/tools/workflow.js.map +1 -0
- package/dist/types.d.ts +144 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +62 -0
- package/dist/types.js.map +1 -0
- package/package.json +71 -0
|
@@ -0,0 +1,538 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local Files Backend for FABER Event Gateway
|
|
3
|
+
*
|
|
4
|
+
* Stores events as individual JSON files in the run's events directory.
|
|
5
|
+
* Uses atomic file operations for concurrent access safety.
|
|
6
|
+
*/
|
|
7
|
+
import * as fs from "node:fs";
|
|
8
|
+
import * as path from "node:path";
|
|
9
|
+
import * as crypto from "node:crypto";
|
|
10
|
+
import { EventTypes, } from "../types.js";
|
|
11
|
+
/**
|
|
12
|
+
* Maximum retry attempts for atomic operations
|
|
13
|
+
*/
|
|
14
|
+
const MAX_RETRIES = 10;
|
|
15
|
+
const RETRY_DELAY_MS = 50;
|
|
16
|
+
/**
|
|
17
|
+
* Error result factories for type-safe error handling
|
|
18
|
+
*/
|
|
19
|
+
function createEmitEventError(runId, type, error) {
|
|
20
|
+
return {
|
|
21
|
+
status: "error",
|
|
22
|
+
operation: "emit-event",
|
|
23
|
+
event_id: 0,
|
|
24
|
+
type: type === "unknown" ? "workflow_error" : type,
|
|
25
|
+
run_id: runId,
|
|
26
|
+
timestamp: new Date().toISOString(),
|
|
27
|
+
event_path: "",
|
|
28
|
+
error,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
function createGetRunError(runId, error) {
|
|
32
|
+
return {
|
|
33
|
+
status: "error",
|
|
34
|
+
operation: "get-run",
|
|
35
|
+
run_id: runId,
|
|
36
|
+
metadata: null,
|
|
37
|
+
state: null,
|
|
38
|
+
error,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
function createConsolidateError(runId, error) {
|
|
42
|
+
return {
|
|
43
|
+
status: "error",
|
|
44
|
+
operation: "consolidate-events",
|
|
45
|
+
run_id: runId,
|
|
46
|
+
events_consolidated: 0,
|
|
47
|
+
output_path: "",
|
|
48
|
+
size_bytes: 0,
|
|
49
|
+
error,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
export class LocalFilesBackend {
|
|
53
|
+
basePath;
|
|
54
|
+
resolvedBasePath;
|
|
55
|
+
constructor(basePath) {
|
|
56
|
+
this.basePath = basePath;
|
|
57
|
+
// Resolve the base path once for path traversal protection
|
|
58
|
+
this.resolvedBasePath = path.resolve(basePath);
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Validate run_id format - strict validation to prevent edge cases
|
|
62
|
+
* - org and project must start and end with alphanumeric
|
|
63
|
+
* - uuid must be valid format
|
|
64
|
+
*/
|
|
65
|
+
validateRunId(runId) {
|
|
66
|
+
// Stricter regex: no leading/trailing underscores or hyphens in org/project
|
|
67
|
+
return /^[a-z0-9][a-z0-9_-]*[a-z0-9]\/[a-z0-9][a-z0-9_-]*[a-z0-9]\/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(runId) || /^[a-z0-9]\/[a-z0-9]\/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(runId);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Get the directory path for a run with path traversal protection
|
|
71
|
+
*/
|
|
72
|
+
getRunDir(runId) {
|
|
73
|
+
const runDir = path.join(this.basePath, runId);
|
|
74
|
+
const resolvedRunDir = path.resolve(runDir);
|
|
75
|
+
// Path traversal protection: ensure resolved path is within base path
|
|
76
|
+
if (!resolvedRunDir.startsWith(this.resolvedBasePath + path.sep)) {
|
|
77
|
+
throw new Error(`Path traversal attempt detected: ${runId}`);
|
|
78
|
+
}
|
|
79
|
+
return runDir;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Get the events directory for a run
|
|
83
|
+
*/
|
|
84
|
+
getEventsDir(runId) {
|
|
85
|
+
return path.join(this.getRunDir(runId), "events");
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Generate ISO timestamp with milliseconds for consistency
|
|
89
|
+
*/
|
|
90
|
+
getTimestamp() {
|
|
91
|
+
return new Date().toISOString();
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Sleep helper for retry backoff
|
|
95
|
+
*/
|
|
96
|
+
sleep(ms) {
|
|
97
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Get next event ID atomically using atomic rename pattern
|
|
101
|
+
*
|
|
102
|
+
* This approach is more reliable than file locking:
|
|
103
|
+
* 1. Read current ID
|
|
104
|
+
* 2. Write new ID to temp file with random suffix
|
|
105
|
+
* 3. Atomically rename temp file to target
|
|
106
|
+
* 4. If rename fails due to race, retry with exponential backoff
|
|
107
|
+
*/
|
|
108
|
+
async getNextEventId(runId) {
|
|
109
|
+
const eventsDir = this.getEventsDir(runId);
|
|
110
|
+
const nextIdFile = path.join(eventsDir, ".next-id");
|
|
111
|
+
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
|
112
|
+
try {
|
|
113
|
+
// Read current ID
|
|
114
|
+
let currentId = 1;
|
|
115
|
+
if (fs.existsSync(nextIdFile)) {
|
|
116
|
+
const content = fs.readFileSync(nextIdFile, "utf-8").trim();
|
|
117
|
+
currentId = parseInt(content, 10) || 1;
|
|
118
|
+
}
|
|
119
|
+
const nextId = currentId + 1;
|
|
120
|
+
// Write to temp file with unique suffix
|
|
121
|
+
const tempFile = `${nextIdFile}.${crypto.randomBytes(8).toString("hex")}`;
|
|
122
|
+
fs.writeFileSync(tempFile, String(nextId), { flag: "wx" });
|
|
123
|
+
try {
|
|
124
|
+
// Atomic rename - this is the critical section
|
|
125
|
+
fs.renameSync(tempFile, nextIdFile);
|
|
126
|
+
return currentId;
|
|
127
|
+
}
|
|
128
|
+
catch (renameErr) {
|
|
129
|
+
// Cleanup temp file if rename failed
|
|
130
|
+
try {
|
|
131
|
+
fs.unlinkSync(tempFile);
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
// Ignore cleanup errors
|
|
135
|
+
}
|
|
136
|
+
// If rename failed, another process won - retry
|
|
137
|
+
if (attempt < MAX_RETRIES - 1) {
|
|
138
|
+
await this.sleep(RETRY_DELAY_MS * (attempt + 1));
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
throw renameErr;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
if (err.code === "EEXIST" &&
|
|
146
|
+
attempt < MAX_RETRIES - 1) {
|
|
147
|
+
// Temp file already exists, retry
|
|
148
|
+
await this.sleep(RETRY_DELAY_MS * (attempt + 1));
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
throw err;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
throw new Error(`Failed to acquire event ID after ${MAX_RETRIES} attempts`);
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Update state.json atomically - CRITICAL operation
|
|
158
|
+
* Throws on failure to ensure state consistency
|
|
159
|
+
*/
|
|
160
|
+
updateState(runDir, eventId, timestamp) {
|
|
161
|
+
const stateFile = path.join(runDir, "state.json");
|
|
162
|
+
if (!fs.existsSync(stateFile)) {
|
|
163
|
+
throw new Error(`State file not found: ${stateFile}`);
|
|
164
|
+
}
|
|
165
|
+
const state = JSON.parse(fs.readFileSync(stateFile, "utf-8"));
|
|
166
|
+
state.last_event_id = eventId;
|
|
167
|
+
state.updated_at = timestamp;
|
|
168
|
+
// Write to temp file first
|
|
169
|
+
const tempFile = `${stateFile}.${crypto.randomBytes(8).toString("hex")}`;
|
|
170
|
+
fs.writeFileSync(tempFile, JSON.stringify(state, null, 2));
|
|
171
|
+
// Atomic rename
|
|
172
|
+
try {
|
|
173
|
+
fs.renameSync(tempFile, stateFile);
|
|
174
|
+
}
|
|
175
|
+
catch (err) {
|
|
176
|
+
// Cleanup temp file
|
|
177
|
+
try {
|
|
178
|
+
fs.unlinkSync(tempFile);
|
|
179
|
+
}
|
|
180
|
+
catch {
|
|
181
|
+
// Ignore cleanup errors
|
|
182
|
+
}
|
|
183
|
+
throw new Error(`Failed to update state file: ${err.message}`);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Emit a workflow event
|
|
188
|
+
*/
|
|
189
|
+
async emitEvent(eventData) {
|
|
190
|
+
const { run_id } = eventData;
|
|
191
|
+
// Validate run_id
|
|
192
|
+
if (!run_id || !this.validateRunId(run_id)) {
|
|
193
|
+
return createEmitEventError(run_id || "", eventData.type || "unknown", "Invalid or missing run_id");
|
|
194
|
+
}
|
|
195
|
+
// Validate event type
|
|
196
|
+
if (!eventData.type || !EventTypes.includes(eventData.type)) {
|
|
197
|
+
return createEmitEventError(run_id, "unknown", `Invalid event type: ${eventData.type}`);
|
|
198
|
+
}
|
|
199
|
+
// Get run directory with path traversal protection
|
|
200
|
+
let runDir;
|
|
201
|
+
try {
|
|
202
|
+
runDir = this.getRunDir(run_id);
|
|
203
|
+
}
|
|
204
|
+
catch (err) {
|
|
205
|
+
return createEmitEventError(run_id, eventData.type, err.message);
|
|
206
|
+
}
|
|
207
|
+
// Check run directory exists
|
|
208
|
+
if (!fs.existsSync(runDir)) {
|
|
209
|
+
return createEmitEventError(run_id, eventData.type, `Run directory not found: ${runDir}`);
|
|
210
|
+
}
|
|
211
|
+
// Get next event ID atomically
|
|
212
|
+
let eventId;
|
|
213
|
+
try {
|
|
214
|
+
eventId = await this.getNextEventId(run_id);
|
|
215
|
+
}
|
|
216
|
+
catch (err) {
|
|
217
|
+
return createEmitEventError(run_id, eventData.type, `Failed to get event ID: ${err.message}`);
|
|
218
|
+
}
|
|
219
|
+
const timestamp = this.getTimestamp();
|
|
220
|
+
// Build complete event
|
|
221
|
+
const event = {
|
|
222
|
+
event_id: eventId,
|
|
223
|
+
type: eventData.type,
|
|
224
|
+
timestamp,
|
|
225
|
+
run_id,
|
|
226
|
+
...(eventData.phase && { phase: eventData.phase }),
|
|
227
|
+
...(eventData.step && { step: eventData.step }),
|
|
228
|
+
...(eventData.status && { status: eventData.status }),
|
|
229
|
+
user: eventData.user || process.env.USER || "unknown",
|
|
230
|
+
source: eventData.source || "mcp-gateway",
|
|
231
|
+
...(eventData.message && { message: eventData.message }),
|
|
232
|
+
...(eventData.duration_ms && { duration_ms: eventData.duration_ms }),
|
|
233
|
+
...(eventData.metadata && { metadata: eventData.metadata }),
|
|
234
|
+
...(eventData.artifacts && { artifacts: eventData.artifacts }),
|
|
235
|
+
...(eventData.error && { error: eventData.error }),
|
|
236
|
+
};
|
|
237
|
+
// Write event file
|
|
238
|
+
const eventsDir = this.getEventsDir(run_id);
|
|
239
|
+
const eventFilename = `${String(eventId).padStart(3, "0")}-${eventData.type}.json`;
|
|
240
|
+
const eventPath = path.join(eventsDir, eventFilename);
|
|
241
|
+
fs.writeFileSync(eventPath, JSON.stringify(event, null, 2));
|
|
242
|
+
// Update state.json - CRITICAL: throws on failure
|
|
243
|
+
try {
|
|
244
|
+
this.updateState(runDir, eventId, timestamp);
|
|
245
|
+
}
|
|
246
|
+
catch (err) {
|
|
247
|
+
return createEmitEventError(run_id, eventData.type, `Event written but state update failed: ${err.message}`);
|
|
248
|
+
}
|
|
249
|
+
return {
|
|
250
|
+
status: "success",
|
|
251
|
+
operation: "emit-event",
|
|
252
|
+
event_id: eventId,
|
|
253
|
+
type: eventData.type,
|
|
254
|
+
run_id,
|
|
255
|
+
timestamp,
|
|
256
|
+
event_path: eventPath,
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* Get run state and metadata
|
|
261
|
+
*/
|
|
262
|
+
async getRun(runId, includeEvents = false) {
|
|
263
|
+
if (!this.validateRunId(runId)) {
|
|
264
|
+
return createGetRunError(runId, "Invalid run_id format");
|
|
265
|
+
}
|
|
266
|
+
let runDir;
|
|
267
|
+
try {
|
|
268
|
+
runDir = this.getRunDir(runId);
|
|
269
|
+
}
|
|
270
|
+
catch (err) {
|
|
271
|
+
return createGetRunError(runId, err.message);
|
|
272
|
+
}
|
|
273
|
+
if (!fs.existsSync(runDir)) {
|
|
274
|
+
return createGetRunError(runId, "Run not found");
|
|
275
|
+
}
|
|
276
|
+
const metadataFile = path.join(runDir, "metadata.json");
|
|
277
|
+
const stateFile = path.join(runDir, "state.json");
|
|
278
|
+
try {
|
|
279
|
+
const metadata = JSON.parse(fs.readFileSync(metadataFile, "utf-8"));
|
|
280
|
+
const state = JSON.parse(fs.readFileSync(stateFile, "utf-8"));
|
|
281
|
+
const result = {
|
|
282
|
+
status: "success",
|
|
283
|
+
operation: "get-run",
|
|
284
|
+
run_id: runId,
|
|
285
|
+
metadata,
|
|
286
|
+
state,
|
|
287
|
+
};
|
|
288
|
+
if (includeEvents) {
|
|
289
|
+
const eventsDir = this.getEventsDir(runId);
|
|
290
|
+
if (fs.existsSync(eventsDir)) {
|
|
291
|
+
const eventFiles = fs
|
|
292
|
+
.readdirSync(eventsDir)
|
|
293
|
+
.filter((f) => f.endsWith(".json"));
|
|
294
|
+
result.event_count = eventFiles.length;
|
|
295
|
+
}
|
|
296
|
+
else {
|
|
297
|
+
result.event_count = 0;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
return result;
|
|
301
|
+
}
|
|
302
|
+
catch (err) {
|
|
303
|
+
return createGetRunError(runId, `Failed to read run data: ${err.message}`);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Get events for a run with streaming support for large event sets
|
|
308
|
+
* Uses a generator pattern to avoid loading all events into memory
|
|
309
|
+
*/
|
|
310
|
+
async *getEventsStream(runId) {
|
|
311
|
+
if (!this.validateRunId(runId)) {
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
let eventsDir;
|
|
315
|
+
try {
|
|
316
|
+
eventsDir = this.getEventsDir(runId);
|
|
317
|
+
}
|
|
318
|
+
catch {
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
if (!fs.existsSync(eventsDir)) {
|
|
322
|
+
return;
|
|
323
|
+
}
|
|
324
|
+
const eventFiles = fs
|
|
325
|
+
.readdirSync(eventsDir)
|
|
326
|
+
.filter((f) => f.endsWith(".json"))
|
|
327
|
+
.sort();
|
|
328
|
+
for (const file of eventFiles) {
|
|
329
|
+
try {
|
|
330
|
+
const content = fs.readFileSync(path.join(eventsDir, file), "utf-8");
|
|
331
|
+
yield JSON.parse(content);
|
|
332
|
+
}
|
|
333
|
+
catch {
|
|
334
|
+
// Skip invalid event files
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Get all events for a run (for backward compatibility)
|
|
341
|
+
* For large event sets, prefer getEventsStream
|
|
342
|
+
*/
|
|
343
|
+
async getEvents(runId) {
|
|
344
|
+
const events = [];
|
|
345
|
+
for await (const event of this.getEventsStream(runId)) {
|
|
346
|
+
events.push(event);
|
|
347
|
+
}
|
|
348
|
+
return events;
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* List runs with optional filters
|
|
352
|
+
* Uses an index file for performance when available
|
|
353
|
+
*/
|
|
354
|
+
async listRuns(filters) {
|
|
355
|
+
const { work_id, status, org, project, limit = 20 } = filters;
|
|
356
|
+
const runs = [];
|
|
357
|
+
// Check if base path exists
|
|
358
|
+
if (!fs.existsSync(this.basePath)) {
|
|
359
|
+
return {
|
|
360
|
+
status: "success",
|
|
361
|
+
operation: "list-runs",
|
|
362
|
+
runs: [],
|
|
363
|
+
total: 0,
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
// Check for index file (optimization for large run counts)
|
|
367
|
+
const indexFile = path.join(this.basePath, ".runs-index.json");
|
|
368
|
+
if (fs.existsSync(indexFile)) {
|
|
369
|
+
try {
|
|
370
|
+
const index = JSON.parse(fs.readFileSync(indexFile, "utf-8"));
|
|
371
|
+
let filtered = index;
|
|
372
|
+
if (org) {
|
|
373
|
+
filtered = filtered.filter((r) => r.run_id.startsWith(`${org}/`));
|
|
374
|
+
}
|
|
375
|
+
if (project && org) {
|
|
376
|
+
filtered = filtered.filter((r) => r.run_id.startsWith(`${org}/${project}/`));
|
|
377
|
+
}
|
|
378
|
+
if (work_id) {
|
|
379
|
+
filtered = filtered.filter((r) => r.work_id === work_id);
|
|
380
|
+
}
|
|
381
|
+
if (status) {
|
|
382
|
+
filtered = filtered.filter((r) => r.status === status);
|
|
383
|
+
}
|
|
384
|
+
return {
|
|
385
|
+
status: "success",
|
|
386
|
+
operation: "list-runs",
|
|
387
|
+
runs: filtered.slice(0, limit),
|
|
388
|
+
total: filtered.length,
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
catch {
|
|
392
|
+
// Fall through to directory traversal if index is invalid
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
// Directory traversal fallback
|
|
396
|
+
try {
|
|
397
|
+
const orgs = fs.readdirSync(this.basePath);
|
|
398
|
+
for (const orgName of orgs) {
|
|
399
|
+
// Skip hidden files and index
|
|
400
|
+
if (orgName.startsWith("."))
|
|
401
|
+
continue;
|
|
402
|
+
if (org && orgName !== org)
|
|
403
|
+
continue;
|
|
404
|
+
const orgDir = path.join(this.basePath, orgName);
|
|
405
|
+
if (!fs.statSync(orgDir).isDirectory())
|
|
406
|
+
continue;
|
|
407
|
+
const projects = fs.readdirSync(orgDir);
|
|
408
|
+
for (const projectName of projects) {
|
|
409
|
+
if (projectName.startsWith("."))
|
|
410
|
+
continue;
|
|
411
|
+
if (project && projectName !== project)
|
|
412
|
+
continue;
|
|
413
|
+
const projectDir = path.join(orgDir, projectName);
|
|
414
|
+
if (!fs.statSync(projectDir).isDirectory())
|
|
415
|
+
continue;
|
|
416
|
+
const uuids = fs.readdirSync(projectDir);
|
|
417
|
+
for (const uuid of uuids) {
|
|
418
|
+
if (uuid.startsWith("."))
|
|
419
|
+
continue;
|
|
420
|
+
const runDir = path.join(projectDir, uuid);
|
|
421
|
+
if (!fs.statSync(runDir).isDirectory())
|
|
422
|
+
continue;
|
|
423
|
+
const stateFile = path.join(runDir, "state.json");
|
|
424
|
+
if (!fs.existsSync(stateFile))
|
|
425
|
+
continue;
|
|
426
|
+
try {
|
|
427
|
+
const state = JSON.parse(fs.readFileSync(stateFile, "utf-8"));
|
|
428
|
+
// Apply filters
|
|
429
|
+
if (work_id && state.work_id !== work_id)
|
|
430
|
+
continue;
|
|
431
|
+
if (status && state.status !== status)
|
|
432
|
+
continue;
|
|
433
|
+
runs.push({
|
|
434
|
+
run_id: `${orgName}/${projectName}/${uuid}`,
|
|
435
|
+
work_id: state.work_id,
|
|
436
|
+
status: state.status,
|
|
437
|
+
created_at: state.started_at || state.updated_at,
|
|
438
|
+
updated_at: state.updated_at,
|
|
439
|
+
completed_at: state.completed_at,
|
|
440
|
+
current_phase: state.current_phase,
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
catch {
|
|
444
|
+
// Skip invalid state files
|
|
445
|
+
continue;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
catch (err) {
|
|
452
|
+
return {
|
|
453
|
+
status: "error",
|
|
454
|
+
operation: "list-runs",
|
|
455
|
+
runs: [],
|
|
456
|
+
total: 0,
|
|
457
|
+
error: `Failed to list runs: ${err.message}`,
|
|
458
|
+
};
|
|
459
|
+
}
|
|
460
|
+
// Sort by created_at descending
|
|
461
|
+
runs.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime());
|
|
462
|
+
return {
|
|
463
|
+
status: "success",
|
|
464
|
+
operation: "list-runs",
|
|
465
|
+
runs: runs.slice(0, limit),
|
|
466
|
+
total: runs.length,
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
/**
|
|
470
|
+
* Consolidate events to JSONL format using streaming
|
|
471
|
+
* Avoids loading all events into memory
|
|
472
|
+
*/
|
|
473
|
+
async consolidateEvents(runId) {
|
|
474
|
+
if (!this.validateRunId(runId)) {
|
|
475
|
+
return createConsolidateError(runId, "Invalid run_id format");
|
|
476
|
+
}
|
|
477
|
+
let runDir;
|
|
478
|
+
try {
|
|
479
|
+
runDir = this.getRunDir(runId);
|
|
480
|
+
}
|
|
481
|
+
catch (err) {
|
|
482
|
+
return createConsolidateError(runId, err.message);
|
|
483
|
+
}
|
|
484
|
+
if (!fs.existsSync(runDir)) {
|
|
485
|
+
return createConsolidateError(runId, "Run not found");
|
|
486
|
+
}
|
|
487
|
+
const outputPath = path.join(runDir, "events.jsonl");
|
|
488
|
+
const tempPath = `${outputPath}.${crypto.randomBytes(8).toString("hex")}`;
|
|
489
|
+
let eventCount = 0;
|
|
490
|
+
try {
|
|
491
|
+
// Use write stream for memory efficiency
|
|
492
|
+
const writeStream = fs.createWriteStream(tempPath);
|
|
493
|
+
for await (const event of this.getEventsStream(runId)) {
|
|
494
|
+
writeStream.write(JSON.stringify(event) + "\n");
|
|
495
|
+
eventCount++;
|
|
496
|
+
}
|
|
497
|
+
// Close the stream and wait for completion
|
|
498
|
+
await new Promise((resolve, reject) => {
|
|
499
|
+
writeStream.on("finish", resolve);
|
|
500
|
+
writeStream.on("error", reject);
|
|
501
|
+
writeStream.end();
|
|
502
|
+
});
|
|
503
|
+
// Atomic rename
|
|
504
|
+
fs.renameSync(tempPath, outputPath);
|
|
505
|
+
const stats = fs.statSync(outputPath);
|
|
506
|
+
return {
|
|
507
|
+
status: "success",
|
|
508
|
+
operation: "consolidate-events",
|
|
509
|
+
run_id: runId,
|
|
510
|
+
events_consolidated: eventCount,
|
|
511
|
+
output_path: outputPath,
|
|
512
|
+
size_bytes: stats.size,
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
catch (err) {
|
|
516
|
+
// Cleanup temp file on error
|
|
517
|
+
try {
|
|
518
|
+
fs.unlinkSync(tempPath);
|
|
519
|
+
}
|
|
520
|
+
catch {
|
|
521
|
+
// Ignore cleanup errors
|
|
522
|
+
}
|
|
523
|
+
return createConsolidateError(runId, `Failed to consolidate events: ${err.message}`);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Update the runs index for faster listing
|
|
528
|
+
* Should be called after run completion
|
|
529
|
+
*/
|
|
530
|
+
async updateRunsIndex() {
|
|
531
|
+
const result = await this.listRuns({ limit: 10000 });
|
|
532
|
+
if (result.status === "success") {
|
|
533
|
+
const indexFile = path.join(this.basePath, ".runs-index.json");
|
|
534
|
+
fs.writeFileSync(indexFile, JSON.stringify(result.runs, null, 2));
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
//# sourceMappingURL=local-files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-files.js","sourceRoot":"","sources":["../../src/backends/local-files.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAC;AACtC,OAAO,EAIL,UAAU,GAOX,MAAM,aAAa,CAAC;AAErB;;GAEG;AACH,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,cAAc,GAAG,EAAE,CAAC;AAE1B;;GAEG;AACH,SAAS,oBAAoB,CAC3B,KAAa,EACb,IAA2B,EAC3B,KAAa;IAEb,OAAO;QACL,MAAM,EAAE,OAAO;QACf,SAAS,EAAE,YAAY;QACvB,QAAQ,EAAE,CAAC;QACX,IAAI,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI;QAClD,MAAM,EAAE,KAAK;QACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,UAAU,EAAE,EAAE;QACd,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa,EAAE,KAAa;IACrD,OAAO;QACL,MAAM,EAAE,OAAO;QACf,SAAS,EAAE,SAAS;QACpB,MAAM,EAAE,KAAK;QACb,QAAQ,EAAE,IAA8B;QACxC,KAAK,EAAE,IAA2B;QAClC,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,KAAa,EACb,KAAa;IAEb,OAAO;QACL,MAAM,EAAE,OAAO;QACf,SAAS,EAAE,oBAAoB;QAC/B,MAAM,EAAE,KAAK;QACb,mBAAmB,EAAE,CAAC;QACtB,WAAW,EAAE,EAAE;QACf,UAAU,EAAE,CAAC;QACb,KAAK;KACN,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,iBAAiB;IACX,QAAQ,CAAS;IACjB,gBAAgB,CAAS;IAE1C,YAAY,QAAgB;QAC1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,2DAA2D;QAC3D,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACK,aAAa,CAAC,KAAa;QACjC,4EAA4E;QAC5E,OAAO,0HAA0H,CAAC,IAAI,CACpI,KAAK,CACN,IAAI,oFAAoF,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxG,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,KAAa;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAE5C,sEAAsE;QACtE,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,oCAAoC,KAAK,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,KAAa;QAChC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAClC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,cAAc,CAAC,KAAa;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAEpD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC;gBACH,kBAAkB;gBAClB,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC9B,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC5D,SAAS,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBACzC,CAAC;gBAED,MAAM,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC;gBAE7B,wCAAwC;gBACxC,MAAM,QAAQ,GAAG,GAAG,UAAU,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1E,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAE3D,IAAI,CAAC;oBACH,+CAA+C;oBAC/C,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;oBACpC,OAAO,SAAS,CAAC;gBACnB,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACnB,qCAAqC;oBACrC,IAAI,CAAC;wBACH,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;oBAC1B,CAAC;oBAAC,MAAM,CAAC;wBACP,wBAAwB;oBAC1B,CAAC;oBAED,gDAAgD;oBAChD,IAAI,OAAO,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC;wBAC9B,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;wBACjD,SAAS;oBACX,CAAC;oBACD,MAAM,SAAS,CAAC;gBAClB,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IACG,GAA6B,CAAC,IAAI,KAAK,QAAQ;oBAChD,OAAO,GAAG,WAAW,GAAG,CAAC,EACzB,CAAC;oBACD,kCAAkC;oBAClC,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;oBACjD,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,oCAAoC,WAAW,WAAW,CAAC,CAAC;IAC9E,CAAC;IAED;;;OAGG;IACK,WAAW,CACjB,MAAc,EACd,OAAe,EACf,SAAiB;QAEjB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAElD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,yBAAyB,SAAS,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;QAC9D,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC;QAC9B,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC;QAE7B,2BAA2B;QAC3B,MAAM,QAAQ,GAAG,GAAG,SAAS,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACzE,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAE3D,gBAAgB;QAChB,IAAI,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,oBAAoB;YACpB,IAAI,CAAC;gBACH,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;YACD,MAAM,IAAI,KAAK,CACb,gCAAiC,GAAa,CAAC,OAAO,EAAE,CACzD,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,SAA8B;QAC5C,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAE7B,kBAAkB;QAClB,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3C,OAAO,oBAAoB,CACzB,MAAM,IAAI,EAAE,EACX,SAAS,CAAC,IAAkB,IAAI,SAAS,EAC1C,2BAA2B,CAC5B,CAAC;QACJ,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5D,OAAO,oBAAoB,CACzB,MAAM,EACN,SAAS,EACT,uBAAuB,SAAS,CAAC,IAAI,EAAE,CACxC,CAAC;QACJ,CAAC;QAED,mDAAmD;QACnD,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,oBAAoB,CACzB,MAAM,EACN,SAAS,CAAC,IAAI,EACb,GAAa,CAAC,OAAO,CACvB,CAAC;QACJ,CAAC;QAED,6BAA6B;QAC7B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,OAAO,oBAAoB,CACzB,MAAM,EACN,SAAS,CAAC,IAAI,EACd,4BAA4B,MAAM,EAAE,CACrC,CAAC;QACJ,CAAC;QAED,+BAA+B;QAC/B,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,oBAAoB,CACzB,MAAM,EACN,SAAS,CAAC,IAAI,EACd,2BAA4B,GAAa,CAAC,OAAO,EAAE,CACpD,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAEtC,uBAAuB;QACvB,MAAM,KAAK,GAAe;YACxB,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,SAAS;YACT,MAAM;YACN,GAAG,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC;YAClD,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC;YAC/C,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC;YACrD,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,SAAS;YACrD,MAAM,EAAE,SAAS,CAAC,MAAM,IAAI,aAAa;YACzC,GAAG,CAAC,SAAS,CAAC,OAAO,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC;YACxD,GAAG,CAAC,SAAS,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,CAAC;YACpE,GAAG,CAAC,SAAS,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,CAAC;YAC3D,GAAG,CAAC,SAAS,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,SAAS,EAAE,CAAC;YAC9D,GAAG,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC;SACnD,CAAC;QAEF,mBAAmB;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC5C,MAAM,aAAa,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,OAAO,CAAC;QACnF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QAEtD,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAE5D,kDAAkD;QAClD,IAAI,CAAC;YACH,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,oBAAoB,CACzB,MAAM,EACN,SAAS,CAAC,IAAI,EACd,0CAA2C,GAAa,CAAC,OAAO,EAAE,CACnE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,SAAS,EAAE,YAAY;YACvB,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,MAAM;YACN,SAAS;YACT,UAAU,EAAE,SAAS;SACtB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,aAAa,GAAG,KAAK;QAC/C,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,iBAAiB,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,iBAAiB,CAAC,KAAK,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,OAAO,iBAAiB,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAElD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CACzB,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CACxB,CAAC;YACjB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CACtB,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CACxB,CAAC;YAEd,MAAM,MAAM,GAAiB;gBAC3B,MAAM,EAAE,SAAS;gBACjB,SAAS,EAAE,SAAS;gBACpB,MAAM,EAAE,KAAK;gBACb,QAAQ;gBACR,KAAK;aACN,CAAC;YAEF,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC7B,MAAM,UAAU,GAAG,EAAE;yBAClB,WAAW,CAAC,SAAS,CAAC;yBACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;oBACtC,MAAM,CAAC,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC;gBACzC,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,iBAAiB,CACtB,KAAK,EACL,4BAA6B,GAAa,CAAC,OAAO,EAAE,CACrD,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,CAAC,eAAe,CAAC,KAAa;QAClC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,IAAI,SAAiB,CAAC;QACtB,IAAI,CAAC;YACH,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QAED,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QAED,MAAM,UAAU,GAAG,EAAE;aAClB,WAAW,CAAC,SAAS,CAAC;aACtB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;aAClC,IAAI,EAAE,CAAC;QAEV,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;gBACrE,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAe,CAAC;YAC1C,CAAC;YAAC,MAAM,CAAC;gBACP,2BAA2B;gBAC3B,SAAS;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAC,KAAa;QAC3B,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;YACtD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,OAMd;QACC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QAC9D,MAAM,IAAI,GAAiB,EAAE,CAAC;QAE9B,4BAA4B;QAC5B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClC,OAAO;gBACL,MAAM,EAAE,SAAS;gBACjB,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,EAAE;gBACR,KAAK,EAAE,CAAC;aACT,CAAC;QACJ,CAAC;QAED,2DAA2D;QAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAC/D,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CACtB,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CACpB,CAAC;gBAClB,IAAI,QAAQ,GAAG,KAAK,CAAC;gBAErB,IAAI,GAAG,EAAE,CAAC;oBACR,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC;gBACpE,CAAC;gBACD,IAAI,OAAO,IAAI,GAAG,EAAE,CAAC;oBACnB,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/B,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,GAAG,IAAI,OAAO,GAAG,CAAC,CAC1C,CAAC;gBACJ,CAAC;gBACD,IAAI,OAAO,EAAE,CAAC;oBACZ,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC;gBAC3D,CAAC;gBACD,IAAI,MAAM,EAAE,CAAC;oBACX,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;gBACzD,CAAC;gBAED,OAAO;oBACL,MAAM,EAAE,SAAS;oBACjB,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;oBAC9B,KAAK,EAAE,QAAQ,CAAC,MAAM;iBACvB,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,0DAA0D;YAC5D,CAAC;QACH,CAAC;QAED,+BAA+B;QAC/B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAE3C,KAAK,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,8BAA8B;gBAC9B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtC,IAAI,GAAG,IAAI,OAAO,KAAK,GAAG;oBAAE,SAAS;gBAErC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBACjD,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE;oBAAE,SAAS;gBAEjD,MAAM,QAAQ,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAExC,KAAK,MAAM,WAAW,IAAI,QAAQ,EAAE,CAAC;oBACnC,IAAI,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC;wBAAE,SAAS;oBAC1C,IAAI,OAAO,IAAI,WAAW,KAAK,OAAO;wBAAE,SAAS;oBAEjD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;oBAClD,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE;wBAAE,SAAS;oBAErD,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;oBAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;wBACzB,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;4BAAE,SAAS;wBAEnC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;wBAC3C,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE;4BAAE,SAAS;wBAEjD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;wBAClD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;4BAAE,SAAS;wBAExC,IAAI,CAAC;4BACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CACtB,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CACxB,CAAC;4BAEd,gBAAgB;4BAChB,IAAI,OAAO,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO;gCAAE,SAAS;4BACnD,IAAI,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM;gCAAE,SAAS;4BAEhD,IAAI,CAAC,IAAI,CAAC;gCACR,MAAM,EAAE,GAAG,OAAO,IAAI,WAAW,IAAI,IAAI,EAAE;gCAC3C,OAAO,EAAE,KAAK,CAAC,OAAO;gCACtB,MAAM,EAAE,KAAK,CAAC,MAAM;gCACpB,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU;gCAChD,UAAU,EAAE,KAAK,CAAC,UAAU;gCAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;gCAChC,aAAa,EAAE,KAAK,CAAC,aAAa;6BACnC,CAAC,CAAC;wBACL,CAAC;wBAAC,MAAM,CAAC;4BACP,2BAA2B;4BAC3B,SAAS;wBACX,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,MAAM,EAAE,OAAO;gBACf,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,EAAE;gBACR,KAAK,EAAE,CAAC;gBACR,KAAK,EAAE,wBAAyB,GAAa,CAAC,OAAO,EAAE;aACxD,CAAC;QACJ,CAAC;QAED,gCAAgC;QAChC,IAAI,CAAC,IAAI,CACP,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,IAAI,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CACtE,CAAC;QAEF,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,SAAS,EAAE,WAAW;YACtB,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;YAC1B,KAAK,EAAE,IAAI,CAAC,MAAM;SACnB,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,iBAAiB,CAAC,KAAa;QACnC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,sBAAsB,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,sBAAsB,CAAC,KAAK,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,OAAO,sBAAsB,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,GAAG,UAAU,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAE1E,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,IAAI,CAAC;YACH,yCAAyC;YACzC,MAAM,WAAW,GAAG,EAAE,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;YAEnD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtD,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;gBAChD,UAAU,EAAE,CAAC;YACf,CAAC;YAED,2CAA2C;YAC3C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC1C,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAClC,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAChC,WAAW,CAAC,GAAG,EAAE,CAAC;YACpB,CAAC,CAAC,CAAC;YAEH,gBAAgB;YAChB,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YAEpC,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAEtC,OAAO;gBACL,MAAM,EAAE,SAAS;gBACjB,SAAS,EAAE,oBAAoB;gBAC/B,MAAM,EAAE,KAAK;gBACb,mBAAmB,EAAE,UAAU;gBAC/B,WAAW,EAAE,UAAU;gBACvB,UAAU,EAAE,KAAK,CAAC,IAAI;aACvB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6BAA6B;YAC7B,IAAI,CAAC;gBACH,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;YAED,OAAO,sBAAsB,CAC3B,KAAK,EACL,iCAAkC,GAAa,CAAC,OAAO,EAAE,CAC1D,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe;QACnB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QACrD,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;YAC/D,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* S3 Archive Backend for FABER Event Gateway
|
|
3
|
+
*
|
|
4
|
+
* Archives events to S3 for long-term storage and analysis.
|
|
5
|
+
* Supports consolidation to JSONL format for efficient storage.
|
|
6
|
+
*/
|
|
7
|
+
export interface S3Config {
|
|
8
|
+
bucket: string;
|
|
9
|
+
prefix: string;
|
|
10
|
+
region: string;
|
|
11
|
+
consolidateOnComplete: boolean;
|
|
12
|
+
cleanupLocalAfterArchive: boolean;
|
|
13
|
+
}
|
|
14
|
+
export interface ArchiveResult {
|
|
15
|
+
status: "success" | "error";
|
|
16
|
+
operation: "archive-to-s3";
|
|
17
|
+
run_id: string;
|
|
18
|
+
s3_path: string;
|
|
19
|
+
files_archived: string[];
|
|
20
|
+
size_bytes: number;
|
|
21
|
+
error?: string;
|
|
22
|
+
}
|
|
23
|
+
export declare class S3ArchiveBackend {
|
|
24
|
+
private config;
|
|
25
|
+
private localBasePath;
|
|
26
|
+
constructor(config: S3Config, localBasePath: string);
|
|
27
|
+
/**
|
|
28
|
+
* Execute AWS CLI command
|
|
29
|
+
*/
|
|
30
|
+
private execAwsCli;
|
|
31
|
+
/**
|
|
32
|
+
* Get S3 path for a run
|
|
33
|
+
*/
|
|
34
|
+
private getS3Path;
|
|
35
|
+
/**
|
|
36
|
+
* Get local run directory
|
|
37
|
+
*/
|
|
38
|
+
private getLocalRunDir;
|
|
39
|
+
/**
|
|
40
|
+
* Archive a completed run to S3
|
|
41
|
+
*/
|
|
42
|
+
archiveRun(runId: string): Promise<ArchiveResult>;
|
|
43
|
+
/**
|
|
44
|
+
* List archived runs in S3
|
|
45
|
+
*/
|
|
46
|
+
listArchivedRuns(filters?: {
|
|
47
|
+
org?: string;
|
|
48
|
+
project?: string;
|
|
49
|
+
limit?: number;
|
|
50
|
+
}): Promise<{
|
|
51
|
+
status: "success" | "error";
|
|
52
|
+
runs: Array<{
|
|
53
|
+
run_id: string;
|
|
54
|
+
s3_path: string;
|
|
55
|
+
size_bytes: number;
|
|
56
|
+
}>;
|
|
57
|
+
error?: string;
|
|
58
|
+
}>;
|
|
59
|
+
/**
|
|
60
|
+
* Restore a run from S3 to local storage
|
|
61
|
+
*/
|
|
62
|
+
restoreRun(runId: string): Promise<{
|
|
63
|
+
status: "success" | "error";
|
|
64
|
+
run_id: string;
|
|
65
|
+
local_path: string;
|
|
66
|
+
files_restored: string[];
|
|
67
|
+
error?: string;
|
|
68
|
+
}>;
|
|
69
|
+
/**
|
|
70
|
+
* Helper to list files recursively
|
|
71
|
+
*/
|
|
72
|
+
private listFilesRecursive;
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=s3-archive.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3-archive.d.ts","sourceRoot":"","sources":["../../src/backends/s3-archive.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAYH,MAAM,WAAW,QAAQ;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,qBAAqB,EAAE,OAAO,CAAC;IAC/B,wBAAwB,EAAE,OAAO,CAAC;CACnC;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC;IAC5B,SAAS,EAAE,eAAe,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,gBAAgB;IAEzB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,aAAa;gBADb,MAAM,EAAE,QAAQ,EAChB,aAAa,EAAE,MAAM;IAG/B;;OAEG;YACW,UAAU;IA2BxB;;OAEG;IACH,OAAO,CAAC,SAAS;IAIjB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAuJvD;;OAEG;IACG,gBAAgB,CAAC,OAAO,CAAC,EAAE;QAC/B,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,CAAC;QACV,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC;QAC5B,IAAI,EAAE,KAAK,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACrE,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IAwEF;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;QACvC,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC;QAC5B,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IA8EF;;OAEG;IACH,OAAO,CAAC,kBAAkB;CAe3B"}
|