@exellix/exellix-jobs 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +103 -0
- package/dist/cli.d.ts +6 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +100 -0
- package/dist/cli.js.map +1 -0
- package/dist/collections.d.ts +28 -0
- package/dist/collections.d.ts.map +1 -0
- package/dist/collections.js +2 -0
- package/dist/collections.js.map +1 -0
- package/dist/env.d.ts +17 -0
- package/dist/env.d.ts.map +1 -0
- package/dist/env.js +38 -0
- package/dist/env.js.map +1 -0
- package/dist/http/health.d.ts +10 -0
- package/dist/http/health.d.ts.map +1 -0
- package/dist/http/health.js +20 -0
- package/dist/http/health.js.map +1 -0
- package/dist/http/server.d.ts +10 -0
- package/dist/http/server.d.ts.map +1 -0
- package/dist/http/server.js +85 -0
- package/dist/http/server.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +13 -0
- package/dist/index.js.map +1 -0
- package/dist/memory-matrix-tier.d.ts +6 -0
- package/dist/memory-matrix-tier.d.ts.map +1 -0
- package/dist/memory-matrix-tier.js +20 -0
- package/dist/memory-matrix-tier.js.map +1 -0
- package/dist/metrics.d.ts +13 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/metrics.js +57 -0
- package/dist/metrics.js.map +1 -0
- package/dist/open-matrix-persistence.d.ts +7 -0
- package/dist/open-matrix-persistence.d.ts.map +1 -0
- package/dist/open-matrix-persistence.js +29 -0
- package/dist/open-matrix-persistence.js.map +1 -0
- package/dist/persistence-env.d.ts +7 -0
- package/dist/persistence-env.d.ts.map +1 -0
- package/dist/persistence-env.js +14 -0
- package/dist/persistence-env.js.map +1 -0
- package/dist/runtime-matrix-tier.d.ts +8 -0
- package/dist/runtime-matrix-tier.d.ts.map +1 -0
- package/dist/runtime-matrix-tier.js +57 -0
- package/dist/runtime-matrix-tier.js.map +1 -0
- package/dist/worker/continuous-loop.d.ts +22 -0
- package/dist/worker/continuous-loop.d.ts.map +1 -0
- package/dist/worker/continuous-loop.js +73 -0
- package/dist/worker/continuous-loop.js.map +1 -0
- package/dist/worker/cycle-state.d.ts +22 -0
- package/dist/worker/cycle-state.d.ts.map +1 -0
- package/dist/worker/cycle-state.js +25 -0
- package/dist/worker/cycle-state.js.map +1 -0
- package/dist/worker/discovery.d.ts +32 -0
- package/dist/worker/discovery.d.ts.map +1 -0
- package/dist/worker/discovery.js +104 -0
- package/dist/worker/discovery.js.map +1 -0
- package/dist/worker/matrix-worker.d.ts +57 -0
- package/dist/worker/matrix-worker.d.ts.map +1 -0
- package/dist/worker/matrix-worker.js +103 -0
- package/dist/worker/matrix-worker.js.map +1 -0
- package/dist/worker/wrap-execute-graph.d.ts +14 -0
- package/dist/worker/wrap-execute-graph.d.ts.map +1 -0
- package/dist/worker/wrap-execute-graph.js +61 -0
- package/dist/worker/wrap-execute-graph.js.map +1 -0
- package/dist/xmemory-matrix-tier.d.ts +7 -0
- package/dist/xmemory-matrix-tier.d.ts.map +1 -0
- package/dist/xmemory-matrix-tier.js +65 -0
- package/dist/xmemory-matrix-tier.js.map +1 -0
- package/docs/execution-matrix-worker-service.md +8 -0
- package/docs/openapi.yaml +122 -0
- package/docs/specs.md +305 -0
- package/docs/troubleshooting.md +252 -0
- package/examples/graph-engine-stub.mjs +40 -0
- package/examples/host-bootstrap.mjs +74 -0
- package/package.json +67 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { createExecutionMatrixClaimPauseController, createExecutionMatrixRuntime, createExellixConfigStore, } from '@exellix/exellix-runtime';
|
|
2
|
+
import { createJobsMetrics } from '../metrics.js';
|
|
3
|
+
import { MatrixCycleState } from './cycle-state.js';
|
|
4
|
+
import { runMatrixContinuouslyByCatalogIdWithMetrics } from './continuous-loop.js';
|
|
5
|
+
import { discoverMatrixAssignments } from './discovery.js';
|
|
6
|
+
import { wrapMatrixGraphExecutor } from './wrap-execute-graph.js';
|
|
7
|
+
export async function createMatrixWorker(options) {
|
|
8
|
+
await options.persistence.init?.();
|
|
9
|
+
const runtime = createExecutionMatrixRuntime(options.persistence.rows, options.persistence.failures, {
|
|
10
|
+
snapshots: options.persistence.snapshots,
|
|
11
|
+
serializeClaims: true,
|
|
12
|
+
});
|
|
13
|
+
const configStore = createExellixConfigStore({
|
|
14
|
+
matrices: options.persistence.matrices,
|
|
15
|
+
graphs: options.persistence.graphs,
|
|
16
|
+
});
|
|
17
|
+
const metrics = options.metrics ?? createJobsMetrics();
|
|
18
|
+
const pauseController = createExecutionMatrixClaimPauseController();
|
|
19
|
+
const cycleState = new MatrixCycleState();
|
|
20
|
+
const inFlight = new Map();
|
|
21
|
+
const maxConcurrentClaims = options.maxConcurrentClaims ?? 4;
|
|
22
|
+
let currentAssignments = [];
|
|
23
|
+
const executeGraph = wrapMatrixGraphExecutor({
|
|
24
|
+
inner: options.executeGraph,
|
|
25
|
+
maxConcurrentClaims,
|
|
26
|
+
perGraphConcurrency: options.perGraphConcurrency,
|
|
27
|
+
metrics,
|
|
28
|
+
onInFlightDelta: (graphId, delta) => {
|
|
29
|
+
const cur = inFlight.get(graphId) ?? 0;
|
|
30
|
+
const next = Math.max(0, cur + delta);
|
|
31
|
+
if (next === 0) {
|
|
32
|
+
inFlight.delete(graphId);
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
inFlight.set(graphId, next);
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
});
|
|
39
|
+
const dispatchDeps = {
|
|
40
|
+
runtime,
|
|
41
|
+
configStore,
|
|
42
|
+
executeGraph,
|
|
43
|
+
resolveGraphModel: options.resolveGraphModel,
|
|
44
|
+
resolveGraphSchedulability: (graphId) => configStore.graphs.get(graphId),
|
|
45
|
+
graphSchedulabilityRequireConfig: options.graphSchedulabilityRequireConfig,
|
|
46
|
+
shouldClaimNext: () => pauseController.shouldClaimNext(),
|
|
47
|
+
sourceResolver: options.sourceResolver,
|
|
48
|
+
matrixActivix: options.matrixActivix,
|
|
49
|
+
matrixLogxer: options.matrixLogxer,
|
|
50
|
+
executeOverrides: options.executeOverrides,
|
|
51
|
+
runtimeScope: options.runtimeScope,
|
|
52
|
+
runtimeObjects: options.runtimeObjects,
|
|
53
|
+
};
|
|
54
|
+
const getStatus = (filter) => {
|
|
55
|
+
const activeJobsByGraphId = {};
|
|
56
|
+
for (const [gid, n] of inFlight) {
|
|
57
|
+
activeJobsByGraphId[gid] = n;
|
|
58
|
+
}
|
|
59
|
+
const assignments = filter?.matrixCatalogId
|
|
60
|
+
? currentAssignments.filter((a) => a.matrixCatalogId === filter.matrixCatalogId)
|
|
61
|
+
: currentAssignments;
|
|
62
|
+
const catalogIds = assignments.map((a) => a.matrixCatalogId);
|
|
63
|
+
const graphIds = [...new Set(assignments.flatMap((a) => a.graphIds))].sort();
|
|
64
|
+
return {
|
|
65
|
+
...(options.matrixWorkerId !== undefined ? { matrixWorkerId: options.matrixWorkerId } : {}),
|
|
66
|
+
assignedMatrixCatalogIds: catalogIds,
|
|
67
|
+
assignedGraphIds: graphIds,
|
|
68
|
+
activeJobsByGraphId,
|
|
69
|
+
activeJobs: { ...activeJobsByGraphId },
|
|
70
|
+
maxConcurrency: maxConcurrentClaims,
|
|
71
|
+
paused: pauseController.isPaused(),
|
|
72
|
+
loops: cycleState.snapshot(filter),
|
|
73
|
+
};
|
|
74
|
+
};
|
|
75
|
+
const startSupervised = async (signal, opts) => {
|
|
76
|
+
const assignments = opts && 'assignments' in opts && opts.assignments !== undefined
|
|
77
|
+
? opts.assignments
|
|
78
|
+
: await discoverMatrixAssignments(configStore);
|
|
79
|
+
currentAssignments = assignments;
|
|
80
|
+
await Promise.all(assignments.map((a) => runMatrixContinuouslyByCatalogIdWithMetrics(dispatchDeps, {
|
|
81
|
+
matrixCatalogId: a.matrixCatalogId,
|
|
82
|
+
matrixItemId: a.matrixItemId,
|
|
83
|
+
matrixRunId: a.matrixRunId,
|
|
84
|
+
signal,
|
|
85
|
+
cycleState,
|
|
86
|
+
}, metrics)));
|
|
87
|
+
};
|
|
88
|
+
const close = async () => {
|
|
89
|
+
await options.persistence.close();
|
|
90
|
+
};
|
|
91
|
+
return {
|
|
92
|
+
runtime,
|
|
93
|
+
configStore,
|
|
94
|
+
metrics,
|
|
95
|
+
pauseController,
|
|
96
|
+
connectivityProbe: options.persistence.probe,
|
|
97
|
+
getStatus,
|
|
98
|
+
startSupervised,
|
|
99
|
+
close,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
export { discoverMatrixAssignments, listGraphSummaries } from './discovery.js';
|
|
103
|
+
//# sourceMappingURL=matrix-worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"matrix-worker.js","sourceRoot":"","sources":["../../src/worker/matrix-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,yCAAyC,EACzC,4BAA4B,EAC5B,wBAAwB,GAQzB,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAA0B,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,2CAA2C,EAAE,MAAM,sBAAsB,CAAC;AACnF,OAAO,EAAE,yBAAyB,EAAyB,MAAM,gBAAgB,CAAC;AAClF,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAmDlE,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IACzE,MAAM,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,4BAA4B,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE;QACnG,SAAS,EAAE,OAAO,CAAC,WAAW,CAAC,SAAS;QACxC,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,wBAAwB,CAAC;QAC3C,QAAQ,EAAE,OAAO,CAAC,WAAW,CAAC,QAAQ;QACtC,MAAM,EAAE,OAAO,CAAC,WAAW,CAAC,MAAM;KACnC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACvD,MAAM,eAAe,GAAG,yCAAyC,EAAE,CAAC;IACpE,MAAM,UAAU,GAAG,IAAI,gBAAgB,EAAE,CAAC;IAC1C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,MAAM,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,CAAC,CAAC;IAC7D,IAAI,kBAAkB,GAAuB,EAAE,CAAC;IAEhD,MAAM,YAAY,GAAG,uBAAuB,CAAC;QAC3C,KAAK,EAAE,OAAO,CAAC,YAAY;QAC3B,mBAAmB;QACnB,mBAAmB,EAAE,OAAO,CAAC,mBAAmB;QAChD,OAAO;QACP,eAAe,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAClC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACvC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;YACtC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,YAAY,GAA6B;QAC7C,OAAO;QACP,WAAW;QACX,YAAY;QACZ,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;QAC5C,0BAA0B,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC;QACxE,gCAAgC,EAAE,OAAO,CAAC,gCAAgC;QAC1E,eAAe,EAAE,GAAG,EAAE,CAAC,eAAe,CAAC,eAAe,EAAE;QACxD,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,cAAc,EAAE,OAAO,CAAC,cAAc;KACvC,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,MAAqC,EAAsB,EAAE;QAC9E,MAAM,mBAAmB,GAA2B,EAAE,CAAC;QACvD,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,QAAQ,EAAE,CAAC;YAChC,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QACD,MAAM,WAAW,GAAG,MAAM,EAAE,eAAe;YACzC,CAAC,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,KAAK,MAAM,CAAC,eAAe,CAAC;YAChF,CAAC,CAAC,kBAAkB,CAAC;QACvB,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7E,OAAO;YACL,GAAG,CAAC,OAAO,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3F,wBAAwB,EAAE,UAAU;YACpC,gBAAgB,EAAE,QAAQ;YAC1B,mBAAmB;YACnB,UAAU,EAAE,EAAE,GAAG,mBAAmB,EAAE;YACtC,cAAc,EAAE,mBAAmB;YACnC,MAAM,EAAE,eAAe,CAAC,QAAQ,EAAE;YAClC,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC;SACnC,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,eAAe,GAAG,KAAK,EAC3B,MAAmB,EACnB,IAA2C,EAC5B,EAAE;QACjB,MAAM,WAAW,GACf,IAAI,IAAI,aAAa,IAAI,IAAI,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;YAC7D,CAAC,CAAC,IAAI,CAAC,WAAW;YAClB,CAAC,CAAC,MAAM,yBAAyB,CAAC,WAAW,CAAC,CAAC;QACnD,kBAAkB,GAAG,WAAW,CAAC;QACjC,MAAM,OAAO,CAAC,GAAG,CACf,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACpB,2CAA2C,CACzC,YAAY,EACZ;YACE,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,YAAY,EAAE,CAAC,CAAC,YAAY;YAC5B,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,MAAM;YACN,UAAU;SACX,EACD,OAAO,CACR,CACF,CACF,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACtC,MAAM,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IACpC,CAAC,CAAC;IAEF,OAAO;QACL,OAAO;QACP,WAAW;QACX,OAAO;QACP,eAAe;QACf,iBAAiB,EAAE,OAAO,CAAC,WAAW,CAAC,KAAK;QAC5C,SAAS;QACT,eAAe;QACf,KAAK;KACN,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,yBAAyB,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { MatrixGraphExecutor } from '@exellix/exellix-runtime';
|
|
2
|
+
import type { JobsMetrics } from '../metrics.js';
|
|
3
|
+
export type WrapMatrixGraphExecutorOptions = {
|
|
4
|
+
inner: MatrixGraphExecutor;
|
|
5
|
+
maxConcurrentClaims: number;
|
|
6
|
+
perGraphConcurrency?: Record<string, number>;
|
|
7
|
+
metrics?: JobsMetrics;
|
|
8
|
+
onInFlightDelta?: (graphId: string, delta: 1 | -1) => void;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Global and optional per-graph concurrency around graph-engine execution, plus §11 counters/histogram.
|
|
12
|
+
*/
|
|
13
|
+
export declare function wrapMatrixGraphExecutor(opts: WrapMatrixGraphExecutorOptions): MatrixGraphExecutor;
|
|
14
|
+
//# sourceMappingURL=wrap-execute-graph.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wrap-execute-graph.d.ts","sourceRoot":"","sources":["../../src/worker/wrap-execute-graph.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAEpE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAKjD,MAAM,MAAM,8BAA8B,GAAG;IAC3C,KAAK,EAAE,mBAAmB,CAAC;IAC3B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC;CAC5D,CAAC;AAkBF;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,8BAA8B,GAAG,mBAAmB,CA2CjG"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import pLimit from 'p-limit';
|
|
2
|
+
function resolveGraphId(input) {
|
|
3
|
+
const job = input.runtime?.job;
|
|
4
|
+
const fromJob = job?.graphId ?? job?.graph_id;
|
|
5
|
+
if (typeof fromJob === 'string' && fromJob.length > 0) {
|
|
6
|
+
return fromJob;
|
|
7
|
+
}
|
|
8
|
+
const id = input.model?.id;
|
|
9
|
+
return typeof id === 'string' && id.length > 0 ? id : 'unknown';
|
|
10
|
+
}
|
|
11
|
+
function resolveMatrixCatalogId(input) {
|
|
12
|
+
const job = input.runtime?.job;
|
|
13
|
+
const raw = job?.matrixCatalogId ?? job?.matrix_catalog_id;
|
|
14
|
+
return typeof raw === 'string' && raw.length > 0 ? raw : 'unknown';
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Global and optional per-graph concurrency around graph-engine execution, plus §11 counters/histogram.
|
|
18
|
+
*/
|
|
19
|
+
export function wrapMatrixGraphExecutor(opts) {
|
|
20
|
+
const globalLimit = pLimit(opts.maxConcurrentClaims);
|
|
21
|
+
const perGraphLimits = new Map();
|
|
22
|
+
const perGraphMax = opts.perGraphConcurrency ?? {};
|
|
23
|
+
const getPerGraphLimit = (graphId) => {
|
|
24
|
+
const cap = perGraphMax[graphId];
|
|
25
|
+
if (cap === undefined || cap < 1) {
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
let lim = perGraphLimits.get(graphId);
|
|
29
|
+
if (!lim) {
|
|
30
|
+
lim = pLimit(cap);
|
|
31
|
+
perGraphLimits.set(graphId, lim);
|
|
32
|
+
}
|
|
33
|
+
return lim;
|
|
34
|
+
};
|
|
35
|
+
return async (input) => {
|
|
36
|
+
const graphId = resolveGraphId(input);
|
|
37
|
+
const matrixCatalogId = resolveMatrixCatalogId(input);
|
|
38
|
+
const run = async () => {
|
|
39
|
+
const t0 = performance.now();
|
|
40
|
+
opts.metrics?.matrixClaimsStartedTotal.inc({ graph_id: graphId, matrix_catalog_id: matrixCatalogId });
|
|
41
|
+
opts.onInFlightDelta?.(graphId, 1);
|
|
42
|
+
try {
|
|
43
|
+
const result = await opts.inner(input);
|
|
44
|
+
opts.metrics?.matrixClaimsCompletedTotal.inc({ graph_id: graphId, status: result.status });
|
|
45
|
+
return result;
|
|
46
|
+
}
|
|
47
|
+
catch (err) {
|
|
48
|
+
opts.metrics?.matrixClaimsFailedTotal.inc({ graph_id: graphId });
|
|
49
|
+
throw err;
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
opts.onInFlightDelta?.(graphId, -1);
|
|
53
|
+
const secs = (performance.now() - t0) / 1000;
|
|
54
|
+
opts.metrics?.executeGraphDurationSeconds.observe({ graph_id: graphId }, secs);
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
const per = getPerGraphLimit(graphId);
|
|
58
|
+
return globalLimit(() => (per ? per(run) : run()));
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=wrap-execute-graph.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wrap-execute-graph.js","sourceRoot":"","sources":["../../src/worker/wrap-execute-graph.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,SAAS,CAAC;AAc7B,SAAS,cAAc,CAAC,KAAwB;IAC9C,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,EAAE,GAA0C,CAAC;IACtE,MAAM,OAAO,GAAG,GAAG,EAAE,OAAO,IAAI,GAAG,EAAE,QAAQ,CAAC;IAC9C,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,MAAM,EAAE,GAAG,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAC3B,OAAO,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AAClE,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAwB;IACtD,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,EAAE,GAA0C,CAAC;IACtE,MAAM,GAAG,GAAG,GAAG,EAAE,eAAe,IAAI,GAAG,EAAE,iBAAiB,CAAC;IAC3D,OAAO,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;AACrE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAoC;IAC1E,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAqC,CAAC;IACpE,MAAM,WAAW,GAAG,IAAI,CAAC,mBAAmB,IAAI,EAAE,CAAC;IAEnD,MAAM,gBAAgB,GAAG,CAAC,OAAe,EAAyC,EAAE;QAClF,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QACjC,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAClB,cAAc,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IAEF,OAAO,KAAK,EAAE,KAAwB,EAA+B,EAAE;QACrE,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,eAAe,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;QAEtD,MAAM,GAAG,GAAG,KAAK,IAAiC,EAAE;YAClD,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,EAAE,wBAAwB,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,CAAC,CAAC;YACtG,IAAI,CAAC,eAAe,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACvC,IAAI,CAAC,OAAO,EAAE,0BAA0B,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC3F,OAAO,MAAM,CAAC;YAChB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,OAAO,EAAE,uBAAuB,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;gBACjE,MAAM,GAAG,CAAC;YACZ,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,eAAe,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC;gBAC7C,IAAI,CAAC,OAAO,EAAE,2BAA2B,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC;YACjF,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,GAAG,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACtC,OAAO,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { ExellixMatrixPersistenceHandles, OpenExellixMatrixPersistenceOptions } from './collections.js';
|
|
2
|
+
/**
|
|
3
|
+
* Resolves {@link createExellixMatrixDataTier} from `@exellix/exellix-runtime` (≥ 3.1)
|
|
4
|
+
* so `@exellix/exellix-jobs` stays free of direct `@x12i/xronox-store` imports (see `docs/specs.md` §5–§6).
|
|
5
|
+
*/
|
|
6
|
+
export declare function openExellixMatrixPersistenceFromXmemoryStore(options: OpenExellixMatrixPersistenceOptions): Promise<ExellixMatrixPersistenceHandles>;
|
|
7
|
+
//# sourceMappingURL=xmemory-matrix-tier.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xmemory-matrix-tier.d.ts","sourceRoot":"","sources":["../src/xmemory-matrix-tier.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,+BAA+B,EAE/B,mCAAmC,EACpC,MAAM,kBAAkB,CAAC;AA8C1B;;;GAGG;AACH,wBAAsB,4CAA4C,CAChE,OAAO,EAAE,mCAAmC,GAC3C,OAAO,CAAC,+BAA+B,CAAC,CAuB1C"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
const FACTORY = 'createExellixMatrixDataTier';
|
|
2
|
+
const FACTORY_PACKAGE = '@exellix/exellix-runtime';
|
|
3
|
+
async function loadMatrixDataTierFactory() {
|
|
4
|
+
try {
|
|
5
|
+
const mod = (await import(FACTORY_PACKAGE));
|
|
6
|
+
const factory = mod[FACTORY];
|
|
7
|
+
if (typeof factory === 'function') {
|
|
8
|
+
return factory;
|
|
9
|
+
}
|
|
10
|
+
throw new Error(`"${FACTORY}" is not exported`);
|
|
11
|
+
}
|
|
12
|
+
catch (err) {
|
|
13
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
14
|
+
throw new Error(`@exellix/exellix-jobs: "${FACTORY}" is not available from ${FACTORY_PACKAGE} (peer ^3.1.0 required). ${message} ` +
|
|
15
|
+
'Wire persistence by passing pre-built handles to createMatrixWorker(), ' +
|
|
16
|
+
'or use openExellixMatrixPersistence() / openExellixMatrixPersistenceFromRuntimeFactories().');
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
function extractProbeHandles(tier) {
|
|
20
|
+
if (!tier || typeof tier !== 'object') {
|
|
21
|
+
return undefined;
|
|
22
|
+
}
|
|
23
|
+
const t = tier;
|
|
24
|
+
const probe = {};
|
|
25
|
+
if (t.executionStore !== undefined) {
|
|
26
|
+
probe.executionStore = t.executionStore;
|
|
27
|
+
}
|
|
28
|
+
if (t.configStore !== undefined) {
|
|
29
|
+
probe.configStore = t.configStore;
|
|
30
|
+
}
|
|
31
|
+
if (typeof t.executionMongoUri === 'string') {
|
|
32
|
+
probe.executionMongoUri = t.executionMongoUri;
|
|
33
|
+
}
|
|
34
|
+
if (typeof t.configMongoUri === 'string') {
|
|
35
|
+
probe.configMongoUri = t.configMongoUri;
|
|
36
|
+
}
|
|
37
|
+
return Object.keys(probe).length > 0 ? probe : undefined;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Resolves {@link createExellixMatrixDataTier} from `@exellix/exellix-runtime` (≥ 3.1)
|
|
41
|
+
* so `@exellix/exellix-jobs` stays free of direct `@x12i/xronox-store` imports (see `docs/specs.md` §5–§6).
|
|
42
|
+
*/
|
|
43
|
+
export async function openExellixMatrixPersistenceFromXmemoryStore(options) {
|
|
44
|
+
const factory = await loadMatrixDataTierFactory();
|
|
45
|
+
const tier = await factory(options);
|
|
46
|
+
const t = tier;
|
|
47
|
+
const { rows, failures, matrices, graphs, close } = t;
|
|
48
|
+
if (!rows || !failures || !matrices || !graphs || typeof close !== 'function') {
|
|
49
|
+
throw new Error(`@exellix/exellix-jobs: "${FACTORY}" returned an object missing rows, failures, matrices, graphs, or close().`);
|
|
50
|
+
}
|
|
51
|
+
if (t.init) {
|
|
52
|
+
await t.init();
|
|
53
|
+
}
|
|
54
|
+
const probe = extractProbeHandles(tier);
|
|
55
|
+
return {
|
|
56
|
+
rows,
|
|
57
|
+
failures,
|
|
58
|
+
snapshots: t.snapshots,
|
|
59
|
+
matrices,
|
|
60
|
+
graphs,
|
|
61
|
+
close: () => Promise.resolve(close.call(tier)),
|
|
62
|
+
...(probe ? { probe } : {}),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
//# sourceMappingURL=xmemory-matrix-tier.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xmemory-matrix-tier.js","sourceRoot":"","sources":["../src/xmemory-matrix-tier.ts"],"names":[],"mappings":"AAMA,MAAM,OAAO,GAAG,6BAA6B,CAAC;AAC9C,MAAM,eAAe,GAAG,0BAA0B,CAAC;AAEnD,KAAK,UAAU,yBAAyB;IAGtC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,eAAe,CAAC,CAA4B,CAAC;QACvE,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7B,IAAI,OAAO,OAAO,KAAK,UAAU,EAAE,CAAC;YAClC,OAAO,OAA6E,CAAC;QACvF,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,IAAI,OAAO,mBAAmB,CAAC,CAAC;IAClD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,IAAI,KAAK,CACb,2BAA2B,OAAO,2BAA2B,eAAe,4BAA4B,OAAO,GAAG;YAChH,yEAAyE;YACzE,6FAA6F,CAChG,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAa;IACxC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,CAAC,GAAG,IAA+B,CAAC;IAC1C,MAAM,KAAK,GAAyC,EAAE,CAAC;IACvD,IAAI,CAAC,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACnC,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC,cAAwE,CAAC;IACpG,CAAC;IACD,IAAI,CAAC,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QAChC,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC,WAAkE,CAAC;IAC3F,CAAC;IACD,IAAI,OAAO,CAAC,CAAC,iBAAiB,KAAK,QAAQ,EAAE,CAAC;QAC5C,KAAK,CAAC,iBAAiB,GAAG,CAAC,CAAC,iBAAiB,CAAC;IAChD,CAAC;IACD,IAAI,OAAO,CAAC,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;QACzC,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC;IAC1C,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3D,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,4CAA4C,CAChE,OAA4C;IAE5C,MAAM,OAAO,GAAG,MAAM,yBAAyB,EAAE,CAAC;IAClD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,IAAiF,CAAC;IAC5F,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IACtD,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,IAAI,OAAO,KAAK,KAAK,UAAU,EAAE,CAAC;QAC9E,MAAM,IAAI,KAAK,CACb,2BAA2B,OAAO,4EAA4E,CAC/G,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IACjB,CAAC;IACD,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IACxC,OAAO;QACL,IAAI;QACJ,QAAQ;QACR,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,QAAQ;QACR,MAAM;QACN,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Execution matrix worker service (historical)
|
|
2
|
+
|
|
3
|
+
This document is **historical**. Normative worker behavior lives in:
|
|
4
|
+
|
|
5
|
+
- **[`docs/specs.md`](./specs.md)** — operator HTTP, metrics, concurrency, discovery (in this package)
|
|
6
|
+
- **`@exellix/exellix-jobs` README** — programmatic API and CLI bootstrap
|
|
7
|
+
|
|
8
|
+
Use **`exellix-jobs serve`** with **`EXELLIX_JOBS_HOST_BOOTSTRAP`** (see [`examples/host-bootstrap.mjs`](../examples/host-bootstrap.mjs)) instead of wiring a bespoke worker process in `@exellix/exellix-runtime`.
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
openapi: 3.0.3
|
|
2
|
+
info:
|
|
3
|
+
title: Exellix Jobs Operator API
|
|
4
|
+
version: 1.0.0
|
|
5
|
+
description: |
|
|
6
|
+
Operator HTTP surface for `@exellix/exellix-jobs`. Base path is configurable via `JOBS_HTTP_PREFIX` (default `/v1`).
|
|
7
|
+
servers:
|
|
8
|
+
- url: /v1
|
|
9
|
+
paths:
|
|
10
|
+
/health:
|
|
11
|
+
get:
|
|
12
|
+
summary: Liveness or dependency readiness
|
|
13
|
+
parameters:
|
|
14
|
+
- in: query
|
|
15
|
+
name: probe
|
|
16
|
+
schema:
|
|
17
|
+
type: string
|
|
18
|
+
enum: [dependencies]
|
|
19
|
+
description: When `dependencies`, runs connectivity probe (Mongo / XMemory env / optional Activix).
|
|
20
|
+
responses:
|
|
21
|
+
'200':
|
|
22
|
+
description: Health payload
|
|
23
|
+
/graphs:
|
|
24
|
+
get:
|
|
25
|
+
summary: Graph summaries with matrix catalog references
|
|
26
|
+
responses:
|
|
27
|
+
'200':
|
|
28
|
+
description: Graph list
|
|
29
|
+
/workers/status:
|
|
30
|
+
get:
|
|
31
|
+
summary: Worker introspection
|
|
32
|
+
parameters:
|
|
33
|
+
- in: query
|
|
34
|
+
name: matrixCatalogId
|
|
35
|
+
schema:
|
|
36
|
+
type: string
|
|
37
|
+
description: Filter status to a single matrix catalog id.
|
|
38
|
+
responses:
|
|
39
|
+
'200':
|
|
40
|
+
description: Worker status including loop timestamps
|
|
41
|
+
/metrics:
|
|
42
|
+
get:
|
|
43
|
+
summary: Prometheus metrics
|
|
44
|
+
responses:
|
|
45
|
+
'200':
|
|
46
|
+
description: text/plain exposition format
|
|
47
|
+
/graphs/{graphId}/operational:
|
|
48
|
+
patch:
|
|
49
|
+
summary: Patch graph operational state
|
|
50
|
+
security:
|
|
51
|
+
- ApiKeyAuth: []
|
|
52
|
+
parameters:
|
|
53
|
+
- in: path
|
|
54
|
+
name: graphId
|
|
55
|
+
required: true
|
|
56
|
+
schema:
|
|
57
|
+
type: string
|
|
58
|
+
requestBody:
|
|
59
|
+
required: true
|
|
60
|
+
content:
|
|
61
|
+
application/json:
|
|
62
|
+
schema:
|
|
63
|
+
type: object
|
|
64
|
+
required: [expectedVersion]
|
|
65
|
+
properties:
|
|
66
|
+
expectedVersion:
|
|
67
|
+
type: integer
|
|
68
|
+
operationalState:
|
|
69
|
+
type: string
|
|
70
|
+
responses:
|
|
71
|
+
'200':
|
|
72
|
+
description: Updated graph record
|
|
73
|
+
'401':
|
|
74
|
+
description: Unauthorized
|
|
75
|
+
/graphs/{graphId}/publication:
|
|
76
|
+
patch:
|
|
77
|
+
summary: Patch graph publication status
|
|
78
|
+
security:
|
|
79
|
+
- ApiKeyAuth: []
|
|
80
|
+
parameters:
|
|
81
|
+
- in: path
|
|
82
|
+
name: graphId
|
|
83
|
+
required: true
|
|
84
|
+
schema:
|
|
85
|
+
type: string
|
|
86
|
+
requestBody:
|
|
87
|
+
required: true
|
|
88
|
+
content:
|
|
89
|
+
application/json:
|
|
90
|
+
schema:
|
|
91
|
+
type: object
|
|
92
|
+
required: [expectedVersion]
|
|
93
|
+
properties:
|
|
94
|
+
expectedVersion:
|
|
95
|
+
type: integer
|
|
96
|
+
publicationStatus:
|
|
97
|
+
type: string
|
|
98
|
+
responses:
|
|
99
|
+
'200':
|
|
100
|
+
description: Updated graph record
|
|
101
|
+
/workers/pause:
|
|
102
|
+
post:
|
|
103
|
+
summary: Pause new matrix claims (extension)
|
|
104
|
+
security:
|
|
105
|
+
- ApiKeyAuth: []
|
|
106
|
+
responses:
|
|
107
|
+
'200':
|
|
108
|
+
description: Paused
|
|
109
|
+
/workers/resume:
|
|
110
|
+
post:
|
|
111
|
+
summary: Resume matrix claims (extension)
|
|
112
|
+
security:
|
|
113
|
+
- ApiKeyAuth: []
|
|
114
|
+
responses:
|
|
115
|
+
'200':
|
|
116
|
+
description: Resumed
|
|
117
|
+
components:
|
|
118
|
+
securitySchemes:
|
|
119
|
+
ApiKeyAuth:
|
|
120
|
+
type: apiKey
|
|
121
|
+
in: header
|
|
122
|
+
name: x-exellix-jobs-api-key
|