@x12i/memorix-pipeline-orchestrator 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -0
- package/dist/browser.d.ts +9 -0
- package/dist/browser.d.ts.map +1 -0
- package/dist/browser.js +9 -0
- package/dist/browser.js.map +1 -0
- package/dist/catalog.d.ts +11 -0
- package/dist/catalog.d.ts.map +1 -0
- package/dist/catalog.js +11 -0
- package/dist/catalog.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +67 -0
- package/dist/cli.js.map +1 -0
- package/dist/collections.d.ts +25 -0
- package/dist/collections.d.ts.map +1 -0
- package/dist/collections.js +50 -0
- package/dist/collections.js.map +1 -0
- package/dist/examples/fetch-interfaces.d.ts +12 -0
- package/dist/examples/fetch-interfaces.d.ts.map +1 -0
- package/dist/examples/fetch-interfaces.js +206 -0
- package/dist/examples/fetch-interfaces.js.map +1 -0
- package/dist/graph.d.ts +16 -0
- package/dist/graph.d.ts.map +1 -0
- package/dist/graph.js +78 -0
- package/dist/graph.js.map +1 -0
- package/dist/handlers/ops.d.ts +8 -0
- package/dist/handlers/ops.d.ts.map +1 -0
- package/dist/handlers/ops.js +112 -0
- package/dist/handlers/ops.js.map +1 -0
- package/dist/handlers/pipelines.d.ts +9 -0
- package/dist/handlers/pipelines.d.ts.map +1 -0
- package/dist/handlers/pipelines.js +170 -0
- package/dist/handlers/pipelines.js.map +1 -0
- package/dist/handlers/runs.d.ts +13 -0
- package/dist/handlers/runs.d.ts.map +1 -0
- package/dist/handlers/runs.js +157 -0
- package/dist/handlers/runs.js.map +1 -0
- package/dist/handlers/schedules.d.ts +8 -0
- package/dist/handlers/schedules.d.ts.map +1 -0
- package/dist/handlers/schedules.js +123 -0
- package/dist/handlers/schedules.js.map +1 -0
- package/dist/handlers/shared.d.ts +11 -0
- package/dist/handlers/shared.d.ts.map +1 -0
- package/dist/handlers/shared.js +47 -0
- package/dist/handlers/shared.js.map +1 -0
- package/dist/handlers/steps.d.ts +8 -0
- package/dist/handlers/steps.d.ts.map +1 -0
- package/dist/handlers/steps.js +49 -0
- package/dist/handlers/steps.js.map +1 -0
- package/dist/handlers/templates.d.ts +8 -0
- package/dist/handlers/templates.d.ts.map +1 -0
- package/dist/handlers/templates.js +98 -0
- package/dist/handlers/templates.js.map +1 -0
- package/dist/http/server.d.ts +22 -0
- package/dist/http/server.d.ts.map +1 -0
- package/dist/http/server.js +106 -0
- package/dist/http/server.js.map +1 -0
- package/dist/index.d.ts +25 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24 -0
- package/dist/index.js.map +1 -0
- package/dist/pipeline-handlers.d.ts +12 -0
- package/dist/pipeline-handlers.d.ts.map +1 -0
- package/dist/pipeline-handlers.js +238 -0
- package/dist/pipeline-handlers.js.map +1 -0
- package/dist/runtime/build-invoke-envelope.d.ts +31 -0
- package/dist/runtime/build-invoke-envelope.d.ts.map +1 -0
- package/dist/runtime/build-invoke-envelope.js +258 -0
- package/dist/runtime/build-invoke-envelope.js.map +1 -0
- package/dist/runtime/checkpoints.d.ts +49 -0
- package/dist/runtime/checkpoints.d.ts.map +1 -0
- package/dist/runtime/checkpoints.js +45 -0
- package/dist/runtime/checkpoints.js.map +1 -0
- package/dist/runtime/durable-schedule-ticker.d.ts +23 -0
- package/dist/runtime/durable-schedule-ticker.d.ts.map +1 -0
- package/dist/runtime/durable-schedule-ticker.js +107 -0
- package/dist/runtime/durable-schedule-ticker.js.map +1 -0
- package/dist/runtime/execute-run.d.ts +30 -0
- package/dist/runtime/execute-run.d.ts.map +1 -0
- package/dist/runtime/execute-run.js +639 -0
- package/dist/runtime/execute-run.js.map +1 -0
- package/dist/runtime/execute-step.d.ts +22 -0
- package/dist/runtime/execute-step.d.ts.map +1 -0
- package/dist/runtime/execute-step.js +406 -0
- package/dist/runtime/execute-step.js.map +1 -0
- package/dist/runtime/leases.d.ts +45 -0
- package/dist/runtime/leases.d.ts.map +1 -0
- package/dist/runtime/leases.js +84 -0
- package/dist/runtime/leases.js.map +1 -0
- package/dist/runtime/memory-ports.d.ts +21 -0
- package/dist/runtime/memory-ports.d.ts.map +1 -0
- package/dist/runtime/memory-ports.js +99 -0
- package/dist/runtime/memory-ports.js.map +1 -0
- package/dist/runtime/object-content-resolver.d.ts +23 -0
- package/dist/runtime/object-content-resolver.d.ts.map +1 -0
- package/dist/runtime/object-content-resolver.js +117 -0
- package/dist/runtime/object-content-resolver.js.map +1 -0
- package/dist/runtime/pipeline-services-invoker.d.ts +10 -0
- package/dist/runtime/pipeline-services-invoker.d.ts.map +1 -0
- package/dist/runtime/pipeline-services-invoker.js +69 -0
- package/dist/runtime/pipeline-services-invoker.js.map +1 -0
- package/dist/runtime/ports.d.ts +74 -0
- package/dist/runtime/ports.d.ts.map +1 -0
- package/dist/runtime/ports.js +2 -0
- package/dist/runtime/ports.js.map +1 -0
- package/dist/runtime/preflight.d.ts +39 -0
- package/dist/runtime/preflight.d.ts.map +1 -0
- package/dist/runtime/preflight.js +234 -0
- package/dist/runtime/preflight.js.map +1 -0
- package/dist/runtime/run-context.d.ts +10 -0
- package/dist/runtime/run-context.d.ts.map +1 -0
- package/dist/runtime/run-context.js +72 -0
- package/dist/runtime/run-context.js.map +1 -0
- package/dist/runtime/sandbox-ports.d.ts +9 -0
- package/dist/runtime/sandbox-ports.d.ts.map +1 -0
- package/dist/runtime/sandbox-ports.js +216 -0
- package/dist/runtime/sandbox-ports.js.map +1 -0
- package/dist/runtime/schedule-ticker.d.ts +24 -0
- package/dist/runtime/schedule-ticker.d.ts.map +1 -0
- package/dist/runtime/schedule-ticker.js +155 -0
- package/dist/runtime/schedule-ticker.js.map +1 -0
- package/dist/runtime/schedule-utils.d.ts +11 -0
- package/dist/runtime/schedule-utils.d.ts.map +1 -0
- package/dist/runtime/schedule-utils.js +54 -0
- package/dist/runtime/schedule-utils.js.map +1 -0
- package/dist/seeds/cortex-xdr.d.ts +11 -0
- package/dist/seeds/cortex-xdr.d.ts.map +1 -0
- package/dist/seeds/cortex-xdr.js +1759 -0
- package/dist/seeds/cortex-xdr.js.map +1 -0
- package/dist/seeds/firewall-topology.d.ts +11 -0
- package/dist/seeds/firewall-topology.d.ts.map +1 -0
- package/dist/seeds/firewall-topology.js +1310 -0
- package/dist/seeds/firewall-topology.js.map +1 -0
- package/dist/seeds/platform-smoke.d.ts +22 -0
- package/dist/seeds/platform-smoke.d.ts.map +1 -0
- package/dist/seeds/platform-smoke.js +314 -0
- package/dist/seeds/platform-smoke.js.map +1 -0
- package/dist/seeds/tenable-nessus.d.ts +12 -0
- package/dist/seeds/tenable-nessus.d.ts.map +1 -0
- package/dist/seeds/tenable-nessus.js +880 -0
- package/dist/seeds/tenable-nessus.js.map +1 -0
- package/dist/store/catalox.d.ts +15 -0
- package/dist/store/catalox.d.ts.map +1 -0
- package/dist/store/catalox.js +581 -0
- package/dist/store/catalox.js.map +1 -0
- package/dist/store/memory.d.ts +3 -0
- package/dist/store/memory.d.ts.map +1 -0
- package/dist/store/memory.js +352 -0
- package/dist/store/memory.js.map +1 -0
- package/dist/store/stack.d.ts +13 -0
- package/dist/store/stack.d.ts.map +1 -0
- package/dist/store/stack.js +72 -0
- package/dist/store/stack.js.map +1 -0
- package/dist/store/types.d.ts +50 -0
- package/dist/store/types.d.ts.map +1 -0
- package/dist/store/types.js +45 -0
- package/dist/store/types.js.map +1 -0
- package/dist/types.d.ts +324 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +12 -0
- package/dist/types.js.map +1 -0
- package/dist/validate.d.ts +14 -0
- package/dist/validate.d.ts.map +1 -0
- package/dist/validate.js +263 -0
- package/dist/validate.js.map +1 -0
- package/package.json +72 -0
package/README.md
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# `@x12i/memorix-pipeline-orchestrator`
|
|
2
|
+
|
|
3
|
+
Generic **pipeline step** contract, Catalox-backed registry, and run/execute engine for Memorix.
|
|
4
|
+
|
|
5
|
+
A step is the atomic unit of pipeline work (connector / data / AI). The same shape covers fetch workers, transforms, and AI services — aligned with `@x12i/memorix-pipeline-services` taxonomy.
|
|
6
|
+
|
|
7
|
+
## Capabilities
|
|
8
|
+
|
|
9
|
+
| Layer | What |
|
|
10
|
+
|-------|------|
|
|
11
|
+
| Types + validation + graph | `PipelineStep`, `validatePipelineStepGraph`, `topologicalOrder` |
|
|
12
|
+
| Registry | Catalox catalogs for steps, templates, pipeline definitions, schedules, runs |
|
|
13
|
+
| Runtime | deps → inputs → apiCalls/invoke (typed envelope) → raw → normalize → snapshot; simulation (sandbox ports); schedule ticker |
|
|
14
|
+
| HTTP | Fastify on **:5187**, prefix `/api/pipeline-orchestrator` |
|
|
15
|
+
|
|
16
|
+
Invoke calls to `@x12i/memorix-pipeline-services` use `PipelineServiceRequest` / `PipelineServiceResponse` (sparse envelope + per-type `payload`). See that package’s README for contracts.
|
|
17
|
+
|
|
18
|
+
AI step authoring (object/content selectors, request templates, suggestions): [docs/ai-steps.md](./docs/ai-steps.md).
|
|
19
|
+
|
|
20
|
+
## HTTP
|
|
21
|
+
|
|
22
|
+
| Method | Path | Role |
|
|
23
|
+
|--------|------|------|
|
|
24
|
+
| `GET` | `/health` | health |
|
|
25
|
+
| `GET/POST` | `/api/pipeline-orchestrator/steps` | list / create |
|
|
26
|
+
| `GET/PUT/DELETE` | `/api/pipeline-orchestrator/steps/:stepId` | CRUD |
|
|
27
|
+
| `GET/POST` | `/api/pipeline-orchestrator/templates` | list / create templates |
|
|
28
|
+
| `GET/PUT/DELETE` | `/api/pipeline-orchestrator/templates/:id` | template CRUD |
|
|
29
|
+
| `GET/POST` | `/api/pipeline-orchestrator/pipelines` | list / create |
|
|
30
|
+
| `GET/PUT/DELETE` | `/api/pipeline-orchestrator/pipelines/:id` | CRUD (optional `templateId`) |
|
|
31
|
+
| `POST` | `/api/pipeline-orchestrator/pipelines/:id/validate` | graph validate |
|
|
32
|
+
| `GET/POST` | `/api/pipeline-orchestrator/schedules` | list / activate schedule |
|
|
33
|
+
| `GET/PUT/DELETE` | `/api/pipeline-orchestrator/schedules/:id` | schedule CRUD |
|
|
34
|
+
| `POST` | `/api/pipeline-orchestrator/runs?wait=1` | start run (`mode`: `ondemand` \| `simulation`) |
|
|
35
|
+
| `GET` | `/api/pipeline-orchestrator/runs/:runId` | get run |
|
|
36
|
+
| `POST` | `/api/pipeline-orchestrator/runs/:runId/cancel` | cancel |
|
|
37
|
+
| `POST` | `/api/pipeline-orchestrator/runs/:runId/retry` | retry from step (`fromStepId`) |
|
|
38
|
+
|
|
39
|
+
## Dev
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
cd memorix-pipeline-orchestrator
|
|
43
|
+
npm install
|
|
44
|
+
npm run dev
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Env: same Catalox/Mongo vars as other Memorix services. Optional `PIPELINE_SERVICES_URL` (default `http://127.0.0.1:5185/api/pipeline-services`) for worker invoke. Optional `PIPELINE_ORCHESTRATOR_TOKEN` for bearer auth.
|
|
48
|
+
|
|
49
|
+
## Catalogs
|
|
50
|
+
|
|
51
|
+
- `memorix-pipeline-steps`
|
|
52
|
+
- `memorix-pipeline-templates`
|
|
53
|
+
- `memorix-pipeline-definitions`
|
|
54
|
+
- `memorix-pipeline-schedules`
|
|
55
|
+
- `memorix-pipeline-runs`
|
|
56
|
+
|
|
57
|
+
When the HTTP server starts, an in-process schedule ticker (every 30s) fires due
|
|
58
|
+
active schedules as real on-demand runs. The ticker only runs while the process
|
|
59
|
+
is up.
|
|
60
|
+
|
|
61
|
+
## Distinct from
|
|
62
|
+
|
|
63
|
+
| Package | Role |
|
|
64
|
+
|---------|------|
|
|
65
|
+
| `memorix-pipeline-services` | Worker registry + HTTP invoke |
|
|
66
|
+
| `memorix-pipeline` | Entity enrichment stages |
|
|
67
|
+
| **this package** | Step graphs + runs |
|
|
68
|
+
|
|
69
|
+
## Scripts
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
npm test
|
|
73
|
+
npm run build
|
|
74
|
+
|
|
75
|
+
# Upsert platform-smoke template + steps + pipeline (agent-platform-smoke)
|
|
76
|
+
npm run seed:platform-smoke
|
|
77
|
+
|
|
78
|
+
# Regenerate platform-smoke catalox-seeds JSON from src/seeds/platform-smoke.ts
|
|
79
|
+
npm run generate:seed-json
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### Integration pack orchestration seeds (external)
|
|
83
|
+
|
|
84
|
+
MOVE-OUT integration pipeline steps, definitions, and templates (firewall topology,
|
|
85
|
+
Cortex XDR, Tenable Nessus, vuln-analytics) live in
|
|
86
|
+
`.operational/package-seed-packs/<pack-id>/` and are applied via
|
|
87
|
+
`@x12i/memorix-package-seed`. See
|
|
88
|
+
[`.operational/package-seed-packs/README.md`](../.operational/package-seed-packs/README.md).
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
cd memorix-package-seed
|
|
92
|
+
node dist/cli.js validate --dir ../.operational/package-seed-packs/firewall-topology
|
|
93
|
+
node dist/cli.js apply --dir ../.operational/package-seed-packs/firewall-topology --agent agent-firewall-topology
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Platform-smoke authored source: `src/seeds/platform-smoke.ts` (also written to
|
|
97
|
+
`catalox-seeds/inputs/platform-smoke-*.json` on seed/generate runs). Template
|
|
98
|
+
`tpl_platform_smoke` groups the synthetic acceptance pipeline for
|
|
99
|
+
`agent-platform-smoke`.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Browser-safe surface: types, validation, graph helpers only (no Node/HTTP/Catalox). */
|
|
2
|
+
export { MEMORIX_PIPELINE_STEPS_CATALOG, MEMORIX_PIPELINE_DEFINITIONS_CATALOG, MEMORIX_PIPELINE_RUNS_CATALOG, MEMORIX_PIPELINE_TEMPLATES_CATALOG, MEMORIX_PIPELINE_SCHEDULES_CATALOG, } from "./catalog.js";
|
|
3
|
+
export { PIPELINE_STEP_LIFECYCLE_STATUSES, type PipelineStepCategory, type PipelineStageName, type PipelineStepLifecycleStatus, type StepInputSource, type StepRequiredInput, type StepApiCall, type StepRawSaveMode, type StepRawOutput, type StepNormalizedOutput, type StepSnapshotFinalization, type StepSnapshotOutput, type StepStatusBehavior, type StepErrorBehavior, type PipelineStepInvoke, type PipelineStepRequestTemplate, type ObjectTarget, type PipelineObjectTypeRef, type PipelineStepObjectTypes, type PipelineStep, type PipelineStepGraph, type PipelineTemplate, type RegisterPipelineTemplateInput, type UpdatePipelineTemplateInput, type PipelineDefinition, type RegisterPipelineDefinitionInput, type UpdatePipelineDefinitionInput, type PipelineRunStatus, type PipelineRunMode, type StepRun, type PipelineRun, type StartPipelineRunInput, type PipelineScheduleFrequency, type PipelineScheduleStatus, type PipelineSchedule, type RegisterPipelineScheduleInput, type UpdatePipelineScheduleInput, type PipelineStepRecord, } from "./types.js";
|
|
4
|
+
export { rawCollection, snapshotsCollection, contentCollection, resolveRawCollection, resolveSnapshotsCollection, } from "./collections.js";
|
|
5
|
+
export { validatePipelineStep, validatePipelineStepGraph, type PipelineStepValidationIssue, type PipelineStepValidationResult, } from "./validate.js";
|
|
6
|
+
export { topologicalOrder, readySteps, dependenciesSatisfied, PipelineStepGraphError, } from "./graph.js";
|
|
7
|
+
export { getContextPath, setContextPath, mergeRunContext, isMissingContextValue, missingRequiredRunInputs, } from "./runtime/run-context.js";
|
|
8
|
+
export { computeNextRunAt, requiredInputsForMode } from "./runtime/schedule-utils.js";
|
|
9
|
+
//# sourceMappingURL=browser.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":"AAAA,0FAA0F;AAE1F,OAAO,EACL,8BAA8B,EAC9B,oCAAoC,EACpC,6BAA6B,EAC7B,kCAAkC,EAClC,kCAAkC,GACnC,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,gCAAgC,EAChC,KAAK,oBAAoB,EACzB,KAAK,iBAAiB,EACtB,KAAK,2BAA2B,EAChC,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,KAAK,wBAAwB,EAC7B,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,2BAA2B,EAChC,KAAK,YAAY,EACjB,KAAK,qBAAqB,EAC1B,KAAK,uBAAuB,EAC5B,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,6BAA6B,EAClC,KAAK,2BAA2B,EAChC,KAAK,kBAAkB,EACvB,KAAK,+BAA+B,EACpC,KAAK,6BAA6B,EAClC,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,OAAO,EACZ,KAAK,WAAW,EAChB,KAAK,qBAAqB,EAC1B,KAAK,yBAAyB,EAC9B,KAAK,sBAAsB,EAC3B,KAAK,gBAAgB,EACrB,KAAK,6BAA6B,EAClC,KAAK,2BAA2B,EAChC,KAAK,kBAAkB,GACxB,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,aAAa,EACb,mBAAmB,EACnB,iBAAiB,EACjB,oBAAoB,EACpB,0BAA0B,GAC3B,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,GAClC,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,cAAc,EACd,cAAc,EACd,eAAe,EACf,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
|
package/dist/browser.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Browser-safe surface: types, validation, graph helpers only (no Node/HTTP/Catalox). */
|
|
2
|
+
export { MEMORIX_PIPELINE_STEPS_CATALOG, MEMORIX_PIPELINE_DEFINITIONS_CATALOG, MEMORIX_PIPELINE_RUNS_CATALOG, MEMORIX_PIPELINE_TEMPLATES_CATALOG, MEMORIX_PIPELINE_SCHEDULES_CATALOG, } from "./catalog.js";
|
|
3
|
+
export { PIPELINE_STEP_LIFECYCLE_STATUSES, } from "./types.js";
|
|
4
|
+
export { rawCollection, snapshotsCollection, contentCollection, resolveRawCollection, resolveSnapshotsCollection, } from "./collections.js";
|
|
5
|
+
export { validatePipelineStep, validatePipelineStepGraph, } from "./validate.js";
|
|
6
|
+
export { topologicalOrder, readySteps, dependenciesSatisfied, PipelineStepGraphError, } from "./graph.js";
|
|
7
|
+
export { getContextPath, setContextPath, mergeRunContext, isMissingContextValue, missingRequiredRunInputs, } from "./runtime/run-context.js";
|
|
8
|
+
export { computeNextRunAt, requiredInputsForMode } from "./runtime/schedule-utils.js";
|
|
9
|
+
//# sourceMappingURL=browser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"browser.js","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":"AAAA,0FAA0F;AAE1F,OAAO,EACL,8BAA8B,EAC9B,oCAAoC,EACpC,6BAA6B,EAC7B,kCAAkC,EAClC,kCAAkC,GACnC,MAAM,cAAc,CAAC;AAEtB,OAAO,EACL,gCAAgC,GAsCjC,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,aAAa,EACb,mBAAmB,EACnB,iBAAiB,EACjB,oBAAoB,EACpB,0BAA0B,GAC3B,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EACL,oBAAoB,EACpB,yBAAyB,GAG1B,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,cAAc,EACd,cAAc,EACd,eAAe,EACf,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Catalox catalog id for pipeline step definitions. */
|
|
2
|
+
export declare const MEMORIX_PIPELINE_STEPS_CATALOG = "memorix-pipeline-steps";
|
|
3
|
+
/** Catalox catalog id for named pipeline definitions (ordered step ids). */
|
|
4
|
+
export declare const MEMORIX_PIPELINE_DEFINITIONS_CATALOG = "memorix-pipeline-definitions";
|
|
5
|
+
/** Catalox catalog id for pipeline run records. */
|
|
6
|
+
export declare const MEMORIX_PIPELINE_RUNS_CATALOG = "memorix-pipeline-runs";
|
|
7
|
+
/** Catalox catalog id for reusable pipeline templates (shapes). */
|
|
8
|
+
export declare const MEMORIX_PIPELINE_TEMPLATES_CATALOG = "memorix-pipeline-templates";
|
|
9
|
+
/** Catalox catalog id for recurring pipeline schedules. */
|
|
10
|
+
export declare const MEMORIX_PIPELINE_SCHEDULES_CATALOG = "memorix-pipeline-schedules";
|
|
11
|
+
//# sourceMappingURL=catalog.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"catalog.d.ts","sourceRoot":"","sources":["../src/catalog.ts"],"names":[],"mappings":"AAAA,wDAAwD;AACxD,eAAO,MAAM,8BAA8B,2BAA2B,CAAC;AACvE,4EAA4E;AAC5E,eAAO,MAAM,oCAAoC,iCAAiC,CAAC;AACnF,mDAAmD;AACnD,eAAO,MAAM,6BAA6B,0BAA0B,CAAC;AACrE,mEAAmE;AACnE,eAAO,MAAM,kCAAkC,+BAA+B,CAAC;AAC/E,2DAA2D;AAC3D,eAAO,MAAM,kCAAkC,+BAA+B,CAAC"}
|
package/dist/catalog.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Catalox catalog id for pipeline step definitions. */
|
|
2
|
+
export const MEMORIX_PIPELINE_STEPS_CATALOG = "memorix-pipeline-steps";
|
|
3
|
+
/** Catalox catalog id for named pipeline definitions (ordered step ids). */
|
|
4
|
+
export const MEMORIX_PIPELINE_DEFINITIONS_CATALOG = "memorix-pipeline-definitions";
|
|
5
|
+
/** Catalox catalog id for pipeline run records. */
|
|
6
|
+
export const MEMORIX_PIPELINE_RUNS_CATALOG = "memorix-pipeline-runs";
|
|
7
|
+
/** Catalox catalog id for reusable pipeline templates (shapes). */
|
|
8
|
+
export const MEMORIX_PIPELINE_TEMPLATES_CATALOG = "memorix-pipeline-templates";
|
|
9
|
+
/** Catalox catalog id for recurring pipeline schedules. */
|
|
10
|
+
export const MEMORIX_PIPELINE_SCHEDULES_CATALOG = "memorix-pipeline-schedules";
|
|
11
|
+
//# sourceMappingURL=catalog.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"catalog.js","sourceRoot":"","sources":["../src/catalog.ts"],"names":[],"mappings":"AAAA,wDAAwD;AACxD,MAAM,CAAC,MAAM,8BAA8B,GAAG,wBAAwB,CAAC;AACvE,4EAA4E;AAC5E,MAAM,CAAC,MAAM,oCAAoC,GAAG,8BAA8B,CAAC;AACnF,mDAAmD;AACnD,MAAM,CAAC,MAAM,6BAA6B,GAAG,uBAAuB,CAAC;AACrE,mEAAmE;AACnE,MAAM,CAAC,MAAM,kCAAkC,GAAG,4BAA4B,CAAC;AAC/E,2DAA2D;AAC3D,MAAM,CAAC,MAAM,kCAAkC,GAAG,4BAA4B,CAAC"}
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { startMemorixPipelineOrchestratorHttp } from "./http/server.js";
|
|
5
|
+
import { closePipelineOrchestratorStack } from "./store/stack.js";
|
|
6
|
+
function loadDotEnv() {
|
|
7
|
+
const candidates = [
|
|
8
|
+
join(process.cwd(), ".env"),
|
|
9
|
+
join(process.cwd(), "..", "memorix-catalox-api", ".env"),
|
|
10
|
+
join(process.cwd(), "..", "memorix-explorer-api", ".env"),
|
|
11
|
+
];
|
|
12
|
+
for (const path of candidates) {
|
|
13
|
+
if (!existsSync(path))
|
|
14
|
+
continue;
|
|
15
|
+
for (const line of readFileSync(path, "utf8").split("\n")) {
|
|
16
|
+
const trimmed = line.trim();
|
|
17
|
+
if (!trimmed || trimmed.startsWith("#"))
|
|
18
|
+
continue;
|
|
19
|
+
const eq = trimmed.indexOf("=");
|
|
20
|
+
if (eq === -1)
|
|
21
|
+
continue;
|
|
22
|
+
const key = trimmed.slice(0, eq).trim();
|
|
23
|
+
const val = trimmed.slice(eq + 1).trim();
|
|
24
|
+
if (!process.env[key]?.trim())
|
|
25
|
+
process.env[key] = val;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function resolveMonorepoRoot() {
|
|
30
|
+
if (process.env.MEMORIX_MONOREPO_ROOT?.trim())
|
|
31
|
+
return;
|
|
32
|
+
let dir = process.cwd();
|
|
33
|
+
for (let i = 0; i < 12; i++) {
|
|
34
|
+
const marker = join(dir, ".operational/PACKAGE-OWNERSHIP.md");
|
|
35
|
+
if (existsSync(marker)) {
|
|
36
|
+
process.env.MEMORIX_MONOREPO_ROOT = dir;
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const parent = dirname(dir);
|
|
40
|
+
if (parent === dir)
|
|
41
|
+
break;
|
|
42
|
+
dir = parent;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
loadDotEnv();
|
|
46
|
+
resolveMonorepoRoot();
|
|
47
|
+
const cmd = process.argv[2];
|
|
48
|
+
async function main() {
|
|
49
|
+
if (cmd === "serve") {
|
|
50
|
+
const { url } = await startMemorixPipelineOrchestratorHttp();
|
|
51
|
+
console.log(`memorix-pipeline-orchestrator listening on ${url}`);
|
|
52
|
+
process.on("SIGINT", () => {
|
|
53
|
+
void closePipelineOrchestratorStack().finally(() => process.exit(0));
|
|
54
|
+
});
|
|
55
|
+
process.on("SIGTERM", () => {
|
|
56
|
+
void closePipelineOrchestratorStack().finally(() => process.exit(0));
|
|
57
|
+
});
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
console.log("Usage: memorix-pipeline-orchestrator serve");
|
|
61
|
+
process.exit(cmd ? 1 : 0);
|
|
62
|
+
}
|
|
63
|
+
main().catch((err) => {
|
|
64
|
+
console.error(err);
|
|
65
|
+
process.exit(1);
|
|
66
|
+
});
|
|
67
|
+
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,oCAAoC,EAAE,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAE,8BAA8B,EAAE,MAAM,kBAAkB,CAAC;AAElE,SAAS,UAAU;IACjB,MAAM,UAAU,GAAG;QACjB,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC;QAC3B,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,CAAC;QACxD,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,CAAC;KAC1D,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAChC,KAAK,MAAM,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAClD,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAAE,SAAS;YACxB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACzC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE;gBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;QACxD,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB;IAC1B,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,IAAI,EAAE;QAAE,OAAO;IACtD,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,mCAAmC,CAAC,CAAC;QAC9D,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,qBAAqB,GAAG,GAAG,CAAC;YACxC,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,MAAM,KAAK,GAAG;YAAE,MAAM;QAC1B,GAAG,GAAG,MAAM,CAAC;IACf,CAAC;AACH,CAAC;AAED,UAAU,EAAE,CAAC;AACb,mBAAmB,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAE5B,KAAK,UAAU,IAAI;IACjB,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;QACpB,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,oCAAoC,EAAE,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,8CAA8C,GAAG,EAAE,CAAC,CAAC;QACjE,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YACxB,KAAK,8BAA8B,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;YACzB,KAAK,8BAA8B,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QACH,OAAO;IACT,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/** Derive `<objectType>-raw` collection name. */
|
|
2
|
+
export declare function rawCollection(objectType: string): string;
|
|
3
|
+
/** Derive `<objectType>-snapshots` collection name. */
|
|
4
|
+
export declare function snapshotsCollection(objectType: string): string;
|
|
5
|
+
/**
|
|
6
|
+
* Derive `<objectType>-<contentType>` collection name.
|
|
7
|
+
* `contentType` is a descriptor key without a leading hyphen.
|
|
8
|
+
*/
|
|
9
|
+
export declare function contentCollection(objectType: string, contentType: string): string;
|
|
10
|
+
/** Resolve raw collection from step (explicit or derived). */
|
|
11
|
+
export declare function resolveRawCollection(step: {
|
|
12
|
+
objectType?: string;
|
|
13
|
+
rawOutput?: {
|
|
14
|
+
collection?: string;
|
|
15
|
+
};
|
|
16
|
+
}): string | undefined;
|
|
17
|
+
/** Resolve snapshots / content collection from step (explicit, contentType, or snapshots). */
|
|
18
|
+
export declare function resolveSnapshotsCollection(step: {
|
|
19
|
+
objectType?: string;
|
|
20
|
+
snapshotOutput?: {
|
|
21
|
+
collection?: string;
|
|
22
|
+
contentType?: string;
|
|
23
|
+
};
|
|
24
|
+
}): string | undefined;
|
|
25
|
+
//# sourceMappingURL=collections.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"collections.d.ts","sourceRoot":"","sources":["../src/collections.ts"],"names":[],"mappings":"AAAA,iDAAiD;AACjD,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAIxD;AAED,uDAAuD;AACvD,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAI9D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CASjF;AAED,8DAA8D;AAC9D,wBAAgB,oBAAoB,CAAC,IAAI,EAAE;IACzC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACrC,GAAG,MAAM,GAAG,SAAS,CAIrB;AAED,8FAA8F;AAC9F,wBAAgB,0BAA0B,CAAC,IAAI,EAAE;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAChE,GAAG,MAAM,GAAG,SAAS,CAOrB"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/** Derive `<objectType>-raw` collection name. */
|
|
2
|
+
export function rawCollection(objectType) {
|
|
3
|
+
const ot = objectType.trim();
|
|
4
|
+
if (!ot)
|
|
5
|
+
throw new Error("objectType is required");
|
|
6
|
+
return `${ot}-raw`;
|
|
7
|
+
}
|
|
8
|
+
/** Derive `<objectType>-snapshots` collection name. */
|
|
9
|
+
export function snapshotsCollection(objectType) {
|
|
10
|
+
const ot = objectType.trim();
|
|
11
|
+
if (!ot)
|
|
12
|
+
throw new Error("objectType is required");
|
|
13
|
+
return `${ot}-snapshots`;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Derive `<objectType>-<contentType>` collection name.
|
|
17
|
+
* `contentType` is a descriptor key without a leading hyphen.
|
|
18
|
+
*/
|
|
19
|
+
export function contentCollection(objectType, contentType) {
|
|
20
|
+
const ot = objectType.trim();
|
|
21
|
+
if (!ot)
|
|
22
|
+
throw new Error("objectType is required");
|
|
23
|
+
const key = contentType.trim();
|
|
24
|
+
if (!key)
|
|
25
|
+
throw new Error("contentType is required");
|
|
26
|
+
if (key.startsWith("-")) {
|
|
27
|
+
throw new Error(`contentType must not start with '-': ${contentType}`);
|
|
28
|
+
}
|
|
29
|
+
return `${ot}-${key}`;
|
|
30
|
+
}
|
|
31
|
+
/** Resolve raw collection from step (explicit or derived). */
|
|
32
|
+
export function resolveRawCollection(step) {
|
|
33
|
+
if (step.rawOutput?.collection?.trim())
|
|
34
|
+
return step.rawOutput.collection.trim();
|
|
35
|
+
if (step.objectType?.trim())
|
|
36
|
+
return rawCollection(step.objectType);
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
39
|
+
/** Resolve snapshots / content collection from step (explicit, contentType, or snapshots). */
|
|
40
|
+
export function resolveSnapshotsCollection(step) {
|
|
41
|
+
if (step.snapshotOutput?.collection?.trim())
|
|
42
|
+
return step.snapshotOutput.collection.trim();
|
|
43
|
+
if (step.objectType?.trim() && step.snapshotOutput?.contentType?.trim()) {
|
|
44
|
+
return contentCollection(step.objectType, step.snapshotOutput.contentType);
|
|
45
|
+
}
|
|
46
|
+
if (step.objectType?.trim())
|
|
47
|
+
return snapshotsCollection(step.objectType);
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=collections.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"collections.js","sourceRoot":"","sources":["../src/collections.ts"],"names":[],"mappings":"AAAA,iDAAiD;AACjD,MAAM,UAAU,aAAa,CAAC,UAAkB;IAC9C,MAAM,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IACnD,OAAO,GAAG,EAAE,MAAM,CAAC;AACrB,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,mBAAmB,CAAC,UAAkB;IACpD,MAAM,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IACnD,OAAO,GAAG,EAAE,YAAY,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,UAAkB,EAAE,WAAmB;IACvE,MAAM,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;IAC/B,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IACrD,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,wCAAwC,WAAW,EAAE,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,GAAG,EAAE,IAAI,GAAG,EAAE,CAAC;AACxB,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,oBAAoB,CAAC,IAGpC;IACC,IAAI,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;IAChF,IAAI,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE;QAAE,OAAO,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,0BAA0B,CAAC,IAG1C;IACC,IAAI,IAAI,CAAC,cAAc,EAAE,UAAU,EAAE,IAAI,EAAE;QAAE,OAAO,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;IAC1F,IAAI,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,IAAI,EAAE,EAAE,CAAC;QACxE,OAAO,iBAAiB,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE;QAAE,OAAO,mBAAmB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACzE,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { PipelineStep } from "../types.js";
|
|
2
|
+
/** Connector-style step: fetch interfaces via API, then normalize + snapshot. */
|
|
3
|
+
export declare const fetchInterfacesStep: PipelineStep;
|
|
4
|
+
/** Data-style step: normalize vendor raw into common schema via a worker. */
|
|
5
|
+
export declare const normalizeInterfacesStep: PipelineStep;
|
|
6
|
+
/** Minimal auth + scopes stubs so the fetch-interfaces example graph can validate. */
|
|
7
|
+
export declare const authenticateStep: PipelineStep;
|
|
8
|
+
export declare const fetchDeviceIdentityStep: PipelineStep;
|
|
9
|
+
export declare const fetchScopesStep: PipelineStep;
|
|
10
|
+
/** Example connector + data graph for docs and tests. */
|
|
11
|
+
export declare const exampleFirewallPipelineSteps: PipelineStep[];
|
|
12
|
+
//# sourceMappingURL=fetch-interfaces.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch-interfaces.d.ts","sourceRoot":"","sources":["../../src/examples/fetch-interfaces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,iFAAiF;AACjF,eAAO,MAAM,mBAAmB,EAAE,YA2DjC,CAAC;AAEF,6EAA6E;AAC7E,eAAO,MAAM,uBAAuB,EAAE,YA0CrC,CAAC;AAEF,sFAAsF;AACtF,eAAO,MAAM,gBAAgB,EAAE,YA0B9B,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,YA+BrC,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,YA+B7B,CAAC;AAEF,yDAAyD;AACzD,eAAO,MAAM,4BAA4B,EAAE,YAAY,EAMtD,CAAC"}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
/** Connector-style step: fetch interfaces via API, then normalize + snapshot. */
|
|
2
|
+
export const fetchInterfacesStep = {
|
|
3
|
+
id: "fetch-interfaces",
|
|
4
|
+
name: "Fetch Interfaces",
|
|
5
|
+
purpose: "Fetch firewall interfaces as base topology attachment points.",
|
|
6
|
+
category: "connector",
|
|
7
|
+
serviceType: "entity-provider",
|
|
8
|
+
vendor: "generic",
|
|
9
|
+
objectType: "firewall-interfaces",
|
|
10
|
+
dependsOn: ["authenticate", "fetch-device-identity", "fetch-scopes"],
|
|
11
|
+
requiredInputs: [
|
|
12
|
+
{ name: "baseUrl", source: "connector-config", required: true },
|
|
13
|
+
{
|
|
14
|
+
name: "authToken",
|
|
15
|
+
source: "step-output",
|
|
16
|
+
fromStep: "authenticate",
|
|
17
|
+
required: true,
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
name: "scope",
|
|
21
|
+
source: "step-output",
|
|
22
|
+
fromStep: "fetch-scopes",
|
|
23
|
+
required: false,
|
|
24
|
+
},
|
|
25
|
+
],
|
|
26
|
+
apiCalls: [
|
|
27
|
+
{
|
|
28
|
+
method: "GET",
|
|
29
|
+
url: "<baseUrl>/api/v1/interfaces",
|
|
30
|
+
auth: "authToken",
|
|
31
|
+
pagination: false,
|
|
32
|
+
},
|
|
33
|
+
],
|
|
34
|
+
rawOutput: {
|
|
35
|
+
collection: "firewall-interfaces-raw",
|
|
36
|
+
saveMode: "append",
|
|
37
|
+
includeMetadata: true,
|
|
38
|
+
},
|
|
39
|
+
normalizedOutput: {
|
|
40
|
+
enabled: true,
|
|
41
|
+
outputType: "FirewallInterface",
|
|
42
|
+
},
|
|
43
|
+
snapshotOutput: {
|
|
44
|
+
collection: "firewall-interfaces-snapshots",
|
|
45
|
+
saveMode: "snapshot",
|
|
46
|
+
finalization: "after-zone-and-routing-resolution",
|
|
47
|
+
},
|
|
48
|
+
statusBehavior: {
|
|
49
|
+
idempotent: true,
|
|
50
|
+
retryable: true,
|
|
51
|
+
partialAllowed: false,
|
|
52
|
+
completionCondition: "all-scopes-fetched",
|
|
53
|
+
},
|
|
54
|
+
errorBehavior: {
|
|
55
|
+
onApiError: "retry",
|
|
56
|
+
maxRetries: 3,
|
|
57
|
+
onMissingDependency: "block",
|
|
58
|
+
onPartialData: "mark-partial",
|
|
59
|
+
onNormalizationError: "save-raw-and-fail-normalization",
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
/** Data-style step: normalize vendor raw into common schema via a worker. */
|
|
63
|
+
export const normalizeInterfacesStep = {
|
|
64
|
+
id: "normalize-interfaces",
|
|
65
|
+
name: "Normalize Interfaces",
|
|
66
|
+
purpose: "Convert vendor-specific interface raw payloads into FirewallInterface.",
|
|
67
|
+
category: "data",
|
|
68
|
+
serviceType: "normalizer",
|
|
69
|
+
serviceId: "firewall-interface-normalizer",
|
|
70
|
+
objectType: "firewall-interfaces",
|
|
71
|
+
dependsOn: ["fetch-interfaces"],
|
|
72
|
+
requiredInputs: [
|
|
73
|
+
{
|
|
74
|
+
name: "rawBatch",
|
|
75
|
+
source: "step-output",
|
|
76
|
+
fromStep: "fetch-interfaces",
|
|
77
|
+
required: true,
|
|
78
|
+
},
|
|
79
|
+
],
|
|
80
|
+
invoke: {
|
|
81
|
+
serviceType: "normalizer",
|
|
82
|
+
serviceId: "firewall-interface-normalizer",
|
|
83
|
+
},
|
|
84
|
+
normalizedOutput: {
|
|
85
|
+
enabled: true,
|
|
86
|
+
mapper: "firewall-interface-normalizer",
|
|
87
|
+
outputType: "FirewallInterface",
|
|
88
|
+
},
|
|
89
|
+
snapshotOutput: {
|
|
90
|
+
saveMode: "snapshot",
|
|
91
|
+
finalization: "after-normalization",
|
|
92
|
+
},
|
|
93
|
+
statusBehavior: {
|
|
94
|
+
idempotent: true,
|
|
95
|
+
retryable: true,
|
|
96
|
+
completionCondition: "all-raw-normalized",
|
|
97
|
+
},
|
|
98
|
+
errorBehavior: {
|
|
99
|
+
onApiError: "retry",
|
|
100
|
+
maxRetries: 2,
|
|
101
|
+
onMissingDependency: "block",
|
|
102
|
+
onPartialData: "fail",
|
|
103
|
+
onNormalizationError: "fail",
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
/** Minimal auth + scopes stubs so the fetch-interfaces example graph can validate. */
|
|
107
|
+
export const authenticateStep = {
|
|
108
|
+
id: "authenticate",
|
|
109
|
+
name: "Authenticate",
|
|
110
|
+
purpose: "Obtain an API token for the connector.",
|
|
111
|
+
category: "connector",
|
|
112
|
+
serviceType: "health-provider",
|
|
113
|
+
dependsOn: [],
|
|
114
|
+
requiredInputs: [
|
|
115
|
+
{ name: "baseUrl", source: "connector-config", required: true },
|
|
116
|
+
{ name: "credentials", source: "connector-config", required: true },
|
|
117
|
+
],
|
|
118
|
+
apiCalls: [
|
|
119
|
+
{ method: "POST", url: "<baseUrl>/api/auth", pagination: false },
|
|
120
|
+
],
|
|
121
|
+
statusBehavior: {
|
|
122
|
+
idempotent: true,
|
|
123
|
+
retryable: true,
|
|
124
|
+
completionCondition: "token-issued",
|
|
125
|
+
},
|
|
126
|
+
errorBehavior: {
|
|
127
|
+
onApiError: "retry",
|
|
128
|
+
maxRetries: 3,
|
|
129
|
+
onMissingDependency: "block",
|
|
130
|
+
onPartialData: "fail",
|
|
131
|
+
onNormalizationError: "fail",
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
export const fetchDeviceIdentityStep = {
|
|
135
|
+
id: "fetch-device-identity",
|
|
136
|
+
name: "Fetch Device Identity",
|
|
137
|
+
purpose: "Resolve the source device id used by later fetch steps.",
|
|
138
|
+
category: "connector",
|
|
139
|
+
serviceType: "entity-provider",
|
|
140
|
+
dependsOn: ["authenticate"],
|
|
141
|
+
requiredInputs: [
|
|
142
|
+
{ name: "baseUrl", source: "connector-config", required: true },
|
|
143
|
+
{
|
|
144
|
+
name: "authToken",
|
|
145
|
+
source: "step-output",
|
|
146
|
+
fromStep: "authenticate",
|
|
147
|
+
required: true,
|
|
148
|
+
},
|
|
149
|
+
],
|
|
150
|
+
apiCalls: [
|
|
151
|
+
{ method: "GET", url: "<baseUrl>/api/v1/system", auth: "authToken" },
|
|
152
|
+
],
|
|
153
|
+
statusBehavior: {
|
|
154
|
+
idempotent: true,
|
|
155
|
+
retryable: true,
|
|
156
|
+
completionCondition: "identity-fetched",
|
|
157
|
+
},
|
|
158
|
+
errorBehavior: {
|
|
159
|
+
onApiError: "retry",
|
|
160
|
+
maxRetries: 3,
|
|
161
|
+
onMissingDependency: "block",
|
|
162
|
+
onPartialData: "fail",
|
|
163
|
+
onNormalizationError: "fail",
|
|
164
|
+
},
|
|
165
|
+
};
|
|
166
|
+
export const fetchScopesStep = {
|
|
167
|
+
id: "fetch-scopes",
|
|
168
|
+
name: "Fetch Scopes",
|
|
169
|
+
purpose: "List available collection scopes (VSYS/VDOM/tenants).",
|
|
170
|
+
category: "connector",
|
|
171
|
+
serviceType: "scope-provider",
|
|
172
|
+
dependsOn: ["authenticate"],
|
|
173
|
+
requiredInputs: [
|
|
174
|
+
{ name: "baseUrl", source: "connector-config", required: true },
|
|
175
|
+
{
|
|
176
|
+
name: "authToken",
|
|
177
|
+
source: "step-output",
|
|
178
|
+
fromStep: "authenticate",
|
|
179
|
+
required: true,
|
|
180
|
+
},
|
|
181
|
+
],
|
|
182
|
+
apiCalls: [
|
|
183
|
+
{ method: "GET", url: "<baseUrl>/api/v1/scopes", auth: "authToken" },
|
|
184
|
+
],
|
|
185
|
+
statusBehavior: {
|
|
186
|
+
idempotent: true,
|
|
187
|
+
retryable: true,
|
|
188
|
+
completionCondition: "scopes-listed",
|
|
189
|
+
},
|
|
190
|
+
errorBehavior: {
|
|
191
|
+
onApiError: "retry",
|
|
192
|
+
maxRetries: 3,
|
|
193
|
+
onMissingDependency: "block",
|
|
194
|
+
onPartialData: "fail",
|
|
195
|
+
onNormalizationError: "fail",
|
|
196
|
+
},
|
|
197
|
+
};
|
|
198
|
+
/** Example connector + data graph for docs and tests. */
|
|
199
|
+
export const exampleFirewallPipelineSteps = [
|
|
200
|
+
authenticateStep,
|
|
201
|
+
fetchDeviceIdentityStep,
|
|
202
|
+
fetchScopesStep,
|
|
203
|
+
fetchInterfacesStep,
|
|
204
|
+
normalizeInterfacesStep,
|
|
205
|
+
];
|
|
206
|
+
//# sourceMappingURL=fetch-interfaces.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch-interfaces.js","sourceRoot":"","sources":["../../src/examples/fetch-interfaces.ts"],"names":[],"mappings":"AAEA,iFAAiF;AACjF,MAAM,CAAC,MAAM,mBAAmB,GAAiB;IAC/C,EAAE,EAAE,kBAAkB;IACtB,IAAI,EAAE,kBAAkB;IACxB,OAAO,EAAE,+DAA+D;IACxE,QAAQ,EAAE,WAAW;IACrB,WAAW,EAAE,iBAAiB;IAC9B,MAAM,EAAE,SAAS;IACjB,UAAU,EAAE,qBAAqB;IACjC,SAAS,EAAE,CAAC,cAAc,EAAE,uBAAuB,EAAE,cAAc,CAAC;IACpE,cAAc,EAAE;QACd,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,kBAAkB,EAAE,QAAQ,EAAE,IAAI,EAAE;QAC/D;YACE,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,cAAc;YACxB,QAAQ,EAAE,IAAI;SACf;QACD;YACE,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,cAAc;YACxB,QAAQ,EAAE,KAAK;SAChB;KACF;IACD,QAAQ,EAAE;QACR;YACE,MAAM,EAAE,KAAK;YACb,GAAG,EAAE,6BAA6B;YAClC,IAAI,EAAE,WAAW;YACjB,UAAU,EAAE,KAAK;SAClB;KACF;IACD,SAAS,EAAE;QACT,UAAU,EAAE,yBAAyB;QACrC,QAAQ,EAAE,QAAQ;QAClB,eAAe,EAAE,IAAI;KACtB;IACD,gBAAgB,EAAE;QAChB,OAAO,EAAE,IAAI;QACb,UAAU,EAAE,mBAAmB;KAChC;IACD,cAAc,EAAE;QACd,UAAU,EAAE,+BAA+B;QAC3C,QAAQ,EAAE,UAAU;QACpB,YAAY,EAAE,mCAAmC;KAClD;IACD,cAAc,EAAE;QACd,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,IAAI;QACf,cAAc,EAAE,KAAK;QACrB,mBAAmB,EAAE,oBAAoB;KAC1C;IACD,aAAa,EAAE;QACb,UAAU,EAAE,OAAO;QACnB,UAAU,EAAE,CAAC;QACb,mBAAmB,EAAE,OAAO;QAC5B,aAAa,EAAE,cAAc;QAC7B,oBAAoB,EAAE,iCAAiC;KACxD;CACF,CAAC;AAEF,6EAA6E;AAC7E,MAAM,CAAC,MAAM,uBAAuB,GAAiB;IACnD,EAAE,EAAE,sBAAsB;IAC1B,IAAI,EAAE,sBAAsB;IAC5B,OAAO,EAAE,wEAAwE;IACjF,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,YAAY;IACzB,SAAS,EAAE,+BAA+B;IAC1C,UAAU,EAAE,qBAAqB;IACjC,SAAS,EAAE,CAAC,kBAAkB,CAAC;IAC/B,cAAc,EAAE;QACd;YACE,IAAI,EAAE,UAAU;YAChB,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,kBAAkB;YAC5B,QAAQ,EAAE,IAAI;SACf;KACF;IACD,MAAM,EAAE;QACN,WAAW,EAAE,YAAY;QACzB,SAAS,EAAE,+BAA+B;KAC3C;IACD,gBAAgB,EAAE;QAChB,OAAO,EAAE,IAAI;QACb,MAAM,EAAE,+BAA+B;QACvC,UAAU,EAAE,mBAAmB;KAChC;IACD,cAAc,EAAE;QACd,QAAQ,EAAE,UAAU;QACpB,YAAY,EAAE,qBAAqB;KACpC;IACD,cAAc,EAAE;QACd,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,IAAI;QACf,mBAAmB,EAAE,oBAAoB;KAC1C;IACD,aAAa,EAAE;QACb,UAAU,EAAE,OAAO;QACnB,UAAU,EAAE,CAAC;QACb,mBAAmB,EAAE,OAAO;QAC5B,aAAa,EAAE,MAAM;QACrB,oBAAoB,EAAE,MAAM;KAC7B;CACF,CAAC;AAEF,sFAAsF;AACtF,MAAM,CAAC,MAAM,gBAAgB,GAAiB;IAC5C,EAAE,EAAE,cAAc;IAClB,IAAI,EAAE,cAAc;IACpB,OAAO,EAAE,wCAAwC;IACjD,QAAQ,EAAE,WAAW;IACrB,WAAW,EAAE,iBAAiB;IAC9B,SAAS,EAAE,EAAE;IACb,cAAc,EAAE;QACd,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,kBAAkB,EAAE,QAAQ,EAAE,IAAI,EAAE;QAC/D,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,kBAAkB,EAAE,QAAQ,EAAE,IAAI,EAAE;KACpE;IACD,QAAQ,EAAE;QACR,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,oBAAoB,EAAE,UAAU,EAAE,KAAK,EAAE;KACjE;IACD,cAAc,EAAE;QACd,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,IAAI;QACf,mBAAmB,EAAE,cAAc;KACpC;IACD,aAAa,EAAE;QACb,UAAU,EAAE,OAAO;QACnB,UAAU,EAAE,CAAC;QACb,mBAAmB,EAAE,OAAO;QAC5B,aAAa,EAAE,MAAM;QACrB,oBAAoB,EAAE,MAAM;KAC7B;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAiB;IACnD,EAAE,EAAE,uBAAuB;IAC3B,IAAI,EAAE,uBAAuB;IAC7B,OAAO,EAAE,yDAAyD;IAClE,QAAQ,EAAE,WAAW;IACrB,WAAW,EAAE,iBAAiB;IAC9B,SAAS,EAAE,CAAC,cAAc,CAAC;IAC3B,cAAc,EAAE;QACd,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,kBAAkB,EAAE,QAAQ,EAAE,IAAI,EAAE;QAC/D;YACE,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,cAAc;YACxB,QAAQ,EAAE,IAAI;SACf;KACF;IACD,QAAQ,EAAE;QACR,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,yBAAyB,EAAE,IAAI,EAAE,WAAW,EAAE;KACrE;IACD,cAAc,EAAE;QACd,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,IAAI;QACf,mBAAmB,EAAE,kBAAkB;KACxC;IACD,aAAa,EAAE;QACb,UAAU,EAAE,OAAO;QACnB,UAAU,EAAE,CAAC;QACb,mBAAmB,EAAE,OAAO;QAC5B,aAAa,EAAE,MAAM;QACrB,oBAAoB,EAAE,MAAM;KAC7B;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAiB;IAC3C,EAAE,EAAE,cAAc;IAClB,IAAI,EAAE,cAAc;IACpB,OAAO,EAAE,uDAAuD;IAChE,QAAQ,EAAE,WAAW;IACrB,WAAW,EAAE,gBAAgB;IAC7B,SAAS,EAAE,CAAC,cAAc,CAAC;IAC3B,cAAc,EAAE;QACd,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,kBAAkB,EAAE,QAAQ,EAAE,IAAI,EAAE;QAC/D;YACE,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,cAAc;YACxB,QAAQ,EAAE,IAAI;SACf;KACF;IACD,QAAQ,EAAE;QACR,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,yBAAyB,EAAE,IAAI,EAAE,WAAW,EAAE;KACrE;IACD,cAAc,EAAE;QACd,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,IAAI;QACf,mBAAmB,EAAE,eAAe;KACrC;IACD,aAAa,EAAE;QACb,UAAU,EAAE,OAAO;QACnB,UAAU,EAAE,CAAC;QACb,mBAAmB,EAAE,OAAO;QAC5B,aAAa,EAAE,MAAM;QACrB,oBAAoB,EAAE,MAAM;KAC7B;CACF,CAAC;AAEF,yDAAyD;AACzD,MAAM,CAAC,MAAM,4BAA4B,GAAmB;IAC1D,gBAAgB;IAChB,uBAAuB;IACvB,eAAe;IACf,mBAAmB;IACnB,uBAAuB;CACxB,CAAC"}
|
package/dist/graph.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { PipelineStep } from "./types.js";
|
|
2
|
+
export declare class PipelineStepGraphError extends Error {
|
|
3
|
+
constructor(message: string);
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* Topological order of steps (dependencies first).
|
|
7
|
+
* Throws if the graph is invalid or has a cycle.
|
|
8
|
+
*/
|
|
9
|
+
export declare function topologicalOrder(steps: PipelineStep[]): PipelineStep[];
|
|
10
|
+
/**
|
|
11
|
+
* Steps whose dependencies are all in `completedStepIds` and that are not yet completed.
|
|
12
|
+
*/
|
|
13
|
+
export declare function readySteps(steps: PipelineStep[], completedStepIds: Iterable<string>): PipelineStep[];
|
|
14
|
+
/** True when every dependency of `stepId` is in `completedStepIds`. */
|
|
15
|
+
export declare function dependenciesSatisfied(steps: PipelineStep[], stepId: string, completedStepIds: Iterable<string>): boolean;
|
|
16
|
+
//# sourceMappingURL=graph.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graph.d.ts","sourceRoot":"","sources":["../src/graph.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAG/C,qBAAa,sBAAuB,SAAQ,KAAK;gBACnC,OAAO,EAAE,MAAM;CAI5B;AAgBD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,YAAY,EAAE,CA2CtE;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,KAAK,EAAE,YAAY,EAAE,EACrB,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,GACjC,YAAY,EAAE,CAMhB;AAED,uEAAuE;AACvE,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,YAAY,EAAE,EACrB,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,GACjC,OAAO,CAKT"}
|